diff --git a/.dockerignore b/.dockerignore index 2e0a5b855..33cd19e43 100644 --- a/.dockerignore +++ b/.dockerignore @@ -35,3 +35,6 @@ build/ # Note: sites/*/instance_seed/, static/images/, static/external_cache/ ARE # shipped — those are the runtime assets. They live in .gitignore (managed # via HF), not here. + +# Review evidence is not part of the runtime image. +review-reports/ diff --git a/AGENTS.md b/AGENTS.md index dd8e527ed..9e38a9759 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,17 +48,17 @@ Inside the image, sites live at `/opt/WebSyn//`. The path predates the ren # fresh clone ./scripts/fetch_assets.sh # pulls assets from HF ./scripts/build.sh # docker build -t webharbor:dev . -docker run -d -p 8101:8101 -p 40000-40027:40000-40027 webharbor:dev +docker run -d -p 8101:8101 -p 40000-40028:40000-40028 webharbor:dev ``` Or use the published image directly: ```bash -docker run -d -p 8101:8101 -p 40000-40027:40000-40027 \ +docker run -d -p 8101:8101 -p 40000-40028:40000-40028 \ battalion7244/webharbor:latest ``` -Sites are on `40000`-`40027` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane: +Sites are on `40000`-`40028` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane: | Method | Path | Purpose | |--------|---------------------|-------------------------------------------| @@ -136,13 +136,13 @@ python3 -m py_compile sites//app.py # 3. run on alt ports (don't collide with anything you already have running) docker run -d --rm --name wh-test \ - -p 8201:8101 -p 41000-41027:40000-40027 webharbor:dev + -p 8201:8101 -p 41000-41028:40000-40028 webharbor:dev # 4. control plane healthy, all sites alive curl -s http://localhost:8201/health | python3 -m json.tool | head # 5. every site renders 200 -for p in $(seq 41000 41027); do +for p in $(seq 41000 41028); do curl -so /dev/null -w "$p:%{http_code}\n" http://localhost:$p/ done diff --git a/CLAUDE.md b/CLAUDE.md index 94d397d1b..5e84cb62b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,4 +16,4 @@ The full agent guide is loaded above via `@AGENTS.md`. The notes below apply onl ## Existing containers -If a container is already running on `:8101` / `:40000-40027`, treat it as the user's working environment — don't `docker stop` or `docker rm` it without explicit confirmation. Spin up your test container under a different name on alt ports (`:8201`, `:41000-41027`). +If a container is already running on `:8101` / `:40000-40028`, treat it as the user's working environment — don't `docker stop` or `docker rm` it without explicit confirmation. Spin up your test container under a different name on alt ports (`:8201`, `:41000-41028`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3871b5e89..22e4c1c25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ git clone https://github.com//webharbor && cd webharbor ./scripts/fetch_assets.sh # pull current assets ./scripts/new_site.py mywebsite # OR edit an existing site ./scripts/build.sh && docker run -d --rm \ - -p 8101:8101 -p 40000-40027:40000-40027 webharbor:dev + -p 8101:8101 -p 40000-40028:40000-40028 webharbor:dev # iterate locally... ./scripts/extract_assets.sh ../webharbor-static-pr/ # split assets out diff --git a/Dockerfile b/Dockerfile index d35d70bcd..6672ba941 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # WebHarbor — slim, self-contained image. -# 28 Flask mirror sites + control plane on :8101. +# 29 Flask mirror sites + control plane on :8101. FROM python:3.12-slim-bookworm @@ -92,6 +92,15 @@ os.makedirs('instance_seed', exist_ok=True); \ shutil.copy2('instance/rotten_tomatoes.db', 'instance_seed/rotten_tomatoes.db'); \ print('Rotten Tomatoes seed DB generated at build time.')" && rm -rf /opt/WebSyn/rotten_tomatoes/instance -EXPOSE 8101 40000-40027 +# Adopt-a-Pet freezes its relational catalog and captured real-site media. +RUN test -n "$(ls -A /opt/WebSyn/adopt_a_pet/static/images)" && \ + cd /opt/WebSyn/adopt_a_pet && rm -rf instance instance_seed && python3 -c "\ +import app; \ +import os, shutil; \ +os.makedirs('instance_seed', exist_ok=True); \ +shutil.copy2('instance/adopt_a_pet.db', 'instance_seed/adopt_a_pet.db'); \ +print('Adopt-a-Pet seed DB generated at build time.')" && rm -rf /opt/WebSyn/adopt_a_pet/instance + +EXPOSE 8101 40000-40028 CMD ["/opt/websyn_start.sh"] diff --git a/README.md b/README.md index 4189c2099..ee08d0eb7 100644 --- a/README.md +++ b/README.md @@ -36,17 +36,17 @@ WebHarbor takes a different approach. We leverage coding agent (e.g., Claude Cod - **Deep features unlocked** — carts, checkouts, accounts, all fully testable - **Evolving** — harder tasks drive richer mirrors; the environment grows with agents - **RL-ready** — sub-second database resets between rollouts -- **Community-driven** — 28 sites today, scaling to 100+ together +- **Community-driven** — 29 sites today, scaling to 100+ together ## 🚀 Quickstart One command to run all web environments: ```bash -docker run -p 8101:8101 -p 40000-40027:40000-40027 battalion7244/webharbor:latest +docker run -p 8101:8101 -p 40000-40028:40000-40028 battalion7244/webharbor:latest ``` -Then point your agent at `http://localhost:40000` through `http://localhost:40027` to explore 28 local mirrors of WebVoyager sites: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, ESPN, Merriam-Webster, IKEA, Phys.org, Target, TED, Ohio State University, Rotten Tomatoes, Compass, Walmart Careers, FedEx, WebMD Doctor, Healthline, and Kaggle`. +Then point your agent at `http://localhost:40000` through `http://localhost:40027` to explore 29 local mirrors of WebVoyager sites: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, ESPN, Merriam-Webster, IKEA, Phys.org, Target, TED, Ohio State University, Rotten Tomatoes, Compass, Walmart Careers, FedEx, WebMD Doctor, Healthline, Kaggle, and Adopt-a-Pet`. For sub-second reset between rollouts, expose the control plane and call `/reset/`: diff --git a/agent_demo/README.md b/agent_demo/README.md index ee26ed3ab..0570c745c 100644 --- a/agent_demo/README.md +++ b/agent_demo/README.md @@ -19,7 +19,7 @@ export OPENAI_BASE_URL=https://api.openai.com/v1 # or your Azure / vLLM endpoi ## Run a task -WebHarbor must already be running locally (`docker run -p 8101:8101 -p 40000-40027:40000-40027 battalion7244/webharbor:latest`). +WebHarbor must already be running locally (`docker run -p 8101:8101 -p 40000-40028:40000-40028 battalion7244/webharbor:latest`). Run a single task from a site's `tasks.jsonl`: diff --git a/control_server.py b/control_server.py index 939613ecb..e4f07f064 100644 --- a/control_server.py +++ b/control_server.py @@ -30,6 +30,7 @@ 'ikea', 'phys_org', 'target', 'ted', 'osu', 'rotten_tomatoes', 'compass', 'walmart_careers', 'fedex', 'webmd_doctor', 'healthline', 'kaggle', + 'adopt_a_pet', ] BASE_PORT = 40000 WEBSYN_DIR = '/opt/WebSyn' diff --git a/review-reports/ADOPT-A-PET-FINAL-AUDIT.md b/review-reports/ADOPT-A-PET-FINAL-AUDIT.md new file mode 100644 index 000000000..363fc6124 --- /dev/null +++ b/review-reports/ADOPT-A-PET-FINAL-AUDIT.md @@ -0,0 +1,39 @@ +# Adopt-a-Pet final task and UI audit + +Audit date: 2026-09-10. Branch: `add-adopt-a-pet-mirror`. + +Each task began with `POST /reset/adopt_a_pet`, cleared browser cookies, and loaded `http://localhost:41024/`. Playwright used visible roles, labels, and link names for all searching, filtering, pagination-dependent selection, login, registration, favorites, alerts, adoption inquiries, shelters, and account confirmation. No destination URL, database record, or source answer replaced a user interaction. + +| Tasks | Result | Evidence | +|---|---|---| +| AdoptAPet--0 through AdoptAPet--19 | 20/20 pass | Structured step log and one final screenshot per task in `outputs/adopt-a-pet-review/` | + +The paths covered: location and species search; breed, sex, age, and size filters; pet details; multi-profile comparisons; rescue details; Breed 101; favorites add/remove; adoption inquiry submission; New Pet Alert creation; account registration; account persistence; and editorial resources. + +## Hardening results + +- De-leak: cards show only name, breed, broad age group, sex, and location. Exact age, fee, compatibility, color, rescue contact, and application state require the appropriate detail or account flow. +- Distractors: Arizona has nine pets across multiple cities and breeds; Phoenix, Scottsdale, New York, Seattle, Austin, and Miami each have multiple plausible candidates. Filters narrow genuine sets rather than routing directly to a target. +- Catalog breadth: 20 pets, 6 shelters, 12 primary breeds, two species, four age groups, three sizes, both sexes, six metro areas, and 20 distinct captured images. +- Cross-field consistency: each pet’s city/state agrees with its linked rescue region; shared relational rows drive search, detail, shelter, favorites, alerts, inquiries, and account pages. +- Leak archetypes checked: direct prompt answer; artificial first-result target; pre-sorted winner; result count as answer; detail on card; insufficient distractors; route/slug answer; hidden attribute; accessible-name answer; placeholder answer; mutation-free success; visit-only completion; unrelated state acceptance. None were found. +- Hard reasoning tasks include 2, 3, 5, 12, 14, and 19. Tasks 6–9, 15, and 16 additionally require exact persistent state changes. + +## Visual and asset audit + +Home, search, pet detail, shelter list, and account were checked at 1440, 390, and 320 CSS pixels. Across all 15 combinations there were zero broken images and zero horizontal-overflow failures. Representative committed screenshots: + +- `review-reports/assets/adopt-a-pet-homepage-1440.png` +- `review-reports/assets/adopt-a-pet-homepage-390.png` +- `review-reports/assets/adopt-a-pet-homepage-320.png` + +All pet and editorial photos were harvested from the live homepage asset inventory. Every pet listing uses a distinct captured image; no placeholder or generated image is present. + +## Reset proof + +After the final task audit, both databases had MD5 `376b1ca1b6197540c9e01f7893096287`: + +- `/opt/WebSyn/adopt_a_pet/instance/adopt_a_pet.db` +- `/opt/WebSyn/adopt_a_pet/instance_seed/adopt_a_pet.db` + +Calling `seed_benchmark_users()`, `seed_database()`, and `seed_user_state()` on the initialized runtime did not change that hash. diff --git a/review-reports/assets/adopt-a-pet-homepage-1440.png b/review-reports/assets/adopt-a-pet-homepage-1440.png new file mode 100644 index 000000000..50c693a7f Binary files /dev/null and b/review-reports/assets/adopt-a-pet-homepage-1440.png differ diff --git a/review-reports/assets/adopt-a-pet-homepage-320.png b/review-reports/assets/adopt-a-pet-homepage-320.png new file mode 100644 index 000000000..6480bb4f0 Binary files /dev/null and b/review-reports/assets/adopt-a-pet-homepage-320.png differ diff --git a/review-reports/assets/adopt-a-pet-homepage-390.png b/review-reports/assets/adopt-a-pet-homepage-390.png new file mode 100644 index 000000000..525d87f32 Binary files /dev/null and b/review-reports/assets/adopt-a-pet-homepage-390.png differ diff --git a/review-reports/assets/adopt-a-pet-photo-remediation-1440.png b/review-reports/assets/adopt-a-pet-photo-remediation-1440.png new file mode 100644 index 000000000..da2520b69 Binary files /dev/null and b/review-reports/assets/adopt-a-pet-photo-remediation-1440.png differ diff --git a/review-reports/assets/adopt-a-pet-photo-remediation-390.png b/review-reports/assets/adopt-a-pet-photo-remediation-390.png new file mode 100644 index 000000000..b9153a427 Binary files /dev/null and b/review-reports/assets/adopt-a-pet-photo-remediation-390.png differ diff --git a/sites/adopt_a_pet/_health.py b/sites/adopt_a_pet/_health.py new file mode 100644 index 000000000..7500374e0 --- /dev/null +++ b/sites/adopt_a_pet/_health.py @@ -0,0 +1,3 @@ +"""Per-site health probe (optional, called by control_server).""" +def health(): + return {"ok": True, "site": "adopt_a_pet"} diff --git a/sites/adopt_a_pet/app.py b/sites/adopt_a_pet/app.py new file mode 100644 index 000000000..00f9c660e --- /dev/null +++ b/sites/adopt_a_pet/app.py @@ -0,0 +1,188 @@ +"""Deterministic Adopt-a-Pet mirror with search, accounts, favorites and applications.""" +import os,re,secrets +from functools import wraps +from urllib.parse import quote_plus,urlsplit +from flask import Flask,abort,flash,redirect,render_template,request,session,url_for +from flask_sqlalchemy import SQLAlchemy +from flask_wtf.csrf import CSRFProtect +from sqlalchemy import event +from sqlalchemy.engine import Engine +from werkzeug.security import check_password_hash,generate_password_hash +BASE_DIR=os.path.dirname(os.path.abspath(__file__)); app=Flask(__name__,instance_path=os.path.join(BASE_DIR,'instance')) +app.config.update(SECRET_KEY=os.environ.get('ADOPT_A_PET_SECRET_KEY') or secrets.token_hex(32),SQLALCHEMY_DATABASE_URI='sqlite:///adopt_a_pet.db',SQLALCHEMY_TRACK_MODIFICATIONS=False,MAX_CONTENT_LENGTH=256*1024,WTF_CSRF_TIME_LIMIT=None); db=SQLAlchemy(app); csrf=CSRFProtect(app) +@event.listens_for(Engine,'connect') +def _sqlite_foreign_keys(connection,_record): + cursor=connection.cursor(); cursor.execute('PRAGMA foreign_keys=ON'); cursor.close() +class User(db.Model): + id=db.Column(db.Integer,primary_key=True); email=db.Column(db.String(120),unique=True,nullable=False); name=db.Column(db.String(80),nullable=False); password_hash=db.Column(db.String(255),nullable=False) +class Shelter(db.Model): + id=db.Column(db.Integer,primary_key=True); name=db.Column(db.String(120),unique=True,nullable=False); city=db.Column(db.String(60),nullable=False); state=db.Column(db.String(2),nullable=False); phone=db.Column(db.String(20),nullable=False); email=db.Column(db.String(120),nullable=False) +class Pet(db.Model): + id=db.Column(db.Integer,primary_key=True); slug=db.Column(db.String(120),unique=True,nullable=False); name=db.Column(db.String(60),nullable=False); species=db.Column(db.String(20),nullable=False); breed=db.Column(db.String(100),nullable=False); secondary_breed=db.Column(db.String(100)); sex=db.Column(db.String(10),nullable=False); age_group=db.Column(db.String(20),nullable=False); age_months=db.Column(db.Integer,nullable=False); size=db.Column(db.String(20),nullable=False); color=db.Column(db.String(40),nullable=False); city=db.Column(db.String(60),nullable=False); state=db.Column(db.String(2),nullable=False); postal=db.Column(db.String(10),nullable=False); fee=db.Column(db.Integer,nullable=False); image=db.Column(db.String(100),nullable=False); description=db.Column(db.Text,nullable=False); house_trained=db.Column(db.Boolean,nullable=False); good_dogs=db.Column(db.Boolean,nullable=False); good_cats=db.Column(db.Boolean,nullable=False); good_children=db.Column(db.Boolean,nullable=False); shelter_id=db.Column(db.Integer,db.ForeignKey('shelter.id'),nullable=False); shelter=db.relationship(Shelter,backref='pets') +class Favorite(db.Model): + id=db.Column(db.Integer,primary_key=True); user_id=db.Column(db.Integer,db.ForeignKey('user.id'),nullable=False); pet_id=db.Column(db.Integer,db.ForeignKey('pet.id'),nullable=False); __table_args__=(db.UniqueConstraint('user_id','pet_id'),) +class Application(db.Model): + id=db.Column(db.Integer,primary_key=True); user_id=db.Column(db.Integer,db.ForeignKey('user.id'),nullable=False); pet_id=db.Column(db.Integer,db.ForeignKey('pet.id'),nullable=False); housing=db.Column(db.String(30),nullable=False); experience=db.Column(db.Text,nullable=False); phone=db.Column(db.String(20),nullable=False); status=db.Column(db.String(20),default='Submitted',nullable=False); __table_args__=(db.UniqueConstraint('user_id','pet_id'),) +class PetAlert(db.Model): + id=db.Column(db.Integer,primary_key=True); user_id=db.Column(db.Integer,db.ForeignKey('user.id'),nullable=False); species=db.Column(db.String(20),nullable=False); breed=db.Column(db.String(80),nullable=False); postal=db.Column(db.String(10),nullable=False); radius=db.Column(db.Integer,nullable=False) +HOUSING_OPTIONS=('Own home','Rent with permission','Other') +SPECIES_OPTIONS=('Dog','Cat') +RADIUS_OPTIONS=(10,25,50,100) +USERS=[('alice.j@test.com','Alice Johnson'),('bob.smith@test.com','Bob Smith'),('carol.w@test.com','Carol Williams'),('david.b@test.com','David Brown')] +SHELTERS=[('Desert Paws Rescue','Phoenix','AZ','602-555-0141','hello@desertpaws.test'),('Happy Tails Alliance','Scottsdale','AZ','480-555-0128','adopt@happytails.test'),('City Friends Shelter','New York','NY','212-555-0164','pets@cityfriends.test'),('Pacific Animal Haven','Seattle','WA','206-555-0119','info@pacifichaven.test'),('Lone Star Companions','Austin','TX','512-555-0182','team@lonestar.test'),('Sunshine Pet Rescue','Miami','FL','305-555-0136','adopt@sunshine.test')] +PETS=[ +('sirius','Sirius','Dog','Chihuahua','Terrier','Male','Senior',108,'Small','Tan','Scottsdale','AZ','85251',175,'pets/sirius.avif',True,True,True,False),('waymo','Waymo','Dog','American Pit Bull Terrier','Mixed Breed','Male','Adult',24,'Large','Gray','Phoenix','AZ','85004',225,'pets/waymo.avif',True,True,False,True),('casper','Casper','Cat','Colorpoint Shorthair',None,'Male','Adult',48,'Medium','Cream','Mesa','AZ','85201',125,'pets/casper.avif',True,False,True,True),('neo','Neo','Cat','Domestic Shorthair',None,'Male','Kitten',7,'Small','Black','Scottsdale','AZ','85250',110,'pets/neo.avif',True,True,True,True),('amba','Amba','Cat','Domestic Mediumhair',None,'Female','Kitten',5,'Small','Tabby','Arizona City','AZ','85123',95,'pets/amba.avif',True,True,True,True),('cinders','Cinders','Cat','Domestic Shorthair',None,'Female','Adult',85,'Medium','Tortoiseshell','Sedona','AZ','86336',120,'pets/cinders.jpg',True,False,True,False),('arno','Arno','Dog','German Shepherd Dog','Mixed Breed','Male','Adult',43,'Large','Black and Tan','Casa Grande','AZ','85122',200,'pets/arno.avif',True,True,False,True),('batman','Batman','Dog','Chihuahua','Yorkshire Terrier','Male','Adult',36,'Small','Black','Tucson','AZ','85701',165,'pets/batman.jpg',True,True,True,False),('horus','Horus','Dog','Pointer','Labrador Retriever','Male','Young',16,'Large','White and Black','Phoenix','AZ','85006',210,'pets/horus.avif',True,True,False,True), +('luna','Luna','Dog','Beagle',None,'Female','Young',14,'Medium','Tricolor','New York','NY','10011',250,'pets/luna.jpg',True,True,True,True),('milo','Milo','Cat','Maine Coon',None,'Male','Adult',38,'Large','Orange','New York','NY','10003',150,'pets/milo.jpg',True,False,True,True),('daisy','Daisy','Dog','Golden Retriever',None,'Female','Adult',30,'Large','Golden','Seattle','WA','98109',275,'pets/daisy.jpg',True,True,True,True),('pepper','Pepper','Cat','Domestic Shorthair',None,'Female','Young',13,'Small','Black and White','Seattle','WA','98101',130,'pets/pepper.avif',True,True,True,False),('archie','Archie','Dog','Australian Shepherd',None,'Male','Young',18,'Medium','Merle','Austin','TX','78704',240,'pets/archie.jpg',True,True,False,True),('ruby','Ruby','Dog','Boxer','Mixed Breed','Female','Adult',42,'Large','Fawn','Austin','TX','78702',215,'pets/ruby.jpg',True,True,False,False),('olive','Olive','Cat','Siamese',None,'Female','Adult',27,'Medium','Seal Point','Miami','FL','33130',145,'pets/olive.jpg',True,False,True,True),('teddy','Teddy','Dog','Poodle','Mixed Breed','Male','Senior',96,'Small','White','Miami','FL','33133',185,'pets/teddy.jpg',True,True,True,True),('winston','Winston','Dog','Chihuahua','Mixed Breed','Male','Adult',60,'Small','Tan','Tempe','AZ','85281',230,'pets/winston.avif',True,True,True,False),('yuki','Yuki','Dog','Chihuahua','Mixed Breed','Female','Senior',120,'Small','Cream','Glendale','AZ','85301',205,'pets/yuki.jpg',True,False,True,True),('zorro','Zorro','Dog','Chihuahua','Terrier','Male','Adult',72,'Small','Black','Prescott','AZ','86301',220,'pets/zorro.jpg',True,True,False,False)] +def seed_benchmark_users(): + if User.query.first(): return + for email,name in USERS: db.session.add(User(email=email,name=name,password_hash=generate_password_hash('TestPass123!'))) + db.session.commit() +def seed_database(): + if Pet.query.first(): return + for row in SHELTERS: db.session.add(Shelter(name=row[0],city=row[1],state=row[2],phone=row[3],email=row[4])) + db.session.flush(); shelters=Shelter.query.all() + for i,p in enumerate(PETS): + shelter_index=(i%2 if p[11]=='AZ' else {'NY':2,'WA':3,'TX':4,'FL':5}[p[11]]) + db.session.add(Pet(slug=p[0],name=p[1],species=p[2],breed=p[3],secondary_breed=p[4],sex=p[5],age_group=p[6],age_months=p[7],size=p[8],color=p[9],city=p[10],state=p[11],postal=p[12],fee=p[13],image=p[14],description=f'{p[1]} is an affectionate {p[6].lower()} {p[2].lower()} who enjoys companionship, gentle play, and a comfortable place to relax.',house_trained=p[15],good_dogs=p[16],good_cats=p[17],good_children=p[18],shelter_id=shelters[shelter_index].id)) + db.session.commit() +def seed_user_state(): + if Favorite.query.first() or Application.query.first() or PetAlert.query.first(): return + alice=User.query.filter_by(email='alice.j@test.com').first(); db.session.add(Favorite(user_id=alice.id,pet_id=Pet.query.filter_by(slug='luna').first().id)); db.session.commit() +with app.app_context(): os.makedirs(app.instance_path,exist_ok=True);db.create_all();seed_benchmark_users();seed_database();seed_user_state() +def user(): + raw=session.get('user_id') + if isinstance(raw,bool) or not isinstance(raw,int): return None # a forged cookie must not reach the ORM + return db.session.get(User,raw) +@app.context_processor +def ctx(): return {'current_user':user()} +def required(fn): + @wraps(fn) + def w(*a,**k): + if not user(): flash('Log in to continue.');return redirect(url_for('login',next=request.path)) + return fn(*a,**k) + return w +def score(q,text): + terms=set(re.findall(r'[a-z0-9]+',q.lower()));tokens=set(re.findall(r'[a-z0-9]+',text.lower()));return len(terms&tokens) +def bounded_text(value,field,maximum,required=True,minimum=0): + """Trimmed form value plus an error string; SQLite does not enforce VARCHAR widths, so the app must.""" + text=(value or '').strip() + if required and not text: return text,f'Enter your {field}.' + if len(text)>maximum: return text,f'{field.capitalize()} must be {maximum} characters or fewer.' + if len(text)') +def pet(slug): + p=Pet.query.filter_by(slug=slug).first_or_404();fav=bool(user() and Favorite.query.filter_by(user_id=user().id,pet_id=p.id).first());return render_template('pet.html',pet=p,favorite=fav) +@app.route('/favorite/',methods=['POST']) +@required +def favorite(slug): + p=Pet.query.filter_by(slug=slug).first_or_404();f=Favorite.query.filter_by(user_id=user().id,pet_id=p.id).first() + if f: db.session.delete(f);flash(f'Removed {p.name} from favorites.') + else: db.session.add(Favorite(user_id=user().id,pet_id=p.id));flash(f'{p.name} was added to favorites.') + db.session.commit();return redirect(safe_referrer() or url_for('pet',slug=slug)) +@app.route('/apply/',methods=['GET','POST']) +@required +def apply(slug): + p=Pet.query.filter_by(slug=slug).first_or_404() + if request.method=='POST': + phone,e1=bounded_text(request.form.get('phone'),'phone number',20) + housing,e2=one_of(request.form.get('housing'),'housing option',HOUSING_OPTIONS) + experience,e3=bounded_text(request.form.get('experience'),'pet experience',2000,minimum=15) + error=e1 or e2 or e3 + if error: return render_template('apply.html',pet=p,error=error),400 + old=Application.query.filter_by(user_id=user().id,pet_id=p.id).first() + if old: flash('You already applied for this pet.') + else: db.session.add(Application(user_id=user().id,pet_id=p.id,housing=housing,experience=experience,phone=phone));db.session.commit();return render_template('application_done.html',pet=p) + return render_template('apply.html',pet=p) +@app.route('/alerts',methods=['GET','POST']) +@required +def alerts(): + if request.method=='POST': + species,e1=one_of(request.form.get('species'),'pet type',SPECIES_OPTIONS) + breed,e2=bounded_text(request.form.get('breed'),'breed',80,required=False) + postal,e3=bounded_text(request.form.get('postal'),'postal code',10) + radius,e4=bounded_int(request.form.get('radius'),'distance',RADIUS_OPTIONS) + if not e3 and not re.fullmatch(r'\d{5}',postal): e3='Enter a five-digit postal code.' + error=e1 or e2 or e3 or e4 + if error: return render_template('alerts.html',error=error),400 + db.session.add(PetAlert(user_id=user().id,species=species,breed=breed,postal=postal,radius=radius));db.session.commit();flash('New Pet Alert created.');return redirect(url_for('account')) + return render_template('alerts.html') +@app.route('/account') +@required +def account(): + favs=db.session.query(Pet).join(Favorite,Favorite.pet_id==Pet.id).filter(Favorite.user_id==user().id).all();apps=db.session.query(Application,Pet).join(Pet,Pet.id==Application.pet_id).filter(Application.user_id==user().id).all();return render_template('account.html',favorites=favs,applications=apps,alerts=PetAlert.query.filter_by(user_id=user().id).all()) +@app.route('/shelters') +def shelters(): + q=request.args.get('q','');items=[s for s in Shelter.query.order_by(Shelter.state,Shelter.city,Shelter.name).all() if not q or score(q,f'{s.name} {s.city} {s.state}')];return render_template('shelters.html',shelters=items,q=q) +@app.route('/shelter/') +def shelter(id): return render_template('shelter.html',shelter=db.session.get(Shelter,id) or abort(404)) +@app.route('/breeds') +def breeds(): return render_template('breeds.html',dogs=sorted({p.breed for p in Pet.query.filter_by(species='Dog')}),cats=sorted({p.breed for p in Pet.query.filter_by(species='Cat')})) +@app.route('/login',methods=['GET','POST']) +def login(): + if request.method=='POST': + u=User.query.filter_by(email=request.form.get('email','').lower().strip()).first() + if u and check_password_hash(u.password_hash,request.form.get('password','')):session['user_id']=u.id;flash('Welcome back!');return redirect(internal_path(request.args.get('next')) or url_for('account')) + flash('Email or password is incorrect.') + return render_template('login.html') +@app.route('/register',methods=['GET','POST']) +def register(): + if request.method=='POST': + email,e1=bounded_text(request.form.get('email','').lower(),'email address',120) + name,e2=bounded_text(request.form.get('name'),'name',80) + password=request.form.get('password','') + # deliberately not email_validator: it rejects RFC 6761 special-use TLDs, and this offline + # mirror's own benchmark accounts are @test.com / @example.test addresses. + if not e1 and not re.fullmatch(r"[^@\s]+@[^@\s.]+(?:\.[^@\s.]+)+",email): e1='Enter a valid email address.' + error=e1 or e2 + if error: return render_template('register.html',error=error),400 + if User.query.filter_by(email=email).first():flash('An account already exists for that email.') + elif len(password)<8:flash('Password must be at least 8 characters.') + elif len(password)>256:flash('Password must be 256 characters or fewer.') + else:u=User(email=email,name=name,password_hash=generate_password_hash(password));db.session.add(u);db.session.commit();session['user_id']=u.id;return redirect(url_for('account')) + return render_template('register.html') +@app.route('/logout',methods=['POST']) +def logout():session.clear();return redirect(url_for('index')) +@app.route('/blog') +def blog(): return render_template('blog.html') +@app.route('/_health') +def health():return {'ok':True,'site':'adopt_a_pet'} +@app.errorhandler(400) +def bad_request(_e):return render_template('400.html'),400 +@app.errorhandler(404) +def not_found(_e):return render_template('404.html'),404 +@app.errorhandler(413) +def too_large(_e):return render_template('400.html'),413 +@app.errorhandler(500) +def server_error(_e):db.session.rollback();return render_template('500.html'),500 +if __name__=='__main__':app.run(host='0.0.0.0',port=int(os.environ.get('PORT',5000))) diff --git a/sites/adopt_a_pet/provenance.json b/sites/adopt_a_pet/provenance.json new file mode 100644 index 000000000..94d264bb4 --- /dev/null +++ b/sites/adopt_a_pet/provenance.json @@ -0,0 +1,280 @@ +{ + "updated_at": "2026-09-13", + "source": "https://www.adoptapet.com/", + "notes": "The catalog is a synthetic offline benchmark. These real representative photographs match the listed species, primary breed and coat appearance; they are not identity or real-world availability claims. Names, age, sex, temperament, fees and shelter data remain the reviewed benchmark facts. Editorial article banners are used only by the blog, never as pet photographs.", + "pet_photos": [ + { + "pet": "amba", + "file": "static/images/pets/amba.avif", + "source_url": "https://www.adoptapet.com/", + "source": "Previously captured Adopt-a-Pet listing photograph, 2026-09-09", + "original_file": "pet-05.avif", + "changes": "Reassigned to the visually matching synthetic listing; original image bytes retained.", + "dimensions": [ + 716, + 680 + ] + }, + { + "pet": "archie", + "file": "static/images/pets/archie.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/a/a7/Blue_Merle_Australian_Shepherd_%22Luke%22.jpg/960px-Blue_Merle_Australian_Shepherd_%22Luke%22.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:Blue_Merle_Australian_Shepherd_%22Luke%22.jpg", + "author": "GizaDog", + "license": "CC BY-SA 3.0", + "license_url": "https://creativecommons.org/licenses/by-sa/3.0", + "source_description": "Blue Merle Australian Shepherd \"Luke\"", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 732 + ] + }, + { + "pet": "arno", + "file": "static/images/pets/arno.avif", + "source_url": "https://media.adoptapet.com/image/upload/d_Fallback-Photo_Dog-v3.png/c_auto,g_auto,w_358,ar_142:135,dpr_2/f_auto,q_auto/[REDACTED_ID_14258b336818]", + "source": "User-provided Adopt-a-Pet reference capture, 2026-09-10", + "capture_file": "assets/aca59924dc6ae3282559.bin", + "changes": "Original image bytes retained; source identifier was redacted by the capture tool.", + "dimensions": [ + 716, + 680 + ] + }, + { + "pet": "batman", + "file": "static/images/pets/batman.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/b/b8/Little_Man_Chihuahua_by_David_Shankbone.jpg/960px-Little_Man_Chihuahua_by_David_Shankbone.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:Little_Man_Chihuahua_by_David_Shankbone.jpg", + "author": "David Shankbone", + "license": "Attribution", + "license_url": "", + "source_description": "A Chihuahua in the grass of Manhattan's East River Park.", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 1169 + ] + }, + { + "pet": "casper", + "file": "static/images/pets/casper.avif", + "source_url": "https://www.adoptapet.com/", + "source": "Previously captured Adopt-a-Pet listing photograph, 2026-09-09", + "original_file": "pet-01.avif", + "changes": "Reassigned to the visually matching synthetic listing; original image bytes retained.", + "dimensions": [ + 716, + 681 + ] + }, + { + "pet": "cinders", + "file": "static/images/pets/cinders.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/f/f1/Tortoiseshell_cat.JPG/960px-Tortoiseshell_cat.JPG?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:Tortoiseshell_cat.JPG", + "author": "Whoosher", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "source_description": "A female tortoiseshell cat", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 720 + ] + }, + { + "pet": "daisy", + "file": "static/images/pets/daisy.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/1/1c/D%C3%BClmen%2C_Hausd%C3%BClmen%2C_Golden_Retriever_--_2022_--_5530.jpg/960px-D%C3%BClmen%2C_Hausd%C3%BClmen%2C_Golden_Retriever_--_2022_--_5530.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:D%C3%BClmen,_Hausd%C3%BClmen,_Golden_Retriever_--_2022_--_5530.jpg", + "author": "Dietmar Rabich", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "source_description": "Golden Retriever in the wood “Schmalo” near Hausdülmen, Dülmen, North Rhine-Westphalia, Germany", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 960 + ] + }, + { + "pet": "horus", + "file": "static/images/pets/horus.avif", + "source_url": "https://www.adoptapet.com/", + "source": "Previously captured Adopt-a-Pet listing photograph, 2026-09-09", + "original_file": "pet-09.avif", + "changes": "Reassigned to the visually matching synthetic listing; original image bytes retained.", + "dimensions": [ + 716, + 680 + ] + }, + { + "pet": "luna", + "file": "static/images/pets/luna.jpg", + "source_url": "https://upload.wikimedia.org/wikipedia/commons/5/50/Beagle_Tricolor_Jessie.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail_unscaled", + "description_url": "https://commons.wikimedia.org/wiki/File:Beagle_Tricolor_Jessie.jpg", + "author": "Mario Kühlich", + "license": "Copyrighted free use", + "license_url": "", + "source_description": "Beagle Weibchen (Tricolor)", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 800, + 600 + ] + }, + { + "pet": "milo", + "file": "static/images/pets/milo.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/b/b5/Maine_Coon_cat_%22Mitts%22.jpg/960px-Maine_Coon_cat_%22Mitts%22.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:Maine_Coon_cat_%22Mitts%22.jpg", + "author": "Signbear999", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "source_description": "An orange and white Maine Coon cat named Mitts resting on the top of the couch.", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 720 + ] + }, + { + "pet": "neo", + "file": "static/images/pets/neo.avif", + "source_url": "https://www.adoptapet.com/", + "source": "Previously captured Adopt-a-Pet listing photograph, 2026-09-09", + "original_file": "pet-07.avif", + "changes": "Reassigned to the visually matching synthetic listing; original image bytes retained.", + "dimensions": [ + 716, + 680 + ] + }, + { + "pet": "olive", + "file": "static/images/pets/olive.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/2/2f/A_classic_seal_point_Siamese_cat.jpg/960px-A_classic_seal_point_Siamese_cat.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:A_classic_seal_point_Siamese_cat.jpg", + "author": "Jwads82", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "source_description": "A Siamese cat with all the classic seal point markings.", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 540 + ] + }, + { + "pet": "pepper", + "file": "static/images/pets/pepper.avif", + "source_url": "https://media.adoptapet.com/image/upload/d_Fallback-Photo_Cat-v3.png/c_auto,g_auto,w_358,ar_142:135,dpr_2/f_auto,q_auto/[REDACTED_ID_968a116a901e]", + "source": "User-provided Adopt-a-Pet reference capture, 2026-09-10", + "capture_file": "assets/e7fce893fafe786a3ac3.bin", + "changes": "Original image bytes retained; source identifier was redacted by the capture tool.", + "dimensions": [ + 716, + 680 + ] + }, + { + "pet": "ruby", + "file": "static/images/pets/ruby.jpg", + "source_url": "https://upload.wikimedia.org/wikipedia/commons/e/e4/Boxer_female_brown.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail_unscaled", + "description_url": "https://commons.wikimedia.org/wiki/File:Boxer_female_brown.jpg", + "author": "Flickr user boxercab", + "license": "CC BY 2.0", + "license_url": "https://creativecommons.org/licenses/by/2.0", + "source_description": "A brown female Boxer named Isabella Princesa.", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 588, + 480 + ] + }, + { + "pet": "sirius", + "file": "static/images/pets/sirius.avif", + "source_url": "https://www.adoptapet.com/", + "source": "Previously captured Adopt-a-Pet listing photograph, 2026-09-09", + "original_file": "pet-02.avif", + "changes": "Reassigned to the visually matching synthetic listing; original image bytes retained.", + "dimensions": [ + 716, + 681 + ] + }, + { + "pet": "teddy", + "file": "static/images/pets/teddy.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/1/11/White_Toy_poodle.jpg/960px-White_Toy_poodle.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:White_Toy_poodle.jpg", + "author": "Harpagornis", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "source_description": "Toy poodle", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 640 + ] + }, + { + "pet": "waymo", + "file": "static/images/pets/waymo.avif", + "source_url": "https://www.adoptapet.com/", + "source": "Previously captured Adopt-a-Pet listing photograph, 2026-09-09", + "original_file": "pet-03.avif", + "changes": "Reassigned to the visually matching synthetic listing; original image bytes retained.", + "dimensions": [ + 716, + 680 + ] + }, + { + "pet": "winston", + "file": "static/images/pets/winston.avif", + "source_url": "https://media.adoptapet.com/image/upload/d_Fallback-Photo_Dog-v3.png/c_auto,g_auto,w_358,ar_142:135,dpr_2/f_auto,q_auto/[REDACTED_ID_1b1ca69325d4]", + "source": "User-provided Adopt-a-Pet reference capture, 2026-09-10", + "capture_file": "assets/c448e5e323b262574e88.bin", + "changes": "Original image bytes retained; source identifier was redacted by the capture tool.", + "dimensions": [ + 716, + 681 + ] + }, + { + "pet": "yuki", + "file": "static/images/pets/yuki.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/7/74/Chihuahua_Long-haired_cream.jpg/960px-Chihuahua_Long-haired_cream.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:Chihuahua_Long-haired_cream.jpg", + "author": "Canarian", + "license": "CC BY-SA 4.0", + "license_url": "https://creativecommons.org/licenses/by-sa/4.0", + "source_description": "Long-haired Chihuahua, cream, male.", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 812 + ] + }, + { + "pet": "zorro", + "file": "static/images/pets/zorro.jpg", + "source_url": "https://thumb.wikimedia.org/wikipedia/commons/thumb/9/91/Black_Shorthair_Chihuahua.jpg/960px-Black_Shorthair_Chihuahua.jpg?utm_source=commons.wikimedia.org&utm_campaign=imageinfo&utm_content=thumbnail", + "description_url": "https://commons.wikimedia.org/wiki/File:Black_Shorthair_Chihuahua.jpg", + "author": "Dudechihuahua", + "license": "CC0", + "license_url": "http://creativecommons.org/publicdomain/zero/1.0/deed.en", + "source_description": "A short-haired chihuahua on a front porch", + "changes": "Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.", + "dimensions": [ + 960, + 1280 + ] + } + ] +} diff --git a/sites/adopt_a_pet/requirements.txt b/sites/adopt_a_pet/requirements.txt new file mode 100644 index 000000000..c60ad81ba --- /dev/null +++ b/sites/adopt_a_pet/requirements.txt @@ -0,0 +1,5 @@ +Flask==3.1.0 +Flask-SQLAlchemy==3.1.1 +Flask-WTF==1.2.2 +Werkzeug==3.1.3 +SQLAlchemy==2.0.36 diff --git a/sites/adopt_a_pet/static/css/site.css b/sites/adopt_a_pet/static/css/site.css new file mode 100644 index 000000000..9cac14286 --- /dev/null +++ b/sites/adopt_a_pet/static/css/site.css @@ -0,0 +1,27 @@ +@font-face{font-family:Plantin;src:local(Georgia)} +:root{--ink:#1c0903;--aqua:#6bd2ce;--pale:#b1ece9;--paper:#f3f3ef;--blue:#42a7ef;--yellow:#f4d318} +*{box-sizing:border-box}html{overflow-x:hidden}body{margin:0;color:var(--ink);font-family:Arial,sans-serif;background:#fff}a{color:inherit;text-decoration:none}button,input,select,textarea{font:inherit}.promo{display:none} +.site-header{height:70px;background:var(--aqua);display:flex;align-items:center;gap:36px;padding:0 max(30px,calc((100% - 1210px)/2));position:relative;z-index:20}.logo{font-family:Plantin,Georgia,serif;font-size:38px;font-weight:700;letter-spacing:-2px;white-space:nowrap}.logo span,.logo b{color:inherit}.site-header nav{display:flex;align-items:center;gap:28px;flex:1;margin-left:auto}.site-header nav a{font-size:15px}.site-header .login-link{border:1.5px solid var(--ink);border-radius:30px;padding:10px 30px}.button,button{border:0;border-radius:30px;background:var(--ink);color:#fff;padding:14px 29px;font-weight:700;cursor:pointer}.site-header .button{padding:12px 29px}.small{padding:10px 19px}.hamburger{display:none;font-size:25px}.flash{max-width:900px;margin:15px auto;background:#fff6c9;border-left:5px solid var(--yellow);padding:14px;position:relative;z-index:21} +.hero{background:var(--pale);padding:40px 20px 38px;min-height:600px}.hero-inner{max-width:1210px;margin:auto;display:grid;grid-template-columns:430px 1fr;gap:65px;align-items:center}.hero-copy h1{font-family:Plantin,Georgia,serif;font-size:68px;line-height:1.02;letter-spacing:-2.5px;margin:0 0 14px}.hero-copy h1 i{font-style:normal;color:inherit}.hero-copy h2{font-size:20px;line-height:1.25;font-weight:400;margin:0 0 28px;max-width:420px}.hero form{display:flex;flex-direction:column;gap:14px;box-shadow:none;background:transparent;padding:0}.hero label{position:relative}.hero label>span{position:absolute;z-index:1;left:25px;top:8px;color:#5f5f5f;font-size:13px;font-weight:700}.hero input,.hero select{width:100%;height:59px;border:1.5px solid var(--ink);border-radius:34px;background:#fff;padding:24px 52px 7px 24px;font-size:20px;appearance:none}.hero-select:after{content:'▾';position:absolute;right:24px;top:24px}.hero-actions{display:grid;grid-template-columns:1fr 1fr;gap:15px}.hero-actions .criteria{background:transparent;color:var(--ink);border:1.5px solid var(--ink)}.hero-actions button{height:53px}.smart-search{font-size:14px;margin-top:12px}.smart-search u{font-weight:700} +.pet-showcase{height:480px;position:relative;display:flex;align-items:center;justify-content:center}.showcase-card{position:absolute;width:343px;height:438px;border-radius:17px;padding:24px 20px 20px;overflow:hidden}.showcase-card img{width:100%;height:294px;object-fit:cover;border-radius:58px 58px 72px 72px}.showcase-card h3{font-family:Plantin,Georgia,serif;font-size:46px;line-height:.9;margin:18px 0 4px;text-transform:uppercase}.showcase-card p{font-size:18px;margin:2px 0;font-weight:700}.showcase-card small{font-size:16px}.showcase-card.left{background:var(--blue);left:25px;transform:scale(.77);z-index:1}.showcase-card.center{background:var(--aqua);z-index:3}.showcase-card.right{background:var(--yellow);right:10px;transform:scale(.77);z-index:2}.showcase-card .heart{position:absolute;right:24px;bottom:20px;font-size:33px}.showcase-arrow{position:absolute;font-size:48px;z-index:4}.showcase-arrow.left-arrow{left:-5px}.showcase-arrow.right-arrow{right:-10px} +.partners{height:60px;display:flex;justify-content:center;align-items:center;gap:24px;font-size:12px}.partners b{font-size:20px;color:#e65b22}.partners b:nth-of-type(2){color:#2568b2}.partners b:nth-of-type(3){color:#e52539}.home-content{background:var(--paper);padding-top:48px}.wide{max-width:1256px;margin:45px auto;padding:0 20px}.home-content>.wide{margin-top:0}.section-title,.wide>h1{font-family:Plantin,Georgia,serif;font-size:49px;line-height:1;margin:0 0 30px;text-align:center}.pet-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:24px}.pet-card{border-radius:15px;overflow:hidden;background:var(--aqua);position:relative}.pet-card:nth-child(3n+1){background:var(--blue)}.pet-card:nth-child(3n){background:var(--yellow)}.pet-card img{display:block;width:100%;aspect-ratio:142/135;object-fit:cover}.pet-card div{padding:18px 20px 22px}.pet-card h3{font-family:Plantin,Georgia,serif;font-size:30px;margin:0 0 5px}.pet-card p{margin:5px 0}.pet-card small{color:var(--ink)}.intro{margin-top:0;text-align:center;padding:10px 25px 55px}.intro h2{font-family:Plantin,Georgia,serif;font-size:48px;margin:0 0 20px}.intro p{max-width:650px;margin:12px auto;line-height:1.5} +.search-band{background:var(--aqua);padding:25px max(30px,calc((100% - 1256px)/2))}.search-band h1{font-family:Plantin,Georgia,serif;font-size:52px;margin:0 0 20px}.search-summary{background:white;border-radius:16px;display:grid;grid-template-columns:80px 170px 1fr;align-items:center;height:82px;padding:15px 22px}.search-icon{font-size:30px}.search-summary span{color:#5f5f5f;font-size:13px;font-weight:700;display:block}.search-summary b{font-size:20px}.search-summary>div:last-child{border-left:1px solid var(--ink);padding-left:25px}.alert-strip{margin:30px 0 34px;background:#eee;border-radius:16px;padding:20px;display:flex;align-items:center;gap:18px}.alert-strip div{flex:1}.alert-strip h2{font-size:20px;margin:0 0 6px}.alert-strip p{margin:0}.results-layout{display:grid;grid-template-columns:315px 1fr;gap:20px}.filters{background:var(--pale);border-radius:16px;padding:24px 20px;display:flex;flex-direction:column;gap:20px}.filters label{display:flex;flex-direction:column;gap:7px;font-size:18px;font-weight:700}.filters input,.filters select,.auth input,.auth select,.auth textarea,.shelter-search input{padding:13px 18px;border:1px solid var(--ink);border-radius:28px;background:white}.filters button{align-self:stretch}.results-main .results-meta{display:flex;justify-content:space-between;margin:4px 0 28px}.results-main .pet-grid{grid-template-columns:repeat(3,minmax(0,1fr));gap:23px}.pages{display:flex;justify-content:center;gap:20px;margin:30px}.pages a{background:var(--ink);color:#fff;padding:10px 22px;border-radius:22px} +.detail{max-width:1210px;margin:42px auto;padding:0 20px;display:grid;grid-template-columns:minmax(0,2.15fr) minmax(300px,.85fr);gap:24px;align-items:start}.pet-gallery{min-width:0}.pet-gallery>img{width:100%;height:480px;object-fit:contain;background:#202020;border-radius:15px;display:block}.thumbs{display:flex;gap:9px;margin-top:16px}.thumbs img{width:76px;height:76px;object-fit:cover;border-radius:11px}.detail article{background:var(--aqua);border-radius:16px;padding:25px}.detail h1{font-family:Plantin,Georgia,serif;font-size:52px;margin:0 0 8px}.detail h2{font-size:22px;margin:8px 0}.eyebrow{font-size:12px;font-weight:700;letter-spacing:.06em}.actions{display:flex;gap:10px;align-items:center;margin:24px 0}.actions .button,.actions button{display:block;text-align:center;width:100%;padding-left:12px;padding-right:12px}.detail dl div{display:flex;justify-content:space-between;border-bottom:1px solid #4da6a2;padding:12px 0}.detail dd{font-weight:700}.detail article>a{text-decoration:underline;font-weight:700} +.auth{max-width:520px;margin:55px auto;background:#f2f2ee;padding:35px;border-radius:20px}.auth h1{font-family:Plantin,Georgia,serif;font-size:48px}.auth form{display:flex;flex-direction:column;gap:18px}.auth label,.shelter-search label{display:flex;flex-direction:column;gap:7px;font-weight:700}.auth textarea{min-height:120px}.account{display:flex;align-items:center;gap:15px}.account h1{margin-right:auto}.row{display:flex;gap:25px;background:#f3f3ef;padding:17px;border-bottom:1px solid #ddd}.row b{flex:1}.shelter-search{display:flex;gap:12px;align-items:end;max-width:700px}.shelter-search label{flex:1}.shelters{display:grid;grid-template-columns:repeat(2,1fr);gap:18px;margin-top:25px}.shelters a,.breed-cols div,.article-grid article{background:#f3f3ef;border-radius:16px;padding:22px}.breed-cols{display:grid;grid-template-columns:1fr 1fr;gap:30px}.breed-cols a{display:block;padding:10px;border-bottom:1px solid #ddd}.article-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:20px}.article-grid article{padding:0;overflow:hidden}.article-grid img{width:100%;aspect-ratio:1.5;object-fit:cover}.article-grid h2,.article-grid p{margin:15px}.empty{grid-column:1/-1;text-align:center;padding:40px;background:#f3f3ef} +footer{margin-top:70px;background:var(--aqua);color:var(--ink);padding:50px max(25px,calc((100% - 1210px)/2))}footer .logo{font-size:44px}footer div{display:flex;gap:24px;margin:15px 0}footer p{font-size:12px} +@media(max-width:900px){.site-header{height:54px;padding:0 14px;margin-bottom:38px}.site-header nav{display:flex;position:absolute;top:54px;left:0;right:0;height:38px;padding:0 14px;background:#fff;gap:20px;overflow-x:auto;align-items:center}.site-header nav a{white-space:nowrap;font-size:12px}.site-header .login-link{display:none}.site-header .button{margin-left:auto;padding:8px 17px}.hamburger{display:block}.hero{padding:28px 18px 35px}.hero-inner{grid-template-columns:1fr;gap:15px}.hero-copy h1{font-size:50px}.hero-copy h2{font-size:16px}.pet-showcase{height:360px}.showcase-card{width:260px;height:330px;padding:15px}.showcase-card img{height:210px}.showcase-card h3{font-size:32px;margin-top:11px}.showcase-card p,.showcase-card small{font-size:14px}.showcase-card.left{left:0}.showcase-card.right{right:0}.results-layout{grid-template-columns:1fr}.filters{display:grid;grid-template-columns:repeat(3,1fr)}.results-main .pet-grid{grid-template-columns:repeat(2,1fr)}.detail{grid-template-columns:1fr}.pet-grid{grid-template-columns:repeat(2,1fr)}} +@media(max-width:520px){.logo{font-size:23px;letter-spacing:-1px}.site-header{gap:12px}.hero{min-height:0}.hero-inner{display:block}.hero-copy h1{font-size:43px;line-height:1.02;margin-bottom:12px}.hero-copy h2{margin-bottom:20px}.hero input,.hero select{height:52px;font-size:17px}.hero-actions button{padding:8px 10px}.smart-search{font-size:12px}.pet-showcase{height:235px;margin-top:22px}.showcase-card{width:165px;height:220px;padding:9px;border-radius:10px}.showcase-card img{height:133px;border-radius:35px}.showcase-card h3{font-size:22px;margin:7px 0 2px}.showcase-card p{font-size:10px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}.showcase-card small{font-size:10px}.showcase-card .heart{right:10px;bottom:8px;font-size:18px}.showcase-card.left{left:-17px;transform:scale(.78)}.showcase-card.right{right:-17px;transform:scale(.78)}.showcase-arrow{display:none}.partners{height:50px;gap:12px}.partners b{font-size:14px}.wide{padding:0 14px;margin:30px auto}.section-title,.wide>h1{font-size:38px}.pet-grid,.results-main .pet-grid,.shelters,.article-grid,.breed-cols{grid-template-columns:1fr}.home-content .pet-grid{display:flex;overflow:auto}.home-content .pet-card{min-width:240px}.intro{padding:20px 4px 35px}.intro h2{font-size:38px}.search-band{padding:18px 14px}.search-band h1{font-size:38px}.search-summary{grid-template-columns:48px 100px 1fr;padding:10px;height:72px}.search-summary b{font-size:14px}.alert-strip{align-items:flex-start;flex-direction:column}.filters{grid-template-columns:1fr}.results-meta{font-size:13px}.detail{padding:0 14px;margin:22px auto}.pet-gallery>img{height:315px}.detail article{padding:20px}.detail h1{font-size:42px}.actions{align-items:stretch;flex-direction:column}.actions form,.actions button,.actions .button{width:100%;text-align:center}.account{flex-wrap:wrap}.account h1{width:100%}.row{flex-wrap:wrap}.row b{width:100%;flex:none}footer{margin-top:40px}footer div{flex-wrap:wrap}} +@media(max-width:900px){.site-header .login-link{display:block;border:0;padding:4px;font-size:12px;margin-left:auto}.site-header .button{margin-left:0}} +/* Form controls carry an intrinsic min-content width, so a 1fr grid track cannot shrink below + it and /search and /shelters overflowed the viewport at 768/390/320. Let the tracks and the + controls actually shrink. */ +.filters input,.filters select,.shelter-search input{min-width:0;width:100%;max-width:100%} +.filters label,.shelter-search label{min-width:0} +.shelter-search{flex-wrap:wrap}.shelter-search button{flex:0 0 auto} +@media(max-width:900px){.filters{grid-template-columns:repeat(3,minmax(0,1fr))}} +@media(max-width:520px){.filters{grid-template-columns:minmax(0,1fr)}.shelter-search label{flex:1 0 100%}} +/* Explicit focus ring: the stylesheet previously defined none, leaving only the UA default. */ +a:focus-visible,button:focus-visible,input:focus-visible,select:focus-visible,textarea:focus-visible{outline:3px solid var(--ink);outline-offset:2px} +/* Header items with no destination in this mirror are labels, not dead links. */ +.nav-inert{font-size:15px;color:#444;cursor:default} +.form-error{background:#fff6c9;border-left:5px solid var(--yellow);padding:12px;margin:0 0 6px;font-weight:700} diff --git a/sites/adopt_a_pet/static/icons/.gitkeep b/sites/adopt_a_pet/static/icons/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sites/adopt_a_pet/static/js/.gitkeep b/sites/adopt_a_pet/static/js/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sites/adopt_a_pet/static/photo-credits.html b/sites/adopt_a_pet/static/photo-credits.html new file mode 100644 index 000000000..2c74304ee --- /dev/null +++ b/sites/adopt_a_pet/static/photo-credits.html @@ -0,0 +1,14 @@ + +Photo credits | Adopt a PetAdopt a Pet

Photo credits

This is an offline demonstration with fictional pet and shelter records. The photographs are real, representative images; the names and adoption details are sample data.

Nine listing photographs and the editorial banners were captured from Adopt a Pet. The following eleven listing photographs are reused under their individual licenses.

    +
  • archie photograph by GizaDog — CC BY-SA 3.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • batman photograph by David Shankbone — Attribution. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • cinders photograph by Whoosher — CC BY-SA 4.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • daisy photograph by Dietmar Rabich — CC BY-SA 4.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • luna photograph by Mario Kühlich — Copyrighted free use. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • milo photograph by Signbear999 — CC BY-SA 4.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • olive photograph by Jwads82 — CC BY-SA 4.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • ruby photograph by Flickr user boxercab — CC BY 2.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • teddy photograph by Harpagornis — CC BY-SA 4.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • yuki photograph by Canarian — CC BY-SA 4.0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
  • zorro photograph by Dudechihuahua — CC0. Wikimedia-provided thumbnail, retained without local image edits; displayed with CSS object-fit.
    • +
diff --git a/sites/adopt_a_pet/tasks.jsonl b/sites/adopt_a_pet/tasks.jsonl new file mode 100644 index 000000000..fcd041436 --- /dev/null +++ b/sites/adopt_a_pet/tasks.jsonl @@ -0,0 +1,20 @@ +{"web_name":"Adopt a Pet","id":"AdoptAPet--0","ques":"Search for dogs near Phoenix, AZ. Open Waymo's profile and report his breed, age group, size, and adoption fee.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_0.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a pet search for dogs near Phoenix, AZ (a /search URL carrying the location and Pet type = Dog) and MUST open Waymo's profile page (/pet/waymo). (2) The answer MUST report all four facts exactly as shown on the profile: both breed names, the age group, the size, and the adoption fee in dollars. Breed and age group also appear on the result card, but size and fee are only on the profile. (3) An empty answer, a missing fact, or a size/fee that was not read from the profile is a FAIL. (4) Read-only task: no favorites, applications, alerts or users may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--1","ques":"Search for cats near Scottsdale, AZ. Among the results, find the kitten whose listing is in Scottsdale itself and report the pet's name, exact age in months, color, and whether the pet is good with children.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_1.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a cat search near Scottsdale, AZ and MUST open the profile of the kitten whose result card shows Scottsdale, AZ as its location; the results also contain a second kitten from another Arizona city, and reporting that one is a FAIL. (2) The answer MUST give the pet's name, the exact age in months from the profile (the age group alone is not enough), the color from the profile's details table, and an explicit yes/no for 'good with children' that matches the profile. (3) Empty or partial answers FAIL. (4) Read-only task: no favorites, applications, alerts or users may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--2","ques":"Search with the location \"Phoenix, AZ\" for male dogs (the site returns matches across Arizona). Check every results page, open the matching profiles, and identify the pet with the lowest adoption fee. Report his name, breeds, and fee.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_2.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a search with location Phoenix, AZ (or another Arizona-wide query), Pet type = Dog and Sex = Male; the results span two pages, so results page 2 MUST also be opened. (2) Adoption fees are only on profile pages, so EVERY matching male dog's profile (seven in total) MUST be opened. (3) The answer MUST name the single pet with the lowest fee, list both of its breeds, and give its fee in dollars; if several candidates are listed, the one singled out as lowest must be the correct one. (4) An answer based on a partial search (for example only the Phoenix-city results) is a FAIL. (5) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--3","ques":"Search for adult cats near Arizona. Among pets marked house-trained and good with cats, identify the one with the lowest adoption fee. Report name, city, breed, and fee.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_3.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a search for Arizona (any Arizona-wide location query) with Pet type = Cat and Age = Adult, and MUST open BOTH adult cat profiles, because house-trained, good-with-cats and the fee are only on the profile. (2) The answer MUST name the qualifying pet with the lowest fee and give its city, breed and fee in dollars as shown on the profile. (3) If both cats are discussed, the one singled out as lowest must be the correct one. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--4","ques":"Use the pet search to find Arno, a dog listed in Casa Grande, AZ (the location box matches city, state and postal code, not pet names). Report the rescue name, rescue phone number, and rescue email by opening both the pet profile and the linked shelter page.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_4.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST reach Arno through the pet search (a /search visit), open Arno's profile, and follow the 'Cared for by' link to the rescue's page. (2) The answer MUST give the rescue's name, phone number and e-mail address exactly as shown on the rescue page; phone and e-mail are not on the pet profile. (3) A wrong rescue, a missing contact detail, or an empty answer is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--5","ques":"Search for pets near Seattle, WA. Compare Daisy and Pepper and report each pet's species, age in months, and adoption fee; then identify the lower-fee pet.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_5.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a search near Seattle, WA and MUST open BOTH Daisy's and Pepper's profiles (age in months and fee are only there). (2) The answer MUST give, for each pet, the species, the age in months and the fee in dollars, and MUST state which of the two has the lower fee. (3) Attributing the lower fee to the wrong pet, or omitting the comparison, is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--6","ques":"Log in as alice.j@test.com with password TestPass123!, find Sirius, add him to favorites, and confirm both Sirius and the existing Luna appear under Favorite pets.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_6.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST log in as alice.j@test.com through the login form, open Sirius's profile and use the Favorite button there, then open the account page AFTER favoriting. (2) The database after-state MUST contain exactly one new favorite (Alice -> Sirius) while Alice's existing Luna favorite remains; no other table may change. (3) The answer MUST confirm that both Sirius and Luna are listed. (4) A self-reported success without the database change, extra favorites, or a removed Luna is a FAIL."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--7","ques":"Log in as alice.j@test.com with password TestPass123!, remove Luna from favorites, and confirm Luna is no longer listed under Favorite pets.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_7.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST log in as alice.j@test.com, open Luna's profile and use 'Remove favorite' there (the account page has no remove control), then open the account page. (2) The database after-state MUST show exactly the Alice -> Luna favorite row removed, nothing added, and no other table changed. (3) The answer MUST confirm Luna is no longer listed. (4) An unchanged database or any additional write is a FAIL."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--8","ques":"Log in as bob.smith@test.com with password TestPass123!, find Daisy, and submit an adoption inquiry using phone 206-555-0199, housing 'Rent with permission', and experience 'I have cared for two family dogs for eight years.' Confirm the inquiry is Submitted in your account.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_8.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST log in as bob.smith@test.com, open Daisy's profile, open the inquiry form via 'Ask about Daisy' (/apply/daisy), submit it, and then open the account page. (2) The database after-state MUST contain exactly one new application row for Bob and Daisy with phone 206-555-0199, housing 'Rent with permission', the exact experience sentence, and status Submitted; no other table may change. (3) The answer MUST confirm that the inquiry for Daisy is Submitted. (4) Wrong field values, a duplicate row, or no row is a FAIL."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--9","ques":"Log in as carol.w@test.com with password TestPass123! and create a New Pet Alert for Siamese cats within 50 miles of postal code 33130. Confirm the alert appears in the account.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_9.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST log in as carol.w@test.com, open the New Pet Alert form (/alerts), submit it, and open the account page. (2) The database after-state MUST contain exactly one new alert for Carol with Pet type Cat, breed Siamese, postal code 33130 and a 50-mile radius; no other table may change. (3) The answer MUST confirm the alert with its breed, postal code and radius. (4) Wrong values, a duplicate alert, or no row is a FAIL."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--10","ques":"Use Find a shelter to search for Seattle. Open the matching shelter and report its name, phone, email, and the name of every pet shown there.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_10.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST use Find a shelter with a Seattle query and open the matching shelter's own page. (2) The answer MUST give the shelter's name, phone number and e-mail exactly as shown on that page, plus the name of every pet listed there (there are two). (3) A missing pet name, a missing contact detail, or an empty answer is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--11","ques":"Open Breed 101 and select Maine Coon. From the results, report the matching pet's name, sex, location, and adoption fee.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_11.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST open Breed 101 (/breeds), follow the Maine Coon link to its results, and open the matching pet's profile, because the fee is only on the profile. (2) The answer MUST give the pet's name, sex, location and adoption fee in dollars as shown. (3) A fee that was not read from the profile, or an empty answer, is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--12","ques":"Search for small dogs near Arizona. Compare all matching profiles and identify the youngest. Report the pet's name, age in months, breed or breeds, and city.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_12.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a dog search for Arizona with Size = Small; result cards show only the age group, so the profile of every Adult candidate (at least) MUST be opened to read ages in months. (2) The answer MUST name the youngest pet and give its age in months, both of its breeds and its city as shown on its profile. (3) Naming a different pet as youngest is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--13","ques":"Search for adult female dogs near Austin, TX. Report the matching pet's name, fee, and whether she is good with cats and children.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_13.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a dog search near Austin, TX filtered to Female (the Adult filter is expected as well) and MUST open the matching pet's profile. (2) The answer MUST give the pet's name, the fee in dollars, and explicit yes/no statements for 'good with cats' and 'good with children' that match the profile's details table. (3) A missing or inverted yes/no statement is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--14","ques":"Search for pets near New York, NY. Open both results and determine which has the lower adoption fee. Report both names and fees, then the lower-fee pet.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_14.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a search near New York, NY and MUST open BOTH result profiles (fees are only there). (2) The answer MUST give both pet names with their fees in dollars and MUST state which pet has the lower fee. (3) Wrong attribution of the lower fee, or no comparison, is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--15","ques":"Log in as david.b@test.com with password TestPass123!, favorite Archie and Ruby, then confirm both appear in Favorite pets.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_15.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST log in as david.b@test.com, open Archie's and Ruby's profiles and favorite each from its profile, then open the account page after both. (2) The database after-state MUST contain exactly two new favorites (David -> Archie and David -> Ruby) and nothing else may change. (3) The answer MUST confirm that both pets are listed. (4) Only one favorite, extra favorites, or no database change is a FAIL."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--16","ques":"Create an account named Jamie Lee with email jamie.lee@example.test and password PetFriend123!, favorite Olive, and confirm Olive appears in the new account.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_16.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST create the account through the Sign Up form with name Jamie Lee, e-mail jamie.lee@example.test and password PetFriend123!, then open Olive's profile, favorite her, and open the account page. (2) The database after-state MUST contain exactly one new user with that e-mail and name whose stored password verifies, and exactly one new favorite (that user -> Olive); no other table may change. (3) The answer MUST confirm that Olive appears in the new account. (4) A missing user row, a wrong e-mail or name, or no favorite is a FAIL."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--17","ques":"Search for senior dogs near Miami, FL. Report the matching pet's name, exact age in months, breeds, adoption fee, and whether the pet is house-trained.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_17.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a dog search near Miami, FL with Age = Senior and MUST open the matching pet's profile. (2) The answer MUST give the name, the exact age in months, both breeds, the fee in dollars and an explicit yes/no for house-trained, all as shown on the profile. (3) The age group instead of months, a missing breed, or an inverted house-trained statement is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--18","ques":"Visit Pet advice and report the titles of the articles about adoption paperwork, adoption fees, and bringing home a newly adopted dog.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_18.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST open the Pet advice page (/blog). (2) The answer MUST quote the three article titles about adoption paperwork, adoption fees and bringing home a newly adopted dog with the on-page wording (minor punctuation or quoting differences are acceptable, paraphrases are not). (3) A missing or invented title is a FAIL. (4) Read-only task: no account state may change."} +{"web_name":"Adopt a Pet","id":"AdoptAPet--19","ques":"Search for dogs near Arizona. Open the matching profiles on every results page and, among the pets marked good with children, identify the one with the lowest adoption fee. Report the pet's name, city, breed or breeds, age in months, and fee.","web":"http://localhost:40028/","upstream_url":"https://www.adoptapet.com/","verifier_path":"sites/adopt_a_pet/verify/verify_19.py","judge_rubric":"FACT CHECKPOINTS: (1) The trajectory MUST include a dog search for Arizona (any Arizona-wide location query); the eight results span two pages, so results page 2 MUST also be opened. (2) 'Good with children' and the fee are only on profile pages, so EVERY one of the eight dog profiles MUST be opened. (3) The answer MUST name the good-with-children pet with the lowest fee and give its city, both breeds, age in months and fee in dollars; if several pets are listed, the one singled out as lowest must be the correct one. (4) Read-only task: no account state may change."} diff --git a/sites/adopt_a_pet/templates/.gitkeep b/sites/adopt_a_pet/templates/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/sites/adopt_a_pet/templates/400.html b/sites/adopt_a_pet/templates/400.html new file mode 100644 index 000000000..665d71c13 --- /dev/null +++ b/sites/adopt_a_pet/templates/400.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}We couldn’t accept that | Adopt a Pet{% endblock %}{% block content %}

We couldn’t accept that request

Some of the details sent with the form were missing, too long, or not one of the offered choices. Please go back and try again.

Back to Adopt a Pet
{% endblock %} diff --git a/sites/adopt_a_pet/templates/404.html b/sites/adopt_a_pet/templates/404.html new file mode 100644 index 000000000..ae1fcc326 --- /dev/null +++ b/sites/adopt_a_pet/templates/404.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}Page not found | Adopt a Pet{% endblock %}{% block content %}

We couldn’t find that page

The page you asked for is not part of this site. It may have moved, or the address may be mistyped.

Back to Adopt a Pet
{% endblock %} diff --git a/sites/adopt_a_pet/templates/500.html b/sites/adopt_a_pet/templates/500.html new file mode 100644 index 000000000..3de8840ab --- /dev/null +++ b/sites/adopt_a_pet/templates/500.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}Something went wrong | Adopt a Pet{% endblock %}{% block content %}

Something went wrong

We hit an unexpected problem handling that request. Nothing was saved. Please try again.

Back to Adopt a Pet
{% endblock %} diff --git a/sites/adopt_a_pet/templates/account.html b/sites/adopt_a_pet/templates/account.html new file mode 100644 index 000000000..096a1fcee --- /dev/null +++ b/sites/adopt_a_pet/templates/account.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import card %}{% block content %}

Favorite pets

{% for p in favorites %}{{card(p)}}{% else %}

No favorite pets yet.

{% endfor %}

Adoption inquiries

{% for a,p in applications %}
{{p.name}}{{a.status}}{{a.phone}}
{% else %}

No inquiries yet.

{% endfor %}

New Pet Alerts

{% for a in alerts %}
{{a.species}} · {{a.breed or 'Any breed'}}{{a.postal}} within {{a.radius}} miles
{% else %}

No alerts yet.

{% endfor %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/alerts.html b/sites/adopt_a_pet/templates/alerts.html new file mode 100644 index 000000000..957831d39 --- /dev/null +++ b/sites/adopt_a_pet/templates/alerts.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Create a New Pet Alert

{% if error %}{% endif %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/application_done.html b/sites/adopt_a_pet/templates/application_done.html new file mode 100644 index 000000000..f0209ae6b --- /dev/null +++ b/sites/adopt_a_pet/templates/application_done.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Your inquiry was sent!

{{pet.shelter.name}} received your inquiry about {{pet.name}}.

View my inquiries
{% endblock %} diff --git a/sites/adopt_a_pet/templates/apply.html b/sites/adopt_a_pet/templates/apply.html new file mode 100644 index 000000000..ee6cf31ed --- /dev/null +++ b/sites/adopt_a_pet/templates/apply.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Ask about {{pet.name}}

Your message will go to {{pet.shelter.name}}.

{% if error %}{% endif %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/base.html b/sites/adopt_a_pet/templates/base.html new file mode 100644 index 000000000..8c0eab4e4 --- /dev/null +++ b/sites/adopt_a_pet/templates/base.html @@ -0,0 +1 @@ +{% block title %}Adopt a Pet{% endblock %}
{% if current_user %}{{current_user.name}}{% else %}Sign Up{% endif %}
{% with m=get_flashed_messages() %}{% if m %}
{{m[-1]}}
{% endif %}{% endwith %}
{% block content %}{% endblock %}
diff --git a/sites/adopt_a_pet/templates/blog.html b/sites/adopt_a_pet/templates/blog.html new file mode 100644 index 000000000..8abf4f14b --- /dev/null +++ b/sites/adopt_a_pet/templates/blog.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Pet adoption advice

{% for img,title in [('article-23c4acda83eaf8ae.avif','What to know about pet adoption paperwork'),('article-67613dbc6f5d35e7.avif','Why is there an adoption fee?'),('article-7df82ed295140b9e.avif','Bringing home your newly adopted dog'),('article-d481e062963ebb2d.avif','Make pet adoption less scary'),('article-d2d957768d445381.avif','Am I ready for a cat?'),('article-87c08b41dcca022d.avif','Things to consider before adopting')] %}

{{title}}

Expert guidance for a happy start with your new companion.

{% endfor %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/breeds.html b/sites/adopt_a_pet/templates/breeds.html new file mode 100644 index 000000000..8b0d9bed8 --- /dev/null +++ b/sites/adopt_a_pet/templates/breeds.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Dog and cat breed information

Dog breeds

{% for b in dogs %}{{b}}{% endfor %}

Cat breeds

{% for b in cats %}{{b}}{% endfor %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/index.html b/sites/adopt_a_pet/templates/index.html new file mode 100644 index 000000000..0b7a9197b --- /dev/null +++ b/sites/adopt_a_pet/templates/index.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import card %}{% block content %}

Ready to
adopt a pet?

Let’s get started. Search adoptable pets from shelters, rescues, and individuals.

Feeling adventurous? Try our new AI SmartSearch BETA

‹{% for p in pets[:3] %}{{p.name}}, {{p.breed}}

{{p.name}}

{{p.breed}}

{{p.sex}}, {{p.age_group}}
{{p.city}}, {{p.state}}♡{% endfor %}›
Our partnerszoetisPedigreePETSMART

Hi, we’re Adopt a Pet

20+ years helping pets in need · 15k+ partners · 223k+ adoptions

We’ll help you find the right pet faster with search filters, New Pet Alerts, and adoption advice.

Pets you may love

{% for p in pets %}{{card(p)}}{% endfor %}

Need to rehome a pet?

Rehoming is a big decision. Our adoption partners can help you find a new home for a pet you can no longer keep, and answer questions about the process.

Browse the shelters and rescues near you, or read our pet advice articles first.

{% endblock %} diff --git a/sites/adopt_a_pet/templates/login.html b/sites/adopt_a_pet/templates/login.html new file mode 100644 index 000000000..2501ccaf7 --- /dev/null +++ b/sites/adopt_a_pet/templates/login.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Log in

New to Adopt a Pet? Sign Up

{% endblock %} diff --git a/sites/adopt_a_pet/templates/macros.html b/sites/adopt_a_pet/templates/macros.html new file mode 100644 index 000000000..9214c3b55 --- /dev/null +++ b/sites/adopt_a_pet/templates/macros.html @@ -0,0 +1 @@ +{% macro card(p) %}{% endmacro %} diff --git a/sites/adopt_a_pet/templates/pet.html b/sites/adopt_a_pet/templates/pet.html new file mode 100644 index 000000000..a3f1a145f --- /dev/null +++ b/sites/adopt_a_pet/templates/pet.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}{{pet.name}} | Adopt a Pet{% endblock %}{% block content %}
{{pet.name}}, {{pet.breed}}
{{pet.name}} photo thumbnail

AVAILABLE FOR ADOPTION

{{pet.name}}

{{pet.breed}}{{' / '~pet.secondary_breed if pet.secondary_breed}}

{{pet.sex}} · {{pet.age_group}} · {{pet.age_months}} months old · {{pet.size}}

{{pet.city}}, {{pet.state}} {{pet.postal}}

Ask about {{pet.name}}

About {{pet.name}}

{{pet.description}}

Adoption fee
${{pet.fee}}
Color
{{pet.color}}
House-trained
{{'Yes' if pet.house_trained else 'No'}}
Good with dogs
{{'Yes' if pet.good_dogs else 'No'}}
Good with cats
{{'Yes' if pet.good_cats else 'No'}}
Good with children
{{'Yes' if pet.good_children else 'No'}}

Cared for by

{{pet.shelter.name}}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/register.html b/sites/adopt_a_pet/templates/register.html new file mode 100644 index 000000000..c6e7cfa73 --- /dev/null +++ b/sites/adopt_a_pet/templates/register.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Create your account

{% if error %}{% endif %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/search.html b/sites/adopt_a_pet/templates/search.html new file mode 100644 index 000000000..865ac38d2 --- /dev/null +++ b/sites/adopt_a_pet/templates/search.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import card %}{% block content %}

Pets for adoption near {{q or 'you'}}

⌕
Type of Pet{{filters.get('species') or 'Any pet'}}
Location{{q or 'Your area'}}

Want to find the right pet faster?

Create a New Pet Alert, and be the first to know when new pets pop up who match your search.

♧   Create an Alert
Showing {{((page-1)*per_page+1) if total else 0}} - {{[page*per_page,total]|min}} of {{total}} available petsSort by: Nearest⌄
{% for p in pets %}{{card(p)}}{% else %}

No exact matches

Try removing one or more filters.

{% endfor %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/shelter.html b/sites/adopt_a_pet/templates/shelter.html new file mode 100644 index 000000000..eadd08e1e --- /dev/null +++ b/sites/adopt_a_pet/templates/shelter.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% from 'macros.html' import card %}{% block content %}

{{shelter.name}}

{{shelter.city}}, {{shelter.state}} · {{shelter.phone}} · {{shelter.email}}

Pets available from this rescue

{% for p in shelter.pets %}{{card(p)}}{% endfor %}
{% endblock %} diff --git a/sites/adopt_a_pet/templates/shelters.html b/sites/adopt_a_pet/templates/shelters.html new file mode 100644 index 000000000..79a760d87 --- /dev/null +++ b/sites/adopt_a_pet/templates/shelters.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block content %}

Find an animal shelter or rescue

{% for s in shelters %}

{{s.name}}

{{s.city}}, {{s.state}}

{{s.pets|length}} adoptable pets

{% else %}

No shelters matched your search.

{% endfor %}
{% endblock %} diff --git a/sites/adopt_a_pet/verify/README.md b/sites/adopt_a_pet/verify/README.md new file mode 100644 index 000000000..2deac2abe --- /dev/null +++ b/sites/adopt_a_pet/verify/README.md @@ -0,0 +1,93 @@ +# Adopt-a-Pet deterministic grading contract + +Each row in `sites/adopt_a_pet/tasks.jsonl` points at `verify_0.py` … `verify_19.py`. The +wrappers share `verify_lib.py` (package, navigation, answer and SQLite-state checks) and +`ground_truth.py` (the frozen 20-pet / 6-shelter / 4-user catalog transcribed from `app.py`). +No verdict depends on an LLM; the `llm_*` helpers exist only for API parity with +`sites/merriam_webster/verify` and short-circuit under `--no_llm True`. + +## Inputs + +```bash +uv run python sites/adopt_a_pet/verify/verify_0.py --run_dir /abs/run --no_llm True \ + [--initial_db /abs/initial.db --after_db /abs/after.db] [--container wh-review] +``` + +Snapshots resolve in this order: explicit flags, `/initial.db` + `/after.db`, +then `docker cp` from `$WH_CONTAINER` (default `wh-review`) at +`/opt/WebSyn/adopt_a_pet/{instance_seed,instance}/adopt_a_pet.db`. Missing or invalid snapshots +fail closed (`database_unavailable` / `snapshot_contract_invalid`, `infra_error: true`). Output is +JSON `{task_id, pass, reason, evidence[]}`; exit 0 = PASS, 1 = FAIL. `agent_demo/eval_judge.py +--run_dir RUN --verifier True` is the normal entry point. + +## Package validation (every task) + +- exact `task_id`; non-empty `final_answer`; `terminated: true` with `termination_reason: agent_done`; +- at least one step; every recorded URL (`start_url`, step `url`s, `final_url`) on the same + loopback host **and port** as `start_url` (alt ports such as 41024 are fine, off-site URLs are not); +- both screenshots referenced by every step exist and decode as PNG. + +## Snapshot contract (every task) + +Both snapshots must have exactly the six tables `user, shelter, pet, favorite, application, +pet_alert` with the column order produced by `app.py`'s models and identical schemas. The initial +snapshot must contain 4 users, 6 shelters, 20 pets, 1 favorite (alice.j -> luna), 0 applications and +0 alerts, and its `pet` / `shelter` / `user` rows must equal `ground_truth.py` (any catalog edit makes +every verifier fail closed instead of grading against stale truth). `pet` and `shelter` must be +row-identical before and after. Read-only tasks additionally require `user, favorite, application, +pet_alert` to be row-identical; stateful tasks require the exact row delta listed below and reject +collateral writes, duplicates, wrong users and stale-state no-ops. + +## Search semantics the gates rely on + +`app.search()` scores the `location` box by token overlap with `" "`, so any +query containing the state token (`Phoenix, AZ`, `Arizona`, `AZ`, `Tucson AZ`) returns **every** +Arizona pet, while a bare city (`Phoenix`) returns only that city. `verify_lib.AZ_WIDE` therefore +accepts any query whose tokens contain `az` or `arizona`. Result pages hold 6 cards; hidden facts +(fee, months, color, house-trained, good-with-*) are only on `/pet/`; shelter phone / e-mail +are only on `/shelter/`. + +## Per-task contract + +| Task | Kind | Navigation gates | Answer / state checks | +|---|---|---|---| +| 0 | read | `/search` location ~ phoenix, species=Dog; `/pet/waymo` | both breeds, age group, size, `$fee` | +| 1 | read | `/search` location ~ scottsdale, species=Cat; `/pet/neo` | name, months, color, good-with-children = yes | +| 2 | read | `/search` AZ-wide, species=Dog, sex=Male **and** `page=2`; all 7 male AZ dog profiles | winner name, both breeds, `$fee`, winner attached to "lowest" | +| 3 | read | `/search` AZ-wide, species=Cat, age=Adult; `/pet/casper` + `/pet/cinders` | winner name, city, breed, `$fee`, attached to "lowest" | +| 4 | read | any `/search`; `/pet/arno`; `/shelter/1` | rescue name, phone (any separators), e-mail | +| 5 | read | `/search` location ~ seattle; `/pet/daisy` + `/pet/pepper` | both names, both species, both months, both fees, lower-fee pet attached | +| 6 | state | `/login` (alice) -> `/pet/sirius` -> `/account` | favorite += {alice->sirius} exactly; luna kept; other tables unchanged; answer names both | +| 7 | state | `/login` (alice) -> `/pet/luna` -> `/account` | favorite -= {alice->luna} exactly; answer names Luna | +| 8 | state | `/login` (bob) -> `/pet/daisy` -> `/apply/daisy` -> `/account` | application += 1 row (bob, daisy, phone digits, housing, exact experience, Submitted) | +| 9 | state | `/login` (carol) -> `/alerts` -> `/account` | pet_alert += 1 row (carol, Cat, siamese, 33130, 50) | +| 10 | read | `/shelters?q~seattle`; `/shelter/4` | shelter name, phone, e-mail, Daisy + Pepper | +| 11 | read | `/breeds`; `/search?breed~maine coon&species=Cat`; `/pet/milo` | name, sex, city, `$fee` | +| 12 | read | `/search` AZ-wide, species=Dog, size=Small; the 3 Adult profiles | winner name, months, both breeds, city, attached to "youngest" | +| 13 | read | `/search` location ~ austin, species=Dog, sex=Female; `/pet/ruby` | name, `$fee`, good-with-cats = no, good-with-children = no | +| 14 | read | `/search` location ~ new york / ny; `/pet/luna` + `/pet/milo` | both names, both fees, lower-fee pet attached | +| 15 | state | `/login` (david) -> `/pet/archie` -> `/account` and -> `/pet/ruby` -> `/account` | favorite += {david->archie, david->ruby} exactly | +| 16 | state | `/register` (e-mail typed) -> `/pet/olive` -> `/account` | user += 1 (e-mail, name, scrypt hash verifies `PetFriend123!`); favorite += {jamie->olive} | +| 17 | read | `/search` location ~ miami, species=Dog, age=Senior; `/pet/teddy` | name, months, both breeds, `$fee`, house-trained = yes | +| 18 | read | `/blog` | the three exact titles (punctuation-insensitive) | +| 19 | read | `/search` AZ-wide, species=Dog **and** `page=2`; all 8 AZ dog profiles | winner name, city, both breeds, months, `$fee`, attached to "lowest" | + +Answer matchers are affirmative and negation-aware (`not $165`, `isn't good with cats` do not +count as matches). Money accepts `$165`, `$165.00`, `165 dollars`; months accept `36 months`, +`36-month-old`, `36 mo`; yes/no statements accept `X: Yes`, `X? No`, `not X`, `she is X`, +`X and Y: No`, `neither X nor Y`. Comparison answers must attach the winner to the comparison +word (`lowest` / `cheaper` / `youngest`, or the inverse `more expensive` on a loser); a bare +report that names only the winner also passes, a list of candidates without a pick does not. + +## Tests + +```bash +cd agent_demo && uv run python -m unittest discover -s ../sites/adopt_a_pet/verify/tests -p 'test_*.py' +``` + +`tests/_support.py` builds synthetic snapshots from `ground_truth.py` and writes trajectories in +the `agent_demo/agent.py` shape; every verifier runs as a subprocess. Coverage per task: genuine +PASS, run-dir snapshot discovery, no-op, wrong task id, off-origin URL, corrupt PNG, missing +snapshots (fail closed), catalog drift (fail closed), shortcut (correct answer without the gate +pages), wrong answers / misattributed comparisons, read-only writes, and for stateful tasks the +state-mismatch, collateral-write, wrong-user, duplicate and reordered-workflow cases. diff --git a/sites/adopt_a_pet/verify/ground_truth.py b/sites/adopt_a_pet/verify/ground_truth.py new file mode 100644 index 000000000..0a9a8ce8c --- /dev/null +++ b/sites/adopt_a_pet/verify/ground_truth.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Frozen Adopt-a-Pet catalog used as verifier ground truth. + +The values below are a transcription of ``PETS`` / ``SHELTERS`` / ``USERS`` in +``sites/adopt_a_pet/app.py`` at the reviewed commit. Every verifier fails closed +(``snapshot_contract_invalid``) when the supplied initial SQLite snapshot no +longer matches this table, so a silent catalog edit can never make a verifier +grade against stale truth. The agent-facing ``tasks.jsonl`` never carries any +of these values. +""" +from __future__ import annotations + +PET_FIELDS = ( + "id", "slug", "name", "species", "breed", "secondary_breed", "sex", "age_group", + "age_months", "size", "color", "city", "state", "postal", "fee", + "house_trained", "good_dogs", "good_cats", "good_children", "shelter_id", +) + +_PET_ROWS = [ + # id slug name species breed secondary sex age mo size color city st postal fee ht gd gc gk shelter + (1, "sirius", "Sirius", "Dog", "Chihuahua", "Terrier", "Male", "Senior", 108, "Small", "Tan", "Scottsdale", "AZ", "85251", 175, 1, 1, 1, 0, 1), + (2, "waymo", "Waymo", "Dog", "American Pit Bull Terrier", "Mixed Breed", "Male", "Adult", 24, "Large", "Gray", "Phoenix", "AZ", "85004", 225, 1, 1, 0, 1, 2), + (3, "casper", "Casper", "Cat", "Colorpoint Shorthair", None, "Male", "Adult", 48, "Medium", "Cream", "Mesa", "AZ", "85201", 125, 1, 0, 1, 1, 1), + (4, "neo", "Neo", "Cat", "Domestic Shorthair", None, "Male", "Kitten", 7, "Small", "Black", "Scottsdale", "AZ", "85250", 110, 1, 1, 1, 1, 2), + (5, "amba", "Amba", "Cat", "Domestic Mediumhair", None, "Female", "Kitten", 5, "Small", "Tabby", "Arizona City", "AZ", "85123", 95, 1, 1, 1, 1, 1), + (6, "cinders", "Cinders", "Cat", "Domestic Shorthair", None, "Female", "Adult", 85, "Medium", "Tortoiseshell", "Sedona", "AZ", "86336", 120, 1, 0, 1, 0, 2), + (7, "arno", "Arno", "Dog", "German Shepherd Dog", "Mixed Breed", "Male", "Adult", 43, "Large", "Black and Tan", "Casa Grande", "AZ", "85122", 200, 1, 1, 0, 1, 1), + (8, "batman", "Batman", "Dog", "Chihuahua", "Yorkshire Terrier", "Male", "Adult", 36, "Small", "Black", "Tucson", "AZ", "85701", 165, 1, 1, 1, 0, 2), + (9, "horus", "Horus", "Dog", "Pointer", "Labrador Retriever","Male", "Young", 16, "Large", "White and Black", "Phoenix", "AZ", "85006", 210, 1, 1, 0, 1, 1), + (10, "luna", "Luna", "Dog", "Beagle", None, "Female", "Young", 14, "Medium", "Tricolor", "New York", "NY", "10011", 250, 1, 1, 1, 1, 3), + (11, "milo", "Milo", "Cat", "Maine Coon", None, "Male", "Adult", 38, "Large", "Orange", "New York", "NY", "10003", 150, 1, 0, 1, 1, 3), + (12, "daisy", "Daisy", "Dog", "Golden Retriever", None, "Female", "Adult", 30, "Large", "Golden", "Seattle", "WA", "98109", 275, 1, 1, 1, 1, 4), + (13, "pepper", "Pepper", "Cat", "Domestic Shorthair", None, "Female", "Young", 13, "Small", "Black and White", "Seattle", "WA", "98101", 130, 1, 1, 1, 0, 4), + (14, "archie", "Archie", "Dog", "Australian Shepherd", None, "Male", "Young", 18, "Medium", "Merle", "Austin", "TX", "78704", 240, 1, 1, 0, 1, 5), + (15, "ruby", "Ruby", "Dog", "Boxer", "Mixed Breed", "Female", "Adult", 42, "Large", "Fawn", "Austin", "TX", "78702", 215, 1, 1, 0, 0, 5), + (16, "olive", "Olive", "Cat", "Siamese", None, "Female", "Adult", 27, "Medium", "Seal Point", "Miami", "FL", "33130", 145, 1, 0, 1, 1, 6), + (17, "teddy", "Teddy", "Dog", "Poodle", "Mixed Breed", "Male", "Senior", 96, "Small", "White", "Miami", "FL", "33133", 185, 1, 1, 1, 1, 6), + (18, "winston", "Winston", "Dog", "Chihuahua", "Mixed Breed", "Male", "Adult", 60, "Small", "Tan", "Tempe", "AZ", "85281", 230, 1, 1, 1, 0, 2), + (19, "yuki", "Yuki", "Dog", "Chihuahua", "Mixed Breed", "Female", "Senior", 120, "Small", "Cream", "Glendale", "AZ", "85301", 205, 1, 0, 1, 1, 1), + (20, "zorro", "Zorro", "Dog", "Chihuahua", "Terrier", "Male", "Adult", 72, "Small", "Black", "Prescott", "AZ", "86301", 220, 1, 1, 0, 0, 2), +] +PETS: list[dict] = [dict(zip(PET_FIELDS, row)) for row in _PET_ROWS] + +SHELTER_FIELDS = ("id", "name", "city", "state", "phone", "email") +_SHELTER_ROWS = [ + (1, "Desert Paws Rescue", "Phoenix", "AZ", "602-555-0141", "hello@desertpaws.test"), + (2, "Happy Tails Alliance", "Scottsdale", "AZ", "480-555-0128", "adopt@happytails.test"), + (3, "City Friends Shelter", "New York", "NY", "212-555-0164", "pets@cityfriends.test"), + (4, "Pacific Animal Haven", "Seattle", "WA", "206-555-0119", "info@pacifichaven.test"), + (5, "Lone Star Companions", "Austin", "TX", "512-555-0182", "team@lonestar.test"), + (6, "Sunshine Pet Rescue", "Miami", "FL", "305-555-0136", "adopt@sunshine.test"), +] +SHELTERS: list[dict] = [dict(zip(SHELTER_FIELDS, row)) for row in _SHELTER_ROWS] + +USER_FIELDS = ("id", "email", "name") +_USER_ROWS = [ + (1, "alice.j@test.com", "Alice Johnson"), + (2, "bob.smith@test.com", "Bob Smith"), + (3, "carol.w@test.com", "Carol Williams"), + (4, "david.b@test.com", "David Brown"), +] +USERS: list[dict] = [dict(zip(USER_FIELDS, row)) for row in _USER_ROWS] +BENCHMARK_PASSWORD = "TestPass123!" + +# (favorite.id, user.id, pet.id) -- alice.j has luna favorited in the seed. +SEED_FAVORITES: list[tuple[int, int, int]] = [(1, 1, 10)] + +BLOG_TITLES = [ + "What to know about pet adoption paperwork", + "Why is there an adoption fee?", + "Bringing home your newly adopted dog", + "Make pet adoption less scary", + "Am I ready for a cat?", + "Things to consider before adopting", +] + + +def pet(slug: str) -> dict: + for row in PETS: + if row["slug"] == slug: + return row + raise KeyError(slug) + + +def pet_by_id(pet_id: int) -> dict: + for row in PETS: + if row["id"] == int(pet_id): + return row + raise KeyError(pet_id) + + +def shelter(shelter_id: int) -> dict: + for row in SHELTERS: + if row["id"] == int(shelter_id): + return row + raise KeyError(shelter_id) + + +def user(email: str) -> dict: + for row in USERS: + if row["email"] == email.lower(): + return row + raise KeyError(email) + + +def breeds(row: dict) -> list[str]: + return [row["breed"]] + ([row["secondary_breed"]] if row["secondary_breed"] else []) + + +def search(location: str = "", species: str = "", breed: str = "", sex: str = "", + age: str = "", size: str = "") -> list[dict]: + """Re-implementation of app.search() ranking over the frozen catalog (used by + tests and by the README to document candidate sets; never by a verdict).""" + import re + + lookup = {"arizona": "AZ", "new york": "New York NY", "washington": "WA", "texas": "TX", "florida": "FL"} + query = lookup.get(location.lower(), location) + + def score(q: str, text: str) -> int: + terms = set(re.findall(r"[a-z0-9]+", q.lower())) + tokens = set(re.findall(r"[a-z0-9]+", text.lower())) + return len(terms & tokens) + + ranked = [] + for row in PETS: + s = score(query, f"{row['city']} {row['state']} {row['postal']}") if query else 1 + if s and (not species or row["species"] == species) \ + and (not breed or breed.lower() in " ".join(breeds(row)).lower()) \ + and (not sex or row["sex"] == sex) and (not age or row["age_group"] == age) \ + and (not size or row["size"] == size): + ranked.append((s, row["name"], row)) + return [item[2] for item in sorted(ranked, key=lambda item: (-item[0], item[1]))] diff --git a/sites/adopt_a_pet/verify/tests/_support.py b/sites/adopt_a_pet/verify/tests/_support.py new file mode 100644 index 000000000..224260c65 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/_support.py @@ -0,0 +1,286 @@ +"""Shared fixtures for the adopt_a_pet verifier tests. + +Synthetic SQLite snapshots with the same six tables / column order as +``instance_seed/adopt_a_pet.db`` (rows come from ``verify/ground_truth.py``) and a +hand-written trajectory writer in the ``agent_demo/agent.py`` format. No docker, no +LLM, no network; every verifier is invoked as a subprocess exactly like eval_judge does. +""" +from __future__ import annotations + +import base64 +import copy +import hashlib +import json +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any + +VERIFY_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(VERIFY_DIR)) +import ground_truth as GT # noqa: E402 + +BASE = "http://localhost:41024" +PASSWORD = GT.BENCHMARK_PASSWORD + +SCHEMA = """ +CREATE TABLE user (id INTEGER NOT NULL, email VARCHAR(120) NOT NULL, name VARCHAR(80) NOT NULL, + password_hash VARCHAR(255) NOT NULL, PRIMARY KEY (id), UNIQUE (email)); +CREATE TABLE shelter (id INTEGER NOT NULL, name VARCHAR(120) NOT NULL, city VARCHAR(60) NOT NULL, + state VARCHAR(2) NOT NULL, phone VARCHAR(20) NOT NULL, email VARCHAR(120) NOT NULL, PRIMARY KEY (id), UNIQUE (name)); +CREATE TABLE pet (id INTEGER NOT NULL, slug VARCHAR(120) NOT NULL, name VARCHAR(60) NOT NULL, species VARCHAR(20) NOT NULL, + breed VARCHAR(100) NOT NULL, secondary_breed VARCHAR(100), sex VARCHAR(10) NOT NULL, age_group VARCHAR(20) NOT NULL, + age_months INTEGER NOT NULL, size VARCHAR(20) NOT NULL, color VARCHAR(40) NOT NULL, city VARCHAR(60) NOT NULL, + state VARCHAR(2) NOT NULL, postal VARCHAR(10) NOT NULL, fee INTEGER NOT NULL, image VARCHAR(100) NOT NULL, + description TEXT NOT NULL, house_trained BOOLEAN NOT NULL, good_dogs BOOLEAN NOT NULL, good_cats BOOLEAN NOT NULL, + good_children BOOLEAN NOT NULL, shelter_id INTEGER NOT NULL, PRIMARY KEY (id), UNIQUE (slug), + FOREIGN KEY(shelter_id) REFERENCES shelter (id)); +CREATE TABLE favorite (id INTEGER NOT NULL, user_id INTEGER NOT NULL, pet_id INTEGER NOT NULL, PRIMARY KEY (id), + UNIQUE (user_id, pet_id), FOREIGN KEY(user_id) REFERENCES user (id), FOREIGN KEY(pet_id) REFERENCES pet (id)); +CREATE TABLE application (id INTEGER NOT NULL, user_id INTEGER NOT NULL, pet_id INTEGER NOT NULL, housing VARCHAR(30) NOT NULL, + experience TEXT NOT NULL, phone VARCHAR(20) NOT NULL, status VARCHAR(20) NOT NULL, PRIMARY KEY (id), + UNIQUE (user_id, pet_id), FOREIGN KEY(user_id) REFERENCES user (id), FOREIGN KEY(pet_id) REFERENCES pet (id)); +CREATE TABLE pet_alert (id INTEGER NOT NULL, user_id INTEGER NOT NULL, species VARCHAR(20) NOT NULL, breed VARCHAR(80) NOT NULL, + postal VARCHAR(10) NOT NULL, radius INTEGER NOT NULL, PRIMARY KEY (id), FOREIGN KEY(user_id) REFERENCES user (id)); +""" + + +def werkzeug_scrypt(password: str, salt: str = "fixturesalt00000") -> str: + digest = hashlib.scrypt(password.encode("utf-8"), salt=salt.encode("utf-8"), n=32768, r=8, p=1, + maxmem=132 * 1024 * 1024, dklen=64).hex() + return f"scrypt:32768:8:1${salt}${digest}" + + +_HASH_CACHE: dict[str, str] = {} + + +def hash_for(password: str) -> str: + if password not in _HASH_CACHE: + _HASH_CACHE[password] = werkzeug_scrypt(password) + return _HASH_CACHE[password] + + +class State: + """Mutable copy of the seeded mutable tables; the catalog is always the frozen one.""" + + def __init__(self) -> None: + self.users = [dict(u, password_hash=hash_for(PASSWORD)) for u in GT.USERS] + self.favorites = [tuple(f) for f in GT.SEED_FAVORITES] # (id, user_id, pet_id) + self.applications: list[dict[str, Any]] = [] + self.alerts: list[dict[str, Any]] = [] + self.pet_overrides: dict[str, dict[str, Any]] = {} + self.extra_sql: list[str] = [] + + # -- helpers ------------------------------------------------------------ + def user_id(self, email: str) -> int: + for u in self.users: + if u["email"] == email.lower(): + return int(u["id"]) + raise KeyError(email) + + def add_user(self, email: str, name: str, password: str) -> int: + new_id = max(u["id"] for u in self.users) + 1 + self.users.append({"id": new_id, "email": email.lower(), "name": name, "password_hash": hash_for(password)}) + return new_id + + def add_favorite(self, email: str, slug: str) -> None: + new_id = max([f[0] for f in self.favorites] + [0]) + 1 + self.favorites.append((new_id, self.user_id(email), GT.pet(slug)["id"])) + + def remove_favorite(self, email: str, slug: str) -> None: + target = (self.user_id(email), GT.pet(slug)["id"]) + before = len(self.favorites) + self.favorites = [f for f in self.favorites if (f[1], f[2]) != target] + assert len(self.favorites) == before - 1, f"no favorite {email}/{slug}" + + def add_application(self, email: str, slug: str, housing: str, experience: str, phone: str, status: str = "Submitted") -> None: + new_id = max([a["id"] for a in self.applications] + [0]) + 1 + self.applications.append({"id": new_id, "user_id": self.user_id(email), "pet_id": GT.pet(slug)["id"], + "housing": housing, "experience": experience, "phone": phone, "status": status}) + + def add_alert(self, email: str, species: str, breed: str, postal: str, radius: int) -> None: + new_id = max([a["id"] for a in self.alerts] + [0]) + 1 + self.alerts.append({"id": new_id, "user_id": self.user_id(email), "species": species, "breed": breed, + "postal": postal, "radius": radius}) + + # -- persistence -------------------------------------------------------- + def write(self, path: Path) -> Path: + con = sqlite3.connect(path) + try: + con.executescript(SCHEMA) + con.executemany("INSERT INTO user(id,email,name,password_hash) VALUES (:id,:email,:name,:password_hash)", self.users) + con.executemany("INSERT INTO shelter(id,name,city,state,phone,email) VALUES (:id,:name,:city,:state,:phone,:email)", GT.SHELTERS) + for pet in GT.PETS: + row = dict(pet) + row.update(self.pet_overrides.get(pet["slug"], {})) + row["image"] = "x.avif" + row["description"] = f"{row['name']} is an affectionate {row['age_group'].lower()} {row['species'].lower()}." + con.execute( + "INSERT INTO pet(id,slug,name,species,breed,secondary_breed,sex,age_group,age_months,size,color,city,state,postal,fee," + "image,description,house_trained,good_dogs,good_cats,good_children,shelter_id) VALUES " + "(:id,:slug,:name,:species,:breed,:secondary_breed,:sex,:age_group,:age_months,:size,:color,:city,:state,:postal,:fee," + ":image,:description,:house_trained,:good_dogs,:good_cats,:good_children,:shelter_id)", row) + con.executemany("INSERT INTO favorite(id,user_id,pet_id) VALUES (?,?,?)", self.favorites) + con.executemany("INSERT INTO application(id,user_id,pet_id,housing,experience,phone,status) VALUES " + "(:id,:user_id,:pet_id,:housing,:experience,:phone,:status)", self.applications) + con.executemany("INSERT INTO pet_alert(id,user_id,species,breed,postal,radius) VALUES " + "(:id,:user_id,:species,:breed,:postal,:radius)", self.alerts) + for statement in self.extra_sql: + con.execute(statement) + con.commit() + finally: + con.close() + return path + + +def step(path: str, action: str = "click", text: str | None = None) -> dict[str, Any]: + """One trajectory step in the agent.py shape; ``path`` is relative to BASE.""" + params: dict[str, Any] = {"text": text} if text is not None else {} + url = path if path.startswith("http") else f"{BASE}{path}" + return {"url": url, "action": action, "params": params} + + +def login_steps(email: str, password: str = PASSWORD) -> list[dict[str, Any]]: + return [step("/login", "input", email), step("/login", "input", password), step("/login", "click"), step("/account")] + + +def register_steps(name: str, email: str, password: str) -> list[dict[str, Any]]: + return [step("/register", "input", name), step("/register", "input", email), step("/register", "input", password), + step("/register", "click"), step("/account")] + + +def only_paths(steps: list[dict[str, Any]], *allowed: str) -> list[dict[str, Any]]: + from urllib.parse import urlparse + + def path_of(item: dict[str, Any]) -> str: + return urlparse(item["url"]).path.rstrip("/") or "/" + + return [item for item in steps if path_of(item) in allowed] + + +def without_path(steps: list[dict[str, Any]], *dropped: str) -> list[dict[str, Any]]: + from urllib.parse import urlparse + + def path_of(item: dict[str, Any]) -> str: + return urlparse(item["url"]).path.rstrip("/") or "/" + + return [item for item in steps if path_of(item) not in dropped] + + +PNG_1PX = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=") + + +def write_run(run_dir: Path, task_id: str, steps: list[dict[str, Any]], answer: str) -> None: + run_dir.mkdir(parents=True, exist_ok=True) + shots = run_dir / "screenshots" + shots.mkdir(exist_ok=True) + numbered = [] + for index, item in enumerate(steps): + before, after = f"step_{index:03d}.png", f"step_{index + 1:03d}.png" + (shots / before).write_bytes(PNG_1PX) + (shots / after).write_bytes(PNG_1PX) + numbered.append({"step": index, **item, "screenshot_before": before, "screenshot_after": after}) + trajectory = { + "task": "synthetic", "task_id": task_id, "start_url": f"{BASE}/", "model": "unit-test", "max_steps": 30, + "steps": numbered, "terminated": bool(answer), "termination_reason": "agent_done" if answer else "max_steps", + "final_url": numbered[-1]["url"] if numbered else f"{BASE}/", "final_answer": answer if answer else None, + } + (run_dir / "trajectory.json").write_text(json.dumps(trajectory, indent=2), encoding="utf-8") + + +class VerifierTestCase(unittest.TestCase): + """Base class: ``self.N`` selects verify_N.py.""" + + N = -1 + + @property + def task_id(self) -> str: + return f"AdoptAPet--{self.N}" + + def verdict(self, steps: list[dict[str, Any]], answer: str, initial: State | None = None, after: State | None = None, + task_id: str | None = None, snapshots_in_run_dir: bool = False, + trajectory_updates: dict[str, Any] | None = None, corrupt_screenshot: bool = False, + no_snapshots: bool = False) -> dict[str, Any]: + initial = initial or State() + after = after if after is not None else copy.deepcopy(initial) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + run_dir = root / "run" + write_run(run_dir, task_id or self.task_id, steps, answer) + if trajectory_updates: + tp = run_dir / "trajectory.json" + t = json.loads(tp.read_text()) + t.update(trajectory_updates) + tp.write_text(json.dumps(t, indent=2)) + if corrupt_screenshot: + next((run_dir / "screenshots").glob("*.png")).write_bytes(b"not a png") + cmd = [sys.executable, str(VERIFY_DIR / f"verify_{self.N}.py"), "--run_dir", str(run_dir), "--no_llm", "True"] + if no_snapshots: + cmd += ["--container", "no-such-container-for-tests"] + elif snapshots_in_run_dir: + initial.write(run_dir / "initial.db") + after.write(run_dir / "after.db") + else: + cmd += ["--initial_db", str(initial.write(root / "initial.db")), "--after_db", str(after.write(root / "after.db"))] + result = subprocess.run(cmd, capture_output=True, text=True) + self.assertTrue(result.stdout.strip(), f"verifier printed nothing; stderr={result.stderr}") + verdict = json.loads(result.stdout) + verdict["returncode"] = result.returncode + return verdict + + def assertPasses(self, verdict: dict[str, Any]) -> None: + self.assertTrue(verdict["pass"], verdict["evidence"]) + self.assertEqual(verdict["returncode"], 0) + self.assertEqual(verdict["reason"], "all checks passed") + + def assertFailsOn(self, verdict: dict[str, Any], reason: str) -> None: + self.assertFalse(verdict["pass"], verdict["evidence"]) + self.assertEqual(verdict["returncode"], 1) + self.assertEqual(verdict["reason"], reason, verdict["evidence"]) + + +class CommonCases: + """Mixed into every per-task test: package-level failure modes shared by all verifiers.""" + + GENUINE_STEPS: list[dict[str, Any]] = [] + ANSWER = "" + + def genuine_after(self) -> State: + return State() + + def test_genuine_run_passes(self) -> None: + self.assertPasses(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=self.genuine_after())) + + def test_run_dir_snapshots_are_discovered(self) -> None: + self.assertPasses(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=self.genuine_after(), snapshots_in_run_dir=True)) + + def test_noop_run_fails(self) -> None: + self.assertFailsOn(self.verdict([step("/")], ""), "final_answer_nonempty") + + def test_other_task_trajectory_fails(self) -> None: + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=self.genuine_after(), task_id="AdoptAPet--99"), + "trajectory_task_matches") + + def test_off_origin_url_fails(self) -> None: + steps = list(self.GENUINE_STEPS) + [step("https://www.adoptapet.com/pet-adoption")] + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=self.genuine_after()), "all_urls_match_local_origin") + + def test_corrupt_screenshot_fails(self) -> None: + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=self.genuine_after(), corrupt_screenshot=True), + "screenshots_decode") + + def test_missing_snapshots_fail_closed(self) -> None: + verdict = self.verdict(self.GENUINE_STEPS, self.ANSWER, no_snapshots=True) + self.assertFalse(verdict["pass"]) + self.assertEqual(verdict["reason"], "database_unavailable") + self.assertTrue(verdict.get("infra_error")) + + def test_catalog_drift_fails_closed(self) -> None: + initial = State() + initial.pet_overrides["batman"] = {"fee": 999} + verdict = self.verdict(self.GENUINE_STEPS, self.ANSWER, initial=initial, after=copy.deepcopy(initial)) + self.assertFalse(verdict["pass"]) + self.assertEqual(verdict["reason"], "snapshot_contract_invalid") diff --git a/sites/adopt_a_pet/verify/tests/test_verify_0.py b/sites/adopt_a_pet/verify/tests/test_verify_0.py new file mode 100644 index 000000000..e92090917 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_0.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, step # noqa: E402 + + +class VerifyTask0Tests(CommonCases, VerifierTestCase): + N = 0 + GENUINE_STEPS = [step("/"), step("/search?location=Phoenix%2C+AZ&species=Dog"), step("/pet/waymo", "done")] + ANSWER = "Waymo: American Pit Bull Terrier / Mixed Breed, Adult, Large, adoption fee $225." + + def test_shortcut_without_search_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/pet/waymo", "done")], self.ANSWER), "visited_dog_search_near_phoenix") + + def test_shortcut_without_profile_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/search?location=Phoenix%2C+AZ&species=Dog", "done")], self.ANSWER), "visited_pet_waymo") + + def test_wrong_fee_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER.replace("$225", "$250")), "answer_has_fee") + + def test_missing_secondary_breed_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Waymo: American Pit Bull Terrier, Adult, Large, $225"), "answer_has_both_breeds") + + def test_read_only_write_fails(self): + after = State() + after.add_favorite("alice.j@test.com", "waymo") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "read_only_favorite_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_1.py b/sites/adopt_a_pet/verify/tests/test_verify_1.py new file mode 100644 index 000000000..5b05e3e4f --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_1.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, step # noqa: E402 + + +class VerifyTask1Tests(CommonCases, VerifierTestCase): + N = 1 + GENUINE_STEPS = [step("/"), step("/search?location=Scottsdale%2C+AZ&species=Cat"), step("/pet/neo", "done")] + ANSWER = "Neo, 7 months old, Black, good with children: Yes." + + def test_shortcut_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/pet/neo", "done")], self.ANSWER), "visited_cat_search_near_scottsdale") + + def test_other_kitten_fails(self): + steps = [step("/"), step("/search?location=Scottsdale%2C+AZ&species=Cat"), step("/pet/amba", "done")] + self.assertFailsOn(self.verdict(steps, "Amba, 5 months, Tabby, good with children: Yes"), "visited_pet_neo") + + def test_wrong_children_flag_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Neo, 7 months, Black, not good with children"), "answer_states_good_with_children_yes") + + def test_age_group_instead_of_months_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Neo, Kitten, Black, good with children: yes"), "answer_has_age_months") + + def test_read_only_write_fails(self): + after = State() + after.add_alert("alice.j@test.com", "Cat", "", "85250", 10) + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "read_only_pet_alert_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_10.py b/sites/adopt_a_pet/verify/tests/test_verify_10.py new file mode 100644 index 000000000..4bc3b02a3 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_10.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step # noqa: E402 + + +class VerifyTask10Tests(CommonCases, VerifierTestCase): + N = 10 + GENUINE_STEPS = [step("/"), step("/shelters"), step("/shelters?q=Seattle"), step("/shelter/4", "done")] + ANSWER = "Pacific Animal Haven — 206-555-0119 — info@pacifichaven.test — pets: Daisy and Pepper." + + def test_no_shelter_search_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/shelters"), step("/shelter/4", "done")], self.ANSWER), "used_shelter_search_for_seattle") + + def test_no_shelter_page_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/shelters?q=Seattle", "done")], self.ANSWER), "visited_shelter_4") + + def test_missing_pet_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Pacific Animal Haven, 206-555-0119, info@pacifichaven.test, pets: Daisy."), "answer_lists_every_pet") + + def test_wrong_email_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Pacific Animal Haven, 206-555-0119, info@pacific.test, Daisy, Pepper"), "answer_has_email") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_11.py b/sites/adopt_a_pet/verify/tests/test_verify_11.py new file mode 100644 index 000000000..6154f34d2 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_11.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step # noqa: E402 + + +class VerifyTask11Tests(CommonCases, VerifierTestCase): + N = 11 + GENUINE_STEPS = [step("/"), step("/breeds"), step("/search?breed=Maine+Coon&species=Cat"), step("/pet/milo", "done")] + ANSWER = "Milo, Male, New York, NY, adoption fee $150." + + def test_no_breed_page_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/search?breed=Maine+Coon&species=Cat"), step("/pet/milo", "done")], self.ANSWER), "visited_breed_101") + + def test_no_profile_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/breeds"), step("/search?breed=Maine+Coon&species=Cat", "done")], self.ANSWER), "visited_pet_milo") + + def test_wrong_fee_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Milo, Male, New York, $250"), "answer_has_fee") + + def test_wrong_sex_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Milo, Female, New York, $150"), "answer_has_sex") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_12.py b/sites/adopt_a_pet/verify/tests/test_verify_12.py new file mode 100644 index 000000000..c7d4d1349 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_12.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step, without_path # noqa: E402 + +FILTERED = "/search?distance=50+miles+or+less&location=Arizona&species=Dog&breed=&sex=&age=&size=Small" + + +class VerifyTask12Tests(CommonCases, VerifierTestCase): + N = 12 + GENUINE_STEPS = [step("/"), step("/search?location=Arizona&species=Dog"), step(FILTERED), + step("/pet/batman"), step("/pet/sirius"), step("/pet/winston"), step("/pet/yuki"), step("/pet/zorro", "done")] + ANSWER = "Youngest: Batman, 36 months, Chihuahua / Yorkshire Terrier, Tucson." + + def test_adults_only_passes(self): + self.assertPasses(self.verdict(without_path(self.GENUINE_STEPS, "/pet/sirius", "/pet/yuki"), self.ANSWER)) + + def test_missing_adult_profile_fails(self): + self.assertFailsOn(self.verdict(without_path(self.GENUINE_STEPS, "/pet/winston"), self.ANSWER), "visited_pet_winston") + + def test_no_size_filter_fails(self): + steps = [step("/"), step("/search?location=Arizona&species=Dog"), step("/pet/batman"), step("/pet/winston"), step("/pet/zorro", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER), "visited_small_dog_search_arizona") + + def test_wrong_pet_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Youngest: Winston, 60 months, Chihuahua / Mixed Breed, Tempe."), "answer_names_winner") + + def test_misattribution_fails(self): + answer = "Batman (Chihuahua / Yorkshire Terrier, Tucson) 36 months, Winston 60 months, Zorro 72 months. The youngest is Zorro." + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, answer), "answer_identifies_youngest") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_13.py b/sites/adopt_a_pet/verify/tests/test_verify_13.py new file mode 100644 index 000000000..cead56ea7 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_13.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step # noqa: E402 + + +class VerifyTask13Tests(CommonCases, VerifierTestCase): + N = 13 + GENUINE_STEPS = [step("/"), step("/search?location=Austin%2C+TX&species=Dog"), + step("/search?distance=50+miles+or+less&location=Austin%2C+TX&species=Dog&breed=&sex=Female&age=Adult&size="), + step("/pet/ruby", "done")] + ANSWER = "Ruby: adoption fee $215; good with cats: No; good with children: No." + + def test_no_sex_filter_fails(self): + steps = [step("/"), step("/search?location=Austin%2C+TX&species=Dog"), step("/pet/ruby", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER), "visited_female_dog_search_austin") + + def test_wrong_cats_flag_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Ruby: $215; good with cats: Yes; good with children: No."), "answer_states_good_with_cats_no") + + def test_wrong_children_flag_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Ruby: $215; good with cats: No; good with children: Yes."), "answer_states_good_with_children_no") + + def test_prose_negation_passes(self): + self.assertPasses(self.verdict(self.GENUINE_STEPS, "Ruby's fee is $215 and she is not good with cats or children.")) + + def test_missing_flags_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Ruby, $215."), "answer_states_good_with_cats_no") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_14.py b/sites/adopt_a_pet/verify/tests/test_verify_14.py new file mode 100644 index 000000000..b6a783608 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_14.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step, without_path # noqa: E402 + + +class VerifyTask14Tests(CommonCases, VerifierTestCase): + N = 14 + GENUINE_STEPS = [step("/"), step("/search?location=New+York%2C+NY&species="), step("/pet/luna"), step("/pet/milo", "done")] + ANSWER = "Luna $250, Milo $150 — Milo is the lower-fee pet." + + def test_missing_profile_fails(self): + self.assertFailsOn(self.verdict(without_path(self.GENUINE_STEPS, "/pet/milo"), self.ANSWER), "visited_pet_milo") + + def test_wrong_comparison_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Luna $250, Milo $150 — Luna has the lower fee."), "answer_identifies_lower_fee_pet") + + def test_wrong_fee_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Luna $205, Milo $150 — Milo is lower."), "answer_has_both_fees") + + def test_ny_only_query_passes(self): + self.assertPasses(self.verdict([step("/"), step("/search?location=NY&species="), step("/pet/luna"), step("/pet/milo", "done")], self.ANSWER)) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_15.py b/sites/adopt_a_pet/verify/tests/test_verify_15.py new file mode 100644 index 000000000..52084174c --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_15.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, login_steps, step # noqa: E402 + +EMAIL = "david.b@test.com" + + +class VerifyTask15Tests(CommonCases, VerifierTestCase): + N = 15 + GENUINE_STEPS = login_steps(EMAIL) + [step("/search?location=Austin%2C+TX&species=Dog"), step("/pet/archie", "click"), + step("/search?location=Austin%2C+TX&species=Dog"), step("/pet/ruby", "click"), step("/account", "done")] + ANSWER = "Archie and Ruby are both listed under Favorite pets." + + def genuine_after(self) -> State: + after = State() + after.add_favorite(EMAIL, "archie") + after.add_favorite(EMAIL, "ruby") + return after + + def test_state_mismatch_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=State()), "favorites_added_exactly") + + def test_only_one_favorited_fails(self): + after = State() + after.add_favorite(EMAIL, "archie") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_added_exactly") + + def test_extra_favorite_fails(self): + after = self.genuine_after() + after.add_favorite(EMAIL, "olive") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_added_exactly") + + def test_account_before_second_pet_fails(self): + steps = login_steps(EMAIL) + [step("/pet/archie", "click"), step("/account"), step("/pet/ruby", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=self.genuine_after()), "login_then_ruby_then_account") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_16.py b/sites/adopt_a_pet/verify/tests/test_verify_16.py new file mode 100644 index 000000000..85f006676 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_16.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, register_steps, step # noqa: E402 + +EMAIL, NAME, PASSWORD = "jamie.lee@example.test", "Jamie Lee", "PetFriend123!" + + +class VerifyTask16Tests(CommonCases, VerifierTestCase): + N = 16 + GENUINE_STEPS = register_steps(NAME, EMAIL, PASSWORD) + [step("/search?location=Miami%2C+FL&species=Cat"), step("/pet/olive", "click"), step("/account", "done")] + ANSWER = "Account created for Jamie Lee; Olive now appears under Favorite pets." + + def genuine_after(self) -> State: + after = State() + after.add_user(EMAIL, NAME, PASSWORD) + after.add_favorite(EMAIL, "olive") + return after + + def test_state_mismatch_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=State()), "exactly_one_user_added") + + def test_user_without_favorite_fails(self): + after = State() + after.add_user(EMAIL, NAME, PASSWORD) + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_added_exactly") + + def test_wrong_password_fails(self): + after = State() + after.add_user(EMAIL, NAME, "SomethingElse1!") + after.add_favorite(EMAIL, "olive") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "new_user_password_verifies") + + def test_wrong_name_fails(self): + after = State() + after.add_user(EMAIL, "Jamie", PASSWORD) + after.add_favorite(EMAIL, "olive") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "new_user_has_expected_email_and_name") + + def test_wrong_email_typed_fails(self): + steps = register_steps(NAME, "jamie@example.test", PASSWORD) + [step("/pet/olive", "click"), step("/account", "done")] + after = State() + after.add_user("jamie@example.test", NAME, PASSWORD) + after.add_favorite("jamie@example.test", "olive") + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=after), "registered_through_form") + + def test_favorite_by_existing_user_fails(self): + after = State() + after.add_user(EMAIL, NAME, PASSWORD) + after.add_favorite("alice.j@test.com", "olive") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_added_exactly") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_17.py b/sites/adopt_a_pet/verify/tests/test_verify_17.py new file mode 100644 index 000000000..536d31ed6 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_17.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step # noqa: E402 + + +class VerifyTask17Tests(CommonCases, VerifierTestCase): + N = 17 + GENUINE_STEPS = [step("/"), step("/search?location=Miami%2C+FL&species=Dog"), + step("/search?distance=50+miles+or+less&location=Miami%2C+FL&species=Dog&breed=&sex=&age=Senior&size="), + step("/pet/teddy", "done")] + ANSWER = "Teddy: 96 months old, Poodle / Mixed Breed, $185, house-trained: Yes." + + def test_no_age_filter_fails(self): + steps = [step("/"), step("/search?location=Miami%2C+FL&species=Dog"), step("/pet/teddy", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER), "visited_senior_dog_search_miami") + + def test_wrong_house_trained_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Teddy: 96 months, Poodle / Mixed Breed, $185, not house-trained."), "answer_states_house_trained_yes") + + def test_missing_months_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Teddy: Senior, Poodle / Mixed Breed, $185, house-trained: Yes."), "answer_has_age_months") + + def test_prose_passes(self): + self.assertPasses(self.verdict(self.GENUINE_STEPS, "Teddy is a 96-month-old Poodle / Mixed Breed with a $185 fee and he is housebroken.")) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_18.py b/sites/adopt_a_pet/verify/tests/test_verify_18.py new file mode 100644 index 000000000..c048ba5db --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_18.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step # noqa: E402 + + +class VerifyTask18Tests(CommonCases, VerifierTestCase): + N = 18 + GENUINE_STEPS = [step("/"), step("/blog", "done")] + ANSWER = ('Titles: "What to know about pet adoption paperwork", "Why is there an adoption fee?", ' + '"Bringing home your newly adopted dog".') + + def test_no_blog_visit_fails(self): + self.assertFailsOn(self.verdict([step("/", "done")], self.ANSWER), "visited_pet_advice") + + def test_paraphrased_title_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, 'What to know about pet adoption paperwork; Why are there adoption fees?; Bringing home your newly adopted dog'), + "answer_has_title_1") + + def test_missing_title_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, 'What to know about pet adoption paperwork; Why is there an adoption fee?'), "answer_has_title_2") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_19.py b/sites/adopt_a_pet/verify/tests/test_verify_19.py new file mode 100644 index 000000000..b9da95370 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_19.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step, without_path # noqa: E402 + +SEARCH = "/search?location=Arizona&species=Dog" + + +class VerifyTask19Tests(CommonCases, VerifierTestCase): + N = 19 + GENUINE_STEPS = [step("/"), step(SEARCH), step("/pet/arno"), step("/pet/batman"), step("/pet/horus"), step("/pet/sirius"), + step("/pet/waymo"), step("/pet/winston"), step(SEARCH + "&page=2"), step("/pet/yuki"), step("/pet/zorro", "done")] + ANSWER = "Arno — Casa Grande, AZ — German Shepherd Dog / Mixed Breed — 43 months — $200 (lowest fee among the dogs good with children)." + + def test_missing_page_2_fails(self): + self.assertFailsOn(self.verdict([s for s in self.GENUINE_STEPS if "page=2" not in s["url"]], self.ANSWER), "visited_results_page_2") + + def test_missing_profile_fails(self): + self.assertFailsOn(self.verdict(without_path(self.GENUINE_STEPS, "/pet/yuki"), self.ANSWER), "visited_pet_yuki") + + def test_wrong_pet_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Batman — Tucson — Chihuahua / Yorkshire Terrier — 36 months — $165"), "answer_names_winner") + + def test_misattribution_fails(self): + answer = "Arno (Casa Grande, German Shepherd Dog / Mixed Breed, 43 months, $200), Horus $210, Waymo $225, Yuki $205: the lowest fee is Yuki." + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, answer), "answer_identifies_lowest_fee_pet") + + def test_phoenix_az_query_passes(self): + steps = [s.copy() for s in self.GENUINE_STEPS] + for s in steps: + s["url"] = s["url"].replace("location=Arizona", "location=Phoenix%2C+AZ") + self.assertPasses(self.verdict(steps, self.ANSWER)) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_2.py b/sites/adopt_a_pet/verify/tests/test_verify_2.py new file mode 100644 index 000000000..12f997f30 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_2.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, step, without_path # noqa: E402 + +FILTERED = "/search?distance=50+miles+or+less&location=Phoenix%2C+AZ&species=Dog&breed=&sex=Male&age=&size=" + + +class VerifyTask2Tests(CommonCases, VerifierTestCase): + N = 2 + GENUINE_STEPS = [step("/"), step("/search?location=Phoenix%2C+AZ&species=Dog"), step(FILTERED), + step("/pet/horus"), step("/pet/waymo"), step("/pet/arno"), step("/pet/batman"), step("/pet/sirius"), + step("/pet/winston"), step(FILTERED + "&page=2"), step("/pet/zorro", "done")] + ANSWER = "Batman (Chihuahua / Yorkshire Terrier) has the lowest adoption fee at $165." + + def test_missing_page_2_fails(self): + steps = [s for s in self.GENUINE_STEPS if "page=2" not in s["url"]] + self.assertFailsOn(self.verdict(steps, self.ANSWER), "visited_results_page_2") + + def test_missing_one_profile_fails(self): + self.assertFailsOn(self.verdict(without_path(self.GENUINE_STEPS, "/pet/zorro"), self.ANSWER), "visited_pet_zorro") + + def test_city_only_search_fails(self): + steps = [step("/"), step("/search?location=Phoenix&species=Dog&sex=Male"), step("/pet/horus"), step("/pet/waymo", "done")] + self.assertFailsOn(self.verdict(steps, "Horus, Pointer / Labrador Retriever, $210"), "visited_male_dog_search_statewide_az") + + def test_wrong_winner_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Horus, Pointer / Labrador Retriever, $210"), "answer_names_winner") + + def test_misattributed_lowest_fails(self): + answer = "Horus $210, Waymo $225, Arno $200, Batman (Chihuahua / Yorkshire Terrier) $165. The lowest fee is Horus." + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, answer), "answer_identifies_lowest_fee_pet") + + def test_listing_with_explicit_winner_passes(self): + answer = "Horus $210, Waymo $225, Arno $200, Batman $165, Sirius $175, Winston $230, Zorro $220 -> lowest: Batman, Chihuahua / Yorkshire Terrier, $165" + self.assertPasses(self.verdict(self.GENUINE_STEPS, answer)) + + def test_read_only_write_fails(self): + after = State() + after.add_favorite("bob.smith@test.com", "batman") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "read_only_favorite_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_3.py b/sites/adopt_a_pet/verify/tests/test_verify_3.py new file mode 100644 index 000000000..a847b6f4f --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_3.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step, without_path # noqa: E402 + + +class VerifyTask3Tests(CommonCases, VerifierTestCase): + N = 3 + GENUINE_STEPS = [step("/"), step("/search?location=Arizona&species=Cat"), + step("/search?distance=50+miles+or+less&location=Arizona&species=Cat&breed=&sex=&age=Adult&size="), + step("/pet/casper"), step("/pet/cinders", "done")] + ANSWER = "Cinders — Sedona, Domestic Shorthair, $120 (lower than Casper at $125)." + + def test_missing_age_filter_fails(self): + steps = [step("/"), step("/search?location=Arizona&species=Cat"), step("/pet/casper"), step("/pet/cinders", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER), "visited_adult_cat_search_arizona") + + def test_missing_second_profile_fails(self): + self.assertFailsOn(self.verdict(without_path(self.GENUINE_STEPS, "/pet/casper"), self.ANSWER), "visited_pet_casper") + + def test_wrong_pet_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Casper — Mesa, Colorpoint Shorthair, $125"), "answer_names_winner") + + def test_misattribution_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Cinders (Sedona, Domestic Shorthair, $120) and Casper ($125); the cheaper one is Casper."), + "answer_identifies_lowest_fee_pet") + + def test_phoenix_az_query_also_counts_as_statewide(self): + steps = [step("/"), step("/search?location=Phoenix%2C+AZ&species=Cat&age=Adult"), step("/pet/casper"), step("/pet/cinders", "done")] + self.assertPasses(self.verdict(steps, self.ANSWER)) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_4.py b/sites/adopt_a_pet/verify/tests/test_verify_4.py new file mode 100644 index 000000000..e13a93f01 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_4.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step, without_path # noqa: E402 + + +class VerifyTask4Tests(CommonCases, VerifierTestCase): + N = 4 + GENUINE_STEPS = [step("/"), step("/search?location=Arizona&species=Dog&breed=German+Shepherd"), step("/pet/arno"), step("/shelter/1", "done")] + ANSWER = "Desert Paws Rescue, phone 602-555-0141, email hello@desertpaws.test." + + def test_no_search_fails(self): + self.assertFailsOn(self.verdict([step("/"), step("/pet/arno"), step("/shelter/1", "done")], self.ANSWER), "used_pet_search") + + def test_no_shelter_page_fails(self): + self.assertFailsOn(self.verdict(without_path(self.GENUINE_STEPS, "/shelter/1") + [step("/pet/arno", "done")], self.ANSWER), "visited_shelter_1") + + def test_wrong_shelter_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Happy Tails Alliance, 480-555-0128, adopt@happytails.test"), "answer_has_rescue_name") + + def test_wrong_phone_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Desert Paws Rescue, 602-555-0114, hello@desertpaws.test"), "answer_has_rescue_phone") + + def test_phone_formats_accepted(self): + self.assertPasses(self.verdict(self.GENUINE_STEPS, "Desert Paws Rescue / (602) 555-0141 / hello@desertpaws.test")) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_5.py b/sites/adopt_a_pet/verify/tests/test_verify_5.py new file mode 100644 index 000000000..834769786 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_5.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, VerifierTestCase, step, without_path # noqa: E402 + + +class VerifyTask5Tests(CommonCases, VerifierTestCase): + N = 5 + GENUINE_STEPS = [step("/"), step("/search?location=Seattle%2C+WA&species="), step("/pet/daisy"), step("/pet/pepper", "done")] + ANSWER = "Daisy: dog, 30 months, $275. Pepper: cat, 13 months, $130. Pepper has the lower fee." + + def test_missing_profile_fails(self): + self.assertFailsOn(self.verdict(without_path(self.GENUINE_STEPS, "/pet/daisy"), self.ANSWER), "visited_pet_daisy") + + def test_no_comparison_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Daisy: dog, 30 months, $275. Pepper: cat, 13 months, $130."), "answer_identifies_lower_fee_pet") + + def test_wrong_comparison_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Daisy: dog, 30 months, $275. Pepper: cat, 13 months, $130. Daisy is the lower-fee pet."), + "answer_identifies_lower_fee_pet") + + def test_wrong_age_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER.replace("13 months", "3 months")), "answer_has_both_ages") + + def test_inverse_phrasing_passes(self): + self.assertPasses(self.verdict(self.GENUINE_STEPS, "Daisy (dog, 30 months, $275) is more expensive than Pepper (cat, 13 months, $130).")) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_6.py b/sites/adopt_a_pet/verify/tests/test_verify_6.py new file mode 100644 index 000000000..e96c04ffe --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_6.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, login_steps, step # noqa: E402 + +EMAIL = "alice.j@test.com" + + +class VerifyTask6Tests(CommonCases, VerifierTestCase): + N = 6 + GENUINE_STEPS = login_steps(EMAIL) + [step("/search?location=Scottsdale%2C+AZ&species=Dog"), step("/pet/sirius", "click"), step("/account", "done")] + ANSWER = "Sirius was added to favorites; Favorite pets now lists Sirius and Luna." + + def genuine_after(self) -> State: + after = State() + after.add_favorite(EMAIL, "sirius") + return after + + def test_state_mismatch_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=State()), "favorites_added_exactly") + + def test_collateral_favorite_fails(self): + after = self.genuine_after() + after.add_favorite(EMAIL, "waymo") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_added_exactly") + + def test_wrong_user_favorited_fails(self): + after = State() + after.add_favorite("bob.smith@test.com", "sirius") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_added_exactly") + + def test_luna_removed_fails(self): + after = self.genuine_after() + after.remove_favorite(EMAIL, "luna") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_removed_exactly") + + def test_no_account_visit_after_action_fails(self): + steps = login_steps(EMAIL) + [step("/pet/sirius", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=self.genuine_after()), "login_then_pet_then_account") + + def test_wrong_login_email_fails(self): + steps = login_steps("bob.smith@test.com") + [step("/pet/sirius", "click"), step("/account", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=self.genuine_after()), "entered_expected_account_email") + + def test_answer_missing_luna_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Added Sirius to favorites.", after=self.genuine_after()), "answer_confirms_both_pets") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_7.py b/sites/adopt_a_pet/verify/tests/test_verify_7.py new file mode 100644 index 000000000..ae5178905 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_7.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, login_steps, step # noqa: E402 + +EMAIL = "alice.j@test.com" + + +class VerifyTask7Tests(CommonCases, VerifierTestCase): + N = 7 + GENUINE_STEPS = login_steps(EMAIL) + [step("/pet/luna", "click"), step("/account", "done")] + ANSWER = "Luna was removed; Favorite pets no longer lists Luna." + + def genuine_after(self) -> State: + after = State() + after.remove_favorite(EMAIL, "luna") + return after + + def test_state_mismatch_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=State()), "favorites_removed_exactly") + + def test_removed_and_added_another_fails(self): + after = self.genuine_after() + after.add_favorite(EMAIL, "sirius") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "favorites_added_exactly") + + def test_no_profile_visit_fails(self): + steps = login_steps(EMAIL) + [step("/account", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=self.genuine_after()), "login_then_pet_then_account") + + def test_collateral_alert_fails(self): + after = self.genuine_after() + after.add_alert(EMAIL, "Dog", "Beagle", "10011", 25) + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "pet_alert_unchanged") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_8.py b/sites/adopt_a_pet/verify/tests/test_verify_8.py new file mode 100644 index 000000000..269db0a55 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_8.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, login_steps, step # noqa: E402 + +EMAIL = "bob.smith@test.com" +EXPERIENCE = "I have cared for two family dogs for eight years." + + +class VerifyTask8Tests(CommonCases, VerifierTestCase): + N = 8 + GENUINE_STEPS = login_steps(EMAIL) + [step("/search?location=Seattle%2C+WA&species=Dog"), step("/pet/daisy"), + step("/apply/daisy", "input", "206-555-0199"), step("/apply/daisy", "input", EXPERIENCE), + step("/apply/daisy", "click"), step("/account", "done")] + ANSWER = "The adoption inquiry for Daisy was sent and shows as Submitted in the account." + + def genuine_after(self) -> State: + after = State() + after.add_application(EMAIL, "daisy", "Rent with permission", EXPERIENCE, "206-555-0199") + return after + + def test_state_mismatch_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=State()), "exactly_one_application_added") + + def test_wrong_housing_fails(self): + after = State() + after.add_application(EMAIL, "daisy", "Own home", EXPERIENCE, "206-555-0199") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "application_housing_exact") + + def test_wrong_phone_fails(self): + after = State() + after.add_application(EMAIL, "daisy", "Rent with permission", EXPERIENCE, "206-555-0119") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "application_phone_exact") + + def test_wrong_experience_fails(self): + after = State() + after.add_application(EMAIL, "daisy", "Rent with permission", "I have cared for dogs for years.", "206-555-0199") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "application_experience_exact") + + def test_trailing_period_tolerated(self): + after = State() + after.add_application(EMAIL, "daisy", "Rent with permission", EXPERIENCE.rstrip("."), "2065550199") + self.assertPasses(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after)) + + def test_wrong_pet_fails(self): + after = State() + after.add_application(EMAIL, "pepper", "Rent with permission", EXPERIENCE, "206-555-0199") + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "application_belongs_to_bob_for_daisy") + + def test_no_apply_page_fails(self): + steps = login_steps(EMAIL) + [step("/pet/daisy"), step("/account", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=self.genuine_after()), "login_pet_apply_account") + + def test_answer_without_status_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, "Done.", after=self.genuine_after()), "answer_confirms_submitted_for_daisy") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_9.py b/sites/adopt_a_pet/verify/tests/test_verify_9.py new file mode 100644 index 000000000..b74052ed6 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_9.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from _support import CommonCases, State, VerifierTestCase, login_steps, step # noqa: E402 + +EMAIL = "carol.w@test.com" + + +class VerifyTask9Tests(CommonCases, VerifierTestCase): + N = 9 + GENUINE_STEPS = login_steps(EMAIL) + [step("/alerts", "input", "Siamese"), step("/alerts", "input", "33130"), + step("/alerts", "click"), step("/account", "done")] + ANSWER = "Created a New Pet Alert for Siamese cats within 50 miles of 33130; it appears in the account." + + def genuine_after(self) -> State: + after = State() + after.add_alert(EMAIL, "Cat", "Siamese", "33130", 50) + return after + + def test_state_mismatch_fails(self): + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=State()), "exactly_one_alert_added") + + def test_wrong_radius_fails(self): + after = State() + after.add_alert(EMAIL, "Cat", "Siamese", "33130", 25) + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "alert_values_exact") + + def test_wrong_species_fails(self): + after = State() + after.add_alert(EMAIL, "Dog", "Siamese", "33130", 50) + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "alert_values_exact") + + def test_duplicate_alert_fails(self): + after = self.genuine_after() + after.add_alert(EMAIL, "Cat", "Siamese", "33130", 50) + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "exactly_one_alert_added") + + def test_wrong_user_fails(self): + after = State() + after.add_alert("alice.j@test.com", "Cat", "Siamese", "33130", 50) + self.assertFailsOn(self.verdict(self.GENUINE_STEPS, self.ANSWER, after=after), "alert_belongs_to_carol") + + def test_no_alerts_page_fails(self): + steps = login_steps(EMAIL) + [step("/account", "done")] + self.assertFailsOn(self.verdict(steps, self.ANSWER, after=self.genuine_after()), "login_alerts_account") + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/tests/test_verify_lib.py b/sites/adopt_a_pet/verify/tests/test_verify_lib.py new file mode 100644 index 000000000..a4467a651 --- /dev/null +++ b/sites/adopt_a_pet/verify/tests/test_verify_lib.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import ground_truth as GT # noqa: E402 +import verify_lib as V # noqa: E402 +from _support import BASE, hash_for # noqa: E402 + + +class MatcherTests(unittest.TestCase): + def test_contains_all_is_affirmative(self): + self.assertTrue(V.contains_all("Breeds: Chihuahua / Yorkshire Terrier", ["Chihuahua", "Yorkshire Terrier"])) + self.assertFalse(V.contains_all("The breed is not Chihuahua", ["Chihuahua"])) + self.assertFalse(V.contains_all("Chihuahuas everywhere", ["Chihuahua"])) # whole-word + + def test_contains_money(self): + for text in ("$165", "fee: $165.00", "165 dollars", "USD 165", "adoption fee is $ 165"): + self.assertTrue(V.contains_money(text, 165), text) + for text in ("$1650", "$16", "165 months", "not $165", "$165 is wrong"): + self.assertFalse(V.contains_money(text, 165), text) + + def test_contains_months(self): + for text in ("36 months old", "36-month-old", "36 mo", "36 mos.", "age: 36 months"): + self.assertTrue(V.contains_months(text, 36), text) + for text in ("136 months", "36", "$36", "not 36 months"): + self.assertFalse(V.contains_months(text, 36), text) + + def test_contains_phone_and_email(self): + for text in ("602-555-0141", "(602) 555-0141", "602.555.0141", "6025550141", "phone 602 555 0141"): + self.assertTrue(V.contains_phone(text, "602-555-0141"), text) + self.assertFalse(V.contains_phone("602-555-01410", "602-555-0141")) + self.assertFalse(V.contains_phone("480-555-0128", "602-555-0141")) + self.assertTrue(V.contains_email("Email: Hello@DesertPaws.test.", "hello@desertpaws.test")) + self.assertFalse(V.contains_email("hello@desertpaws.testing", "hello@desertpaws.test")) + + def test_stated_yes_no_phrasings(self): + yes = ["good with children: Yes", "Good with children — Yes", "good with children? yes", "She is good with children", + "good with cats and children: Yes", "gets along with kids", "good with children (yes)"] + no = ["good with children: No", "not good with children", "Good with children? No", "good with cats or children: No", + "she is not good with cats or children", "neither good with cats nor with children", "good with children = false"] + for text in yes: + self.assertTrue(V.stated_yes_no(text, V.CHILDREN_KW, True), text) + self.assertFalse(V.stated_yes_no(text, V.CHILDREN_KW, False), text) + for text in no: + self.assertTrue(V.stated_yes_no(text, V.CHILDREN_KW, False), text) + self.assertFalse(V.stated_yes_no(text, V.CHILDREN_KW, True), text) + self.assertFalse(V.stated_yes_no("Neo is 7 months old", V.CHILDREN_KW, True)) + self.assertTrue(V.stated_yes_no("House-trained: Yes", V.HOUSE_TRAINED_KW, True)) + self.assertTrue(V.stated_yes_no("Teddy is housebroken", V.HOUSE_TRAINED_KW, True)) + self.assertFalse(V.stated_yes_no("Teddy is not house trained", V.HOUSE_TRAINED_KW, True)) + # conflicting statements resolve to the one nearest the anchor name + text = "Amba: good with children: No. Neo: good with children: Yes." + self.assertTrue(V.stated_yes_no(text, V.CHILDREN_KW, True, anchor_name="Neo")) + self.assertFalse(V.stated_yes_no(text, V.CHILDREN_KW, True, anchor_name="Amba")) + + def test_identifies_comparison_winner(self): + others = ["Horus", "Waymo", "Arno", "Sirius", "Winston", "Zorro"] + good = ["Batman, Chihuahua / Yorkshire Terrier, $165", + "The lowest fee is Batman at $165 (Chihuahua / Yorkshire Terrier).", + "Horus $210, Waymo $225, Arno $200, Batman $165, Sirius $175, Winston $230, Zorro $220. Lowest: Batman ($165).", + "Batman has the lowest fee among Horus, Waymo, Arno, Sirius, Winston and Zorro.", + "Batman ($165) is cheaper than Sirius ($175) and all the others.", + "The lowest fee ($165) belongs to Batman, a Chihuahua / Yorkshire Terrier."] + bad = ["Horus, Pointer / Labrador Retriever, $210", + "Horus $210, Batman $165 -> the lowest fee is Horus.", + "Sirius is cheaper than Batman.", + "Horus $210, Waymo $225, Batman $165"] # lists candidates without singling one out + for text in good: + self.assertTrue(V.identifies(text, "Batman", others, V.LOWEST, V.HIGHEST), text) + for text in bad: + self.assertFalse(V.identifies(text, "Batman", others, V.LOWEST, V.HIGHEST), text) + self.assertTrue(V.identifies("Daisy is more expensive than Pepper.", "Pepper", ["Daisy"], V.LOWEST, V.HIGHEST)) + self.assertTrue(V.identifies("Pepper's fee ($130) is lower than Daisy's ($275).", "Pepper", ["Daisy"], V.LOWEST, V.HIGHEST)) + self.assertFalse(V.identifies("Daisy's fee ($275) is lower than Pepper's ($130).", "Pepper", ["Daisy"], V.LOWEST, V.HIGHEST)) + self.assertTrue(V.identifies("Youngest: Batman (36 months)", "Batman", ["Winston", "Zorro"], V.YOUNGEST, V.OLDEST)) + + def test_contains_phrase_loose(self): + self.assertTrue(V.contains_phrase_loose('"Why is there an adoption fee" and more', "Why is there an adoption fee?")) + self.assertFalse(V.contains_phrase_loose("Why are there adoption fees?", "Why is there an adoption fee?")) + + def test_search_visited_semantics(self): + traj = {"start_url": f"{BASE}/", "steps": [ + {"url": f"{BASE}/search?location=Phoenix%2C+AZ&species=Dog"}, + {"url": f"{BASE}/search?distance=50+miles+or+less&location=Phoenix%2C+AZ&species=Dog&breed=&sex=Male&age=&size=&page=2"}, + {"url": f"{BASE}/search?breed=Maine+Coon&species=Cat"}]} + self.assertTrue(V.search_visited(traj, location_any=V.AZ_WIDE, species="Dog")) + self.assertTrue(V.search_visited(traj, location_any=({"phoenix"},), species="Dog")) + self.assertTrue(V.search_visited(traj, location_any=V.AZ_WIDE, species="Dog", sex="Male", page="2")) + self.assertFalse(V.search_visited(traj, location_any=V.AZ_WIDE, species="Cat", sex="Male")) + self.assertFalse(V.search_visited(traj, location_any=({"tucson"},))) + self.assertTrue(V.search_visited(traj, breed="maine coon", species="Cat")) + self.assertFalse(V.search_visited(traj, breed="siamese")) + self.assertFalse(V.search_visited({"start_url": f"{BASE}/", "steps": [{"url": f"{BASE}/search?location=Phoenix&species=Dog"}]}, + location_any=V.AZ_WIDE, species="Dog")) + + def test_paths_and_origin(self): + self.assertTrue(V.is_site_url("http://127.0.0.1:45003/pet/waymo")) + self.assertTrue(V.is_site_url("http://localhost:40028/")) + self.assertFalse(V.is_site_url("https://www.adoptapet.com/")) + self.assertTrue(V._same_local_origin("http://localhost:41024/account", "http://localhost:41024/")) + self.assertFalse(V._same_local_origin("http://localhost:41023/account", "http://localhost:41024/")) + + def test_password_matches_werkzeug_scrypt(self): + self.assertTrue(V.password_matches(hash_for("PetFriend123!"), "PetFriend123!")) + self.assertFalse(V.password_matches(hash_for("PetFriend123!"), "petfriend123!")) + self.assertFalse(V.password_matches("garbage", "PetFriend123!")) + + def test_frozen_catalog_search_matches_task_candidate_sets(self): + self.assertEqual([p["slug"] for p in GT.search("Phoenix, AZ", "Dog", sex="Male")], + ["horus", "waymo", "arno", "batman", "sirius", "winston", "zorro"]) + self.assertEqual([p["slug"] for p in GT.search("Phoenix", "Dog", sex="Male")], ["horus", "waymo"]) + self.assertEqual([p["slug"] for p in GT.search("Scottsdale, AZ", "Cat")], ["neo", "amba", "casper", "cinders"]) + self.assertEqual([p["slug"] for p in GT.search("Arizona", "Dog", age="Young")], ["horus"]) + self.assertEqual(len(GT.search("Arizona", "Dog")), 8) + + +if __name__ == "__main__": + unittest.main() diff --git a/sites/adopt_a_pet/verify/verify_0.py b/sites/adopt_a_pet/verify/verify_0.py new file mode 100644 index 000000000..a9360633d --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_0.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--0: dogs near Phoenix, AZ -> Waymo's breed, age group, size and fee (read-only). + +Deterministic only. Ground truth is frozen in ground_truth.py (re-validated against the +initial snapshot); nothing here reads tasks.jsonl. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_read_only, check_search_visited, check_trajectory_identity, check_visited_pet, + contains_all, contains_money, final_answer, run_verifier, +) + +TASK_ID = "AdoptAPet--0" +PET = GT.pet("waymo") # American Pit Bull Terrier / Mixed Breed, Adult, Large, $225 + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_search_visited(judge, traj, "visited_dog_search_near_phoenix", + location_any=({"phoenix"}, {"85004"}), species="Dog") + check_visited_pet(judge, traj, PET["slug"]) + judge.check("answer_has_both_breeds", contains_all(answer, GT.breeds(PET)), + f"expected={GT.breeds(PET)!r}, answer={answer!r}") + judge.check("answer_has_age_group", contains_all(answer, [PET["age_group"]]), + f"expected={PET['age_group']!r}, answer={answer!r}") + judge.check("answer_has_size", contains_all(answer, [PET["size"]]), + f"expected={PET['size']!r}, answer={answer!r}") + judge.check("answer_has_fee", contains_money(answer, PET["fee"]), + f"expected=${PET['fee']}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_1.py b/sites/adopt_a_pet/verify/verify_1.py new file mode 100644 index 000000000..7a1b156bc --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_1.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--1: cats near Scottsdale, AZ -> the kitten listed IN Scottsdale (Neo): +name, exact age in months, color, good with children (read-only). + +The Scottsdale, AZ results contain two kittens (Neo in Scottsdale, Amba in Arizona City); +the task text pins the one whose listing is in Scottsdale itself. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + CHILDREN_KW, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pet, contains_all, contains_months, final_answer, run_verifier, stated_yes_no, +) + +TASK_ID = "AdoptAPet--1" +PET = GT.pet("neo") # Neo, 7 months, Black, good with children = Yes + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_search_visited(judge, traj, "visited_cat_search_near_scottsdale", + location_any=({"scottsdale"},), species="Cat") + check_visited_pet(judge, traj, PET["slug"]) + judge.check("answer_names_pet", contains_all(answer, [PET["name"]]), f"expected={PET['name']!r}, answer={answer!r}") + judge.check("answer_has_age_months", contains_months(answer, PET["age_months"]), + f"expected={PET['age_months']} months, answer={answer!r}") + judge.check("answer_has_color", contains_all(answer, [PET["color"]]), f"expected={PET['color']!r}, answer={answer!r}") + judge.check("answer_states_good_with_children_yes", + stated_yes_no(answer, CHILDREN_KW, expected_yes=bool(PET["good_children"]), anchor_name=PET["name"]), + f"expected=good with children: {'Yes' if PET['good_children'] else 'No'}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_10.py b/sites/adopt_a_pet/verify/verify_10.py new file mode 100644 index 000000000..0e069f3c7 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_10.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--10: Find a shelter -> Seattle -> Pacific Animal Haven: name, phone, +e-mail and every pet shown (Daisy, Pepper) (read-only). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_read_only, check_trajectory_identity, check_visited_path, contains_all, contains_email, + contains_phone, final_answer, run_verifier, shelters_search_visited, +) + +TASK_ID = "AdoptAPet--10" +SHELTER = GT.shelter(4) # Pacific Animal Haven +PETS = [p for p in GT.PETS if p["shelter_id"] == SHELTER["id"]] # Daisy, Pepper + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + judge.check("used_shelter_search_for_seattle", shelters_search_visited(traj, ({"seattle"}, {"wa"}, {"pacific"})), + "required=/shelters?q~seattle") + check_visited_path(judge, traj, f"visited_shelter_{SHELTER['id']}", f"/shelter/{SHELTER['id']}") + judge.check("answer_has_shelter_name", contains_all(answer, [SHELTER["name"]]), f"expected={SHELTER['name']!r}, answer={answer!r}") + judge.check("answer_has_phone", contains_phone(answer, SHELTER["phone"]), f"expected={SHELTER['phone']!r}, answer={answer!r}") + judge.check("answer_has_email", contains_email(answer, SHELTER["email"]), f"expected={SHELTER['email']!r}, answer={answer!r}") + judge.check("answer_lists_every_pet", contains_all(answer, [p["name"] for p in PETS]), + f"expected={[p['name'] for p in PETS]!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_11.py b/sites/adopt_a_pet/verify/verify_11.py new file mode 100644 index 000000000..f8e6f1fc3 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_11.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--11: Breed 101 -> Maine Coon -> Milo: name, sex, location, fee (read-only). +The fee is only on the profile page. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_read_only, check_search_visited, check_trajectory_identity, check_visited_path, check_visited_pet, + contains_all, contains_money, final_answer, run_verifier, +) + +TASK_ID = "AdoptAPet--11" +PET = GT.pet("milo") # Male, New York, NY, $150 + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_visited_path(judge, traj, "visited_breed_101", "/breeds") + check_search_visited(judge, traj, "visited_maine_coon_results", breed="Maine Coon", species="Cat") + check_visited_pet(judge, traj, PET["slug"]) + judge.check("answer_names_pet", contains_all(answer, [PET["name"]]), f"expected={PET['name']!r}, answer={answer!r}") + judge.check("answer_has_sex", contains_all(answer, [PET["sex"]]), f"expected={PET['sex']!r}, answer={answer!r}") + judge.check("answer_has_location", contains_all(answer, [PET["city"]]), f"expected={PET['city']!r}, answer={answer!r}") + judge.check("answer_has_fee", contains_money(answer, PET["fee"]), f"expected=${PET['fee']}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_12.py b/sites/adopt_a_pet/verify/verify_12.py new file mode 100644 index 000000000..8f9d42dde --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_12.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--12: small dogs near Arizona -> youngest of five: Batman, 36 months, +Chihuahua / Yorkshire Terrier, Tucson (read-only). + +Cards show only the age GROUP; two candidates are Seniors (older by definition), the three +Adults (Batman 36, Winston 60, Zorro 72 months) must all be opened to find the youngest. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + AZ_WIDE, OLDEST, YOUNGEST, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pets, contains_all, contains_months, final_answer, identifies, run_verifier, +) + +TASK_ID = "AdoptAPet--12" +CANDIDATES = [p["slug"] for p in GT.search("Arizona", "Dog", size="Small")] # 5 small AZ dogs +MUST_OPEN = [s for s in CANDIDATES if GT.pet(s)["age_group"] == "Adult"] # batman, winston, zorro +WINNER = GT.pet("batman") +OTHERS = [GT.pet(s)["name"] for s in CANDIDATES if s != WINNER["slug"]] + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + judge.check("candidate_set_is_five", len(CANDIDATES) == 5 and min(GT.pet(s)["age_months"] for s in CANDIDATES) == WINNER["age_months"], + f"candidates={CANDIDATES!r}") + check_search_visited(judge, traj, "visited_small_dog_search_arizona", location_any=AZ_WIDE, species="Dog", size="Small") + check_visited_pets(judge, traj, MUST_OPEN) + judge.check("answer_names_winner", contains_all(answer, [WINNER["name"]]), f"expected={WINNER['name']!r}, answer={answer!r}") + judge.check("answer_has_age_months", contains_months(answer, WINNER["age_months"]), f"expected={WINNER['age_months']} months, answer={answer!r}") + judge.check("answer_has_both_breeds", contains_all(answer, GT.breeds(WINNER)), f"expected={GT.breeds(WINNER)!r}, answer={answer!r}") + judge.check("answer_has_city", contains_all(answer, [WINNER["city"]]), f"expected={WINNER['city']!r}, answer={answer!r}") + judge.check("answer_identifies_youngest", identifies(answer, WINNER["name"], OTHERS, YOUNGEST, OLDEST), + f"winner={WINNER['name']!r}, others={OTHERS!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_13.py b/sites/adopt_a_pet/verify/verify_13.py new file mode 100644 index 000000000..bfad79329 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_13.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--13: adult female dogs near Austin, TX -> Ruby: fee, good with cats (No), +good with children (No) (read-only). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + CATS_KW, CHILDREN_KW, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pet, contains_all, contains_money, final_answer, run_verifier, stated_yes_no, +) + +TASK_ID = "AdoptAPet--13" +PET = GT.pet("ruby") # $215, good with cats No, good with children No + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_search_visited(judge, traj, "visited_female_dog_search_austin", + location_any=({"austin"}, {"tx"}, {"texas"}), species="Dog", sex="Female") + check_visited_pet(judge, traj, PET["slug"]) + judge.check("answer_names_pet", contains_all(answer, [PET["name"]]), f"expected={PET['name']!r}, answer={answer!r}") + judge.check("answer_has_fee", contains_money(answer, PET["fee"]), f"expected=${PET['fee']}, answer={answer!r}") + judge.check("answer_states_good_with_cats_no", + stated_yes_no(answer, CATS_KW, expected_yes=bool(PET["good_cats"]), anchor_name=PET["name"]), + f"expected=good with cats: {'Yes' if PET['good_cats'] else 'No'}, answer={answer!r}") + judge.check("answer_states_good_with_children_no", + stated_yes_no(answer, CHILDREN_KW, expected_yes=bool(PET["good_children"]), anchor_name=PET["name"]), + f"expected=good with children: {'Yes' if PET['good_children'] else 'No'}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_14.py b/sites/adopt_a_pet/verify/verify_14.py new file mode 100644 index 000000000..dc970908b --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_14.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--14: pets near New York, NY -> Luna ($250) vs Milo ($150), lower = Milo (read-only).""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + HIGHEST, LOWEST, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pets, contains_all, contains_money, final_answer, identifies, run_verifier, +) + +TASK_ID = "AdoptAPet--14" +LUNA, MILO = GT.pet("luna"), GT.pet("milo") + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_search_visited(judge, traj, "visited_search_near_new_york", location_any=({"new", "york"}, {"ny"})) + check_visited_pets(judge, traj, [LUNA["slug"], MILO["slug"]]) + judge.check("answer_names_both_pets", contains_all(answer, [LUNA["name"], MILO["name"]]), f"answer={answer!r}") + judge.check("answer_has_both_fees", contains_money(answer, LUNA["fee"]) and contains_money(answer, MILO["fee"]), + f"expected=${LUNA['fee']} and ${MILO['fee']}, answer={answer!r}") + judge.check("answer_identifies_lower_fee_pet", identifies(answer, MILO["name"], [LUNA["name"]], LOWEST, HIGHEST), + f"winner={MILO['name']!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_15.py b/sites/adopt_a_pet/verify/verify_15.py new file mode 100644 index 000000000..0326cec9e --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_15.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--15: log in as david.b, favorite Archie and Ruby, confirm both in Favorite +pets (stateful: favorite +2 rows exactly, account visited after both profiles). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_exact_favorite_delta, check_paths_in_order, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, contains_all, favorite_slugs, final_answer, run_verifier, +) + +TASK_ID = "AdoptAPet--15" +EMAIL = "david.b@test.com" +PETS = [GT.pet("archie"), GT.pet("ruby")] + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_signed_in_as(judge, traj, EMAIL) + for pet in PETS: + check_paths_in_order(judge, traj, f"login_then_{pet['slug']}_then_account", + [("/login", {}), (f"/pet/{pet['slug']}", {}), ("/account", {})]) + check_exact_favorite_delta(judge, initial_db, after_db, added=[(EMAIL, p["slug"]) for p in PETS], removed=[]) + judge.check("account_lists_both_pets", favorite_slugs(after_db, EMAIL) == {p["slug"] for p in PETS}, + f"after_favorites={sorted(favorite_slugs(after_db, EMAIL))!r}") + check_tables_unchanged(judge, initial_db, after_db, ("user", "application", "pet_alert")) + judge.check("answer_confirms_both_pets", contains_all(answer, [p["name"] for p in PETS]), f"answer={answer!r}") + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_16.py b/sites/adopt_a_pet/verify/verify_16.py new file mode 100644 index 000000000..1c8c0c727 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_16.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--16: register Jamie Lee (jamie.lee@example.test / PetFriend123!), favorite +Olive, confirm in the new account (stateful: user +1 with a verifiable password hash, +favorite +1 for that user, nothing else). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_exact_favorite_delta, check_paths_in_order, check_tables_unchanged, check_trajectory_identity, + contains_all, favorite_slugs, final_answer, normalize_text, password_matches, run_verifier, table_delta, + trajectory_emails, user_row, +) + +TASK_ID = "AdoptAPet--16" +EMAIL = "jamie.lee@example.test" +NAME = "Jamie Lee" +PASSWORD = "PetFriend123!" +PET = GT.pet("olive") + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + judge.check("registered_through_form", any(e == normalize_text(EMAIL) for e in trajectory_emails(traj)), + f"expected_email={EMAIL!r}, entered_emails={trajectory_emails(traj)!r}") + check_paths_in_order(judge, traj, "register_then_pet_then_account", + [("/register", {}), (f"/pet/{PET['slug']}", {}), ("/account", {})]) + delta = table_delta(initial_db, after_db, "user") + judge.check("exactly_one_user_added", len(delta["added"]) == 1 and not delta["removed"] and not delta["changed"], + f"added={[{k: v for k, v in r.items() if k != 'password_hash'} for r in delta['added']]!r}, " + f"removed={len(delta['removed'])}, changed={len(delta['changed'])}") + row = user_row(after_db, EMAIL) + judge.check("new_user_has_expected_email_and_name", + bool(row) and normalize_text(row["email"]) == normalize_text(EMAIL) and normalize_text(row["name"]) == normalize_text(NAME), + f"expected=({EMAIL!r}, {NAME!r}), observed={({k: v for k, v in row.items() if k != 'password_hash'} if row else None)!r}") + judge.check("new_user_password_verifies", bool(row) and password_matches(row["password_hash"], PASSWORD), + f"hash_method={(row['password_hash'].split('$', 1)[0] if row else None)!r}") + check_exact_favorite_delta(judge, initial_db, after_db, added=[(EMAIL, PET["slug"])], removed=[]) + judge.check("new_account_lists_olive", favorite_slugs(after_db, EMAIL) == {PET["slug"]}, + f"after_favorites={sorted(favorite_slugs(after_db, EMAIL))!r}") + check_tables_unchanged(judge, initial_db, after_db, ("application", "pet_alert")) + judge.check("answer_confirms_pet", contains_all(answer, [PET["name"]]), f"answer={answer!r}") + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_17.py b/sites/adopt_a_pet/verify/verify_17.py new file mode 100644 index 000000000..c1063bd0a --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_17.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--17: senior dogs near Miami, FL -> Teddy: 96 months, Poodle / Mixed Breed, +$185, house-trained Yes (read-only). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + HOUSE_TRAINED_KW, Judge, check_read_only, check_search_visited, check_trajectory_identity, check_visited_pet, + contains_all, contains_money, contains_months, final_answer, run_verifier, stated_yes_no, +) + +TASK_ID = "AdoptAPet--17" +PET = GT.pet("teddy") + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_search_visited(judge, traj, "visited_senior_dog_search_miami", + location_any=({"miami"}, {"fl"}, {"florida"}), species="Dog", age="Senior") + check_visited_pet(judge, traj, PET["slug"]) + judge.check("answer_names_pet", contains_all(answer, [PET["name"]]), f"expected={PET['name']!r}, answer={answer!r}") + judge.check("answer_has_age_months", contains_months(answer, PET["age_months"]), f"expected={PET['age_months']} months, answer={answer!r}") + judge.check("answer_has_both_breeds", contains_all(answer, GT.breeds(PET)), f"expected={GT.breeds(PET)!r}, answer={answer!r}") + judge.check("answer_has_fee", contains_money(answer, PET["fee"]), f"expected=${PET['fee']}, answer={answer!r}") + judge.check("answer_states_house_trained_yes", + stated_yes_no(answer, HOUSE_TRAINED_KW, expected_yes=bool(PET["house_trained"]), anchor_name=PET["name"]), + f"expected=house-trained: {'Yes' if PET['house_trained'] else 'No'}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_18.py b/sites/adopt_a_pet/verify/verify_18.py new file mode 100644 index 000000000..cdc899161 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_18.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--18: Pet advice page -> titles of the paperwork, adoption-fee and +bringing-home-a-dog articles (read-only). Titles are page-specific strings; punctuation and +quoting differences are tolerated, wording is not. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_read_only, check_trajectory_identity, check_visited_path, contains_phrase_loose, final_answer, run_verifier, +) + +TASK_ID = "AdoptAPet--18" +TITLES = [GT.BLOG_TITLES[0], GT.BLOG_TITLES[1], GT.BLOG_TITLES[2]] + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_visited_path(judge, traj, "visited_pet_advice", "/blog") + for idx, title in enumerate(TITLES): + judge.check(f"answer_has_title_{idx}", contains_phrase_loose(answer, title), f"expected={title!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_19.py b/sites/adopt_a_pet/verify/verify_19.py new file mode 100644 index 000000000..a503f856d --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_19.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--19: dogs near Arizona -> among the profiles marked good with children, +the lowest adoption fee: Arno, Casa Grande, German Shepherd Dog / Mixed Breed, 43 months, +$200 (read-only). + +Eight Arizona dogs span two results pages; "good with children" and the fee are only on the +profile pages, so every one of the eight profiles and results page 2 are required. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + AZ_WIDE, HIGHEST, LOWEST, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pets, contains_all, contains_money, contains_months, final_answer, identifies, run_verifier, +) + +TASK_ID = "AdoptAPet--19" +CANDIDATES = [p["slug"] for p in GT.search("Arizona", "Dog")] # 8 AZ dogs +QUALIFYING = [s for s in CANDIDATES if GT.pet(s)["good_children"]] # arno, horus, waymo, yuki +WINNER = GT.pet("arno") +OTHERS = [GT.pet(s)["name"] for s in CANDIDATES if s != WINNER["slug"]] + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + judge.check("candidate_set_is_eight_with_four_qualifying", + len(CANDIDATES) == 8 and len(QUALIFYING) == 4 and min(GT.pet(s)["fee"] for s in QUALIFYING) == WINNER["fee"], + f"candidates={CANDIDATES!r}, qualifying={QUALIFYING!r}") + check_search_visited(judge, traj, "visited_dog_search_arizona", location_any=AZ_WIDE, species="Dog") + check_search_visited(judge, traj, "visited_results_page_2", location_any=AZ_WIDE, species="Dog", page="2") + check_visited_pets(judge, traj, CANDIDATES) + judge.check("answer_names_winner", contains_all(answer, [WINNER["name"]]), f"expected={WINNER['name']!r}, answer={answer!r}") + judge.check("answer_has_city", contains_all(answer, [WINNER["city"]]), f"expected={WINNER['city']!r}, answer={answer!r}") + judge.check("answer_has_both_breeds", contains_all(answer, GT.breeds(WINNER)), f"expected={GT.breeds(WINNER)!r}, answer={answer!r}") + judge.check("answer_has_age_months", contains_months(answer, WINNER["age_months"]), f"expected={WINNER['age_months']} months, answer={answer!r}") + judge.check("answer_has_fee", contains_money(answer, WINNER["fee"]), f"expected=${WINNER['fee']}, answer={answer!r}") + judge.check("answer_identifies_lowest_fee_pet", identifies(answer, WINNER["name"], OTHERS, LOWEST, HIGHEST), + f"winner={WINNER['name']!r}, others={OTHERS!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_2.py b/sites/adopt_a_pet/verify/verify_2.py new file mode 100644 index 000000000..ec0c3f731 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_2.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--2: location "Phoenix, AZ" + male dogs -> lowest adoption fee across ALL +matching profiles (7 dogs on two results pages): Batman, Chihuahua / Yorkshire Terrier, $165. + +The mirror scores location by token overlap, so "Phoenix, AZ" returns every Arizona pet; +fees are only on detail pages, so all seven profiles and results page 2 are required. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + AZ_WIDE, HIGHEST, LOWEST, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pets, contains_all, contains_money, final_answer, identifies, run_verifier, +) + +TASK_ID = "AdoptAPet--2" +CANDIDATES = [p["slug"] for p in GT.search("Phoenix, AZ", "Dog", sex="Male")] # 7 male AZ dogs +WINNER = GT.pet("batman") +OTHERS = [GT.pet(s)["name"] for s in CANDIDATES if s != WINNER["slug"]] + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + judge.check("candidate_set_is_seven", len(CANDIDATES) == 7 and min(GT.pet(s)["fee"] for s in CANDIDATES) == WINNER["fee"], + f"candidates={CANDIDATES!r}") + check_search_visited(judge, traj, "visited_male_dog_search_statewide_az", + location_any=AZ_WIDE, species="Dog", sex="Male") + check_search_visited(judge, traj, "visited_results_page_2", + location_any=AZ_WIDE, species="Dog", sex="Male", page="2") + check_visited_pets(judge, traj, CANDIDATES) + judge.check("answer_names_winner", contains_all(answer, [WINNER["name"]]), f"expected={WINNER['name']!r}, answer={answer!r}") + judge.check("answer_has_both_breeds", contains_all(answer, GT.breeds(WINNER)), + f"expected={GT.breeds(WINNER)!r}, answer={answer!r}") + judge.check("answer_has_fee", contains_money(answer, WINNER["fee"]), f"expected=${WINNER['fee']}, answer={answer!r}") + judge.check("answer_identifies_lowest_fee_pet", identifies(answer, WINNER["name"], OTHERS, LOWEST, HIGHEST), + f"winner={WINNER['name']!r}, others={OTHERS!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_3.py b/sites/adopt_a_pet/verify/verify_3.py new file mode 100644 index 000000000..441edf321 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_3.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--3: adult cats near Arizona, house-trained + good with cats, lowest fee -> +Cinders, Sedona, Domestic Shorthair, $120 (read-only). Both adult AZ cats qualify, so both +detail pages are required to compare fees. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + AZ_WIDE, HIGHEST, LOWEST, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pets, contains_all, contains_money, final_answer, identifies, run_verifier, +) + +TASK_ID = "AdoptAPet--3" +CANDIDATES = [p["slug"] for p in GT.search("Arizona", "Cat", age="Adult")] # casper, cinders +WINNER = GT.pet("cinders") +OTHERS = [GT.pet(s)["name"] for s in CANDIDATES if s != WINNER["slug"]] + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + judge.check("candidate_set_is_two_qualifying", len(CANDIDATES) == 2 and all( + GT.pet(s)["house_trained"] and GT.pet(s)["good_cats"] for s in CANDIDATES), f"candidates={CANDIDATES!r}") + check_search_visited(judge, traj, "visited_adult_cat_search_arizona", location_any=AZ_WIDE, species="Cat", age="Adult") + check_visited_pets(judge, traj, CANDIDATES) + judge.check("answer_names_winner", contains_all(answer, [WINNER["name"]]), f"expected={WINNER['name']!r}, answer={answer!r}") + judge.check("answer_has_city", contains_all(answer, [WINNER["city"]]), f"expected={WINNER['city']!r}, answer={answer!r}") + judge.check("answer_has_breed", contains_all(answer, [WINNER["breed"]]), f"expected={WINNER['breed']!r}, answer={answer!r}") + judge.check("answer_has_fee", contains_money(answer, WINNER["fee"]), f"expected=${WINNER['fee']}, answer={answer!r}") + judge.check("answer_identifies_lowest_fee_pet", identifies(answer, WINNER["name"], OTHERS, LOWEST, HIGHEST), + f"winner={WINNER['name']!r}, others={OTHERS!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_4.py b/sites/adopt_a_pet/verify/verify_4.py new file mode 100644 index 000000000..f7bb4484e --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_4.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--4: find Arno through the pet search, then the rescue's name, phone and +e-mail from the pet profile + linked shelter page (read-only). + +The profile names the rescue; phone and e-mail are only on /shelter/. +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_read_only, check_trajectory_identity, check_visited_path, check_visited_pet, + contains_all, contains_email, contains_phone, final_answer, navigated_to_path, run_verifier, +) + +TASK_ID = "AdoptAPet--4" +PET = GT.pet("arno") +SHELTER = GT.shelter(PET["shelter_id"]) # Desert Paws Rescue, 602-555-0141, hello@desertpaws.test + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + judge.check("used_pet_search", navigated_to_path(traj, "/search"), "required_path=/search (any query)") + check_visited_pet(judge, traj, PET["slug"]) + check_visited_path(judge, traj, f"visited_shelter_{SHELTER['id']}", f"/shelter/{SHELTER['id']}") + judge.check("answer_has_rescue_name", contains_all(answer, [SHELTER["name"]]), f"expected={SHELTER['name']!r}, answer={answer!r}") + judge.check("answer_has_rescue_phone", contains_phone(answer, SHELTER["phone"]), f"expected={SHELTER['phone']!r}, answer={answer!r}") + judge.check("answer_has_rescue_email", contains_email(answer, SHELTER["email"]), f"expected={SHELTER['email']!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_5.py b/sites/adopt_a_pet/verify/verify_5.py new file mode 100644 index 000000000..ddd166b55 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_5.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--5: pets near Seattle, WA -> Daisy vs Pepper: species, age in months, +fee, and which has the lower fee (Pepper) (read-only). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + HIGHEST, LOWEST, Judge, check_read_only, check_search_visited, check_trajectory_identity, + check_visited_pets, contains_all, contains_money, contains_months, final_answer, identifies, run_verifier, +) + +TASK_ID = "AdoptAPet--5" +DAISY, PEPPER = GT.pet("daisy"), GT.pet("pepper") # Dog 30 mo $275 / Cat 13 mo $130 + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_search_visited(judge, traj, "visited_search_near_seattle", location_any=({"seattle"}, {"wa"}, {"washington"})) + check_visited_pets(judge, traj, [DAISY["slug"], PEPPER["slug"]]) + judge.check("answer_names_both_pets", contains_all(answer, [DAISY["name"], PEPPER["name"]]), f"answer={answer!r}") + judge.check("answer_has_both_species", contains_all(answer, [DAISY["species"], PEPPER["species"]]), + f"expected={[DAISY['species'], PEPPER['species']]!r}, answer={answer!r}") + judge.check("answer_has_both_ages", contains_months(answer, DAISY["age_months"]) and contains_months(answer, PEPPER["age_months"]), + f"expected={DAISY['age_months']} and {PEPPER['age_months']} months, answer={answer!r}") + judge.check("answer_has_both_fees", contains_money(answer, DAISY["fee"]) and contains_money(answer, PEPPER["fee"]), + f"expected=${DAISY['fee']} and ${PEPPER['fee']}, answer={answer!r}") + judge.check("answer_identifies_lower_fee_pet", identifies(answer, PEPPER["name"], [DAISY["name"]], LOWEST, HIGHEST), + f"winner={PEPPER['name']!r}, answer={answer!r}") + check_read_only(judge, initial_db, after_db) + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_6.py b/sites/adopt_a_pet/verify/verify_6.py new file mode 100644 index 000000000..5caf40d39 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_6.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--6: log in as alice.j, favorite Sirius, confirm Sirius + Luna under +Favorite pets (stateful: favorite +1 row exactly, login -> /pet/sirius -> /account order). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_exact_favorite_delta, check_paths_in_order, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, contains_all, favorite_slugs, final_answer, run_verifier, +) + +TASK_ID = "AdoptAPet--6" +EMAIL = "alice.j@test.com" +ADDED = GT.pet("sirius") +EXISTING = GT.pet("luna") + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_signed_in_as(judge, traj, EMAIL) + check_paths_in_order(judge, traj, "login_then_pet_then_account", + [("/login", {}), (f"/pet/{ADDED['slug']}", {}), ("/account", {})]) + check_exact_favorite_delta(judge, initial_db, after_db, added=[(EMAIL, ADDED["slug"])], removed=[]) + judge.check("account_lists_both_pets", favorite_slugs(after_db, EMAIL) == {ADDED["slug"], EXISTING["slug"]}, + f"after_favorites={sorted(favorite_slugs(after_db, EMAIL))!r}") + check_tables_unchanged(judge, initial_db, after_db, ("user", "application", "pet_alert")) + judge.check("answer_confirms_both_pets", contains_all(answer, [ADDED["name"], EXISTING["name"]]), f"answer={answer!r}") + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_7.py b/sites/adopt_a_pet/verify/verify_7.py new file mode 100644 index 000000000..50437c7f9 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_7.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--7: log in as alice.j, remove Luna from favorites, confirm she is gone +(stateful: favorite -1 row exactly; the only remove control is on /pet/luna). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_exact_favorite_delta, check_paths_in_order, check_signed_in_as, check_tables_unchanged, + check_trajectory_identity, contains_all, favorite_slugs, final_answer, run_verifier, +) + +TASK_ID = "AdoptAPet--7" +EMAIL = "alice.j@test.com" +REMOVED = GT.pet("luna") + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_signed_in_as(judge, traj, EMAIL) + check_paths_in_order(judge, traj, "login_then_pet_then_account", + [("/login", {}), (f"/pet/{REMOVED['slug']}", {}), ("/account", {})]) + check_exact_favorite_delta(judge, initial_db, after_db, added=[], removed=[(EMAIL, REMOVED["slug"])]) + judge.check("account_has_no_favorites_left", favorite_slugs(after_db, EMAIL) == set(), + f"after_favorites={sorted(favorite_slugs(after_db, EMAIL))!r}") + check_tables_unchanged(judge, initial_db, after_db, ("user", "application", "pet_alert")) + judge.check("answer_mentions_pet", contains_all(answer, [REMOVED["name"]]), f"answer={answer!r}") + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_8.py b/sites/adopt_a_pet/verify/verify_8.py new file mode 100644 index 000000000..6acc8e3bb --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_8.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--8: log in as bob.smith, submit an adoption inquiry for Daisy with the +given phone / housing / experience, confirm it is Submitted in the account +(stateful: application +1 row with exact field values). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import ground_truth as GT # noqa: E402 +from verify_lib import ( # noqa: E402 + Judge, check_paths_in_order, check_signed_in_as, check_tables_unchanged, check_trajectory_identity, + contains_all, digits_only, final_answer, normalize_text, run_verifier, table_delta, user_id_for_email, +) + +TASK_ID = "AdoptAPet--8" +EMAIL = "bob.smith@test.com" +PET = GT.pet("daisy") +PHONE = "206-555-0199" +HOUSING = "Rent with permission" +EXPERIENCE = "I have cared for two family dogs for eight years." + + +def _squash(text: object) -> str: + return normalize_text(text).rstrip(" .!") + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_signed_in_as(judge, traj, EMAIL) + check_paths_in_order(judge, traj, "login_pet_apply_account", + [("/login", {}), (f"/pet/{PET['slug']}", {}), (f"/apply/{PET['slug']}", {}), ("/account", {})]) + delta = table_delta(initial_db, after_db, "application") + judge.check("exactly_one_application_added", len(delta["added"]) == 1 and not delta["removed"] and not delta["changed"], + f"added={delta['added']!r}, removed={delta['removed']!r}, changed={delta['changed']!r}") + row = delta["added"][0] if delta["added"] else {} + uid = user_id_for_email(after_db, EMAIL) + judge.check("application_belongs_to_bob_for_daisy", row.get("user_id") == uid and row.get("pet_id") == PET["id"], + f"expected user_id={uid} pet_id={PET['id']}, row={row!r}") + judge.check("application_housing_exact", normalize_text(row.get("housing")) == normalize_text(HOUSING), + f"expected={HOUSING!r}, observed={row.get('housing')!r}") + judge.check("application_phone_exact", digits_only(row.get("phone")) == digits_only(PHONE), + f"expected={PHONE!r}, observed={row.get('phone')!r}") + judge.check("application_experience_exact", _squash(row.get("experience")) == _squash(EXPERIENCE), + f"expected={EXPERIENCE!r}, observed={row.get('experience')!r}") + judge.check("application_status_submitted", normalize_text(row.get("status")) == "submitted", + f"observed_status={row.get('status')!r}") + check_tables_unchanged(judge, initial_db, after_db, ("user", "favorite", "pet_alert")) + judge.check("answer_confirms_submitted_for_daisy", contains_all(answer, [PET["name"], "Submitted"]), f"answer={answer!r}") + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_9.py b/sites/adopt_a_pet/verify/verify_9.py new file mode 100644 index 000000000..b4b1c8320 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_9.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Verify AdoptAPet--9: log in as carol.w, create a New Pet Alert (Cat / Siamese / 33130 / +50 miles), confirm it in the account (stateful: pet_alert +1 row with exact values). +""" +from __future__ import annotations + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from verify_lib import ( # noqa: E402 + Judge, check_paths_in_order, check_signed_in_as, check_tables_unchanged, check_trajectory_identity, + contains_all, contains_count, final_answer, normalize_text, run_verifier, table_delta, user_id_for_email, +) + +TASK_ID = "AdoptAPet--9" +EMAIL = "carol.w@test.com" +SPECIES, BREED, POSTAL, RADIUS = "Cat", "Siamese", "33130", 50 + + +def run_checks(judge: Judge, traj: dict, initial_db: str, after_db: str) -> None: + check_trajectory_identity(judge, traj, TASK_ID) + answer = final_answer(traj) + check_signed_in_as(judge, traj, EMAIL) + check_paths_in_order(judge, traj, "login_alerts_account", [("/login", {}), ("/alerts", {}), ("/account", {})]) + delta = table_delta(initial_db, after_db, "pet_alert") + judge.check("exactly_one_alert_added", len(delta["added"]) == 1 and not delta["removed"] and not delta["changed"], + f"added={delta['added']!r}, removed={delta['removed']!r}, changed={delta['changed']!r}") + row = delta["added"][0] if delta["added"] else {} + uid = user_id_for_email(after_db, EMAIL) + judge.check("alert_belongs_to_carol", row.get("user_id") == uid, f"expected user_id={uid}, row={row!r}") + judge.check("alert_values_exact", + normalize_text(row.get("species")) == normalize_text(SPECIES) and normalize_text(row.get("breed")) == normalize_text(BREED) + and str(row.get("postal") or "").strip() == POSTAL and int(row.get("radius") or 0) == RADIUS, + f"expected=({SPECIES}, {BREED}, {POSTAL}, {RADIUS}), row={row!r}") + check_tables_unchanged(judge, initial_db, after_db, ("user", "favorite", "application")) + judge.check("answer_confirms_alert", contains_all(answer, [BREED, POSTAL]) and contains_count(answer, RADIUS), f"answer={answer!r}") + + +if __name__ == "__main__": + run_verifier(TASK_ID, run_checks) diff --git a/sites/adopt_a_pet/verify/verify_lib.py b/sites/adopt_a_pet/verify/verify_lib.py new file mode 100644 index 000000000..d98a49656 --- /dev/null +++ b/sites/adopt_a_pet/verify/verify_lib.py @@ -0,0 +1,980 @@ +#!/usr/bin/env python3 +"""verify_lib.py — shared deterministic + (parity-only) LLM utilities for Adopt-a-Pet task verification. + +Philosophy: DETERMINISTIC FIRST, fail closed. + 1. Package validation: task id, non-empty final answer, agent_done termination, at least + one step, every recorded URL on the same loopback origin (host + port) as start_url, + every referenced screenshot present and decodable as a PNG. + 2. Navigation gates (anti knowledge-shortcut): the agent MUST have opened the pages a + human needs to read the hidden facts (results pages with the task's filters, every + detail page whose hidden value a comparison depends on, the shelter page, ...). + Stateful tasks additionally enforce page ORDER (login -> action page -> account). + 3. Answer checks: affirmative token / money / month / phone / e-mail / yes-no matchers + against frozen ground truth (``ground_truth.py``). + 4. SQLite snapshot contract: initial and after snapshots must both be supplied (explicit + flags, ``/initial.db`` + ``/after.db``, or ``docker cp`` from + ``$WH_CONTAINER``). Schema + immutable catalog must be identical and must match the + frozen catalog; read-only tasks require the mutable tables to be row-identical; + stateful tasks require EXACT row deltas (no collateral writes, no duplicates). + 5. LLM utilities are kept for API parity with sites/merriam_webster/verify only. No + verdict depends on them; with ``--no_llm True`` they short-circuit. + +Input signature (per task): + --run_dir DIR agent trajectory dir: trajectory.json + screenshots/step_NNN.png + --initial_db PATH initial-state SQLite DB (default: /initial.db, else docker cp instance_seed) + --after_db PATH after-state SQLite DB (default: /after.db, else docker cp instance) + --container NAME docker container to fetch DBs from (default: $WH_CONTAINER or wh-review) + --no_llm True skip LLM-based checks (deterministic-only; the default verdict path) +Output: JSON {task_id, pass, reason, evidence[]} to stdout; exit 0 on PASS, 1 on FAIL. + +NOTE: no ``from __future__ import annotations`` here — simpleArgParser reads the +VerifyArgs dataclass annotations at runtime and needs real types, not strings. +""" +import atexit +import base64 +import hashlib +import hmac +import ipaddress +import json +import os +import re +import sqlite3 +import struct +import subprocess +import sys +import tempfile +import unicodedata +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Sequence +from urllib.parse import parse_qs, urlparse + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import ground_truth as GT # noqa: E402 + +SITE = "adopt_a_pet" +DEFAULT_CONTAINER = os.environ.get("WH_CONTAINER", "wh-review") + +MUTABLE_TABLES = ("user", "favorite", "application", "pet_alert") +IMMUTABLE_TABLES = ("pet", "shelter") +EXPECTED_COLUMNS: dict[str, tuple[str, ...]] = { + "user": ("id", "email", "name", "password_hash"), + "shelter": ("id", "name", "city", "state", "phone", "email"), + "pet": ("id", "slug", "name", "species", "breed", "secondary_breed", "sex", "age_group", + "age_months", "size", "color", "city", "state", "postal", "fee", "image", + "description", "house_trained", "good_dogs", "good_cats", "good_children", "shelter_id"), + "favorite": ("id", "user_id", "pet_id"), + "application": ("id", "user_id", "pet_id", "housing", "experience", "phone", "status"), + "pet_alert": ("id", "user_id", "species", "breed", "postal", "radius"), +} +INITIAL_COUNTS = {"user": 4, "shelter": 6, "pet": 20, "favorite": 1, "application": 0, "pet_alert": 0} + +# Location token sets that make the mirror return EVERY Arizona pet (app.search scores +# "city state postal" token overlap, so any query containing the state token matches all). +AZ_WIDE = ({"az"}, {"arizona"}) + + +# --------------------------------------------------------------------------- # +# CLI / run loading +# --------------------------------------------------------------------------- # +import simpleArgParser as sap # noqa: E402 (available in agent_demo's uv env) + + +def parse_args(): + @dataclass + class VerifyArgs: + run_dir: str = "" + initial_db: str = "" + after_db: str = "" + container: str = DEFAULT_CONTAINER + no_llm: bool = False + + def post_process(self): + if not self.run_dir: + raise SystemExit("--run_dir is required") + + args = sap.parse_args(VerifyArgs) + if not args.run_dir: + raise SystemExit("--run_dir is required") + run_dir = Path(args.run_dir) + if not args.initial_db and (run_dir / "initial.db").is_file(): + args.initial_db = str(run_dir / "initial.db") + if not args.after_db and (run_dir / "after.db").is_file(): + args.after_db = str(run_dir / "after.db") + return args + + +def load_run(run_dir: str | os.PathLike[str]) -> dict[str, Any]: + d = Path(run_dir) + data = json.loads((d / "trajectory.json").read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("trajectory.json must contain a JSON object") + data["_run_dir"] = str(d.resolve()) + data["_shots"] = {p.name: p for p in sorted((d / "screenshots").glob("step_*.png"))} if (d / "screenshots").is_dir() else {} + return data + + +def final_answer(traj: dict[str, Any]) -> str: + return str(traj.get("final_answer") or "").strip() + + +def final_url(traj: dict[str, Any]) -> str: + if traj.get("final_url"): + return str(traj["final_url"]) + for step in reversed(traj.get("steps") or []): + if isinstance(step, dict) and step.get("url"): + return str(step["url"]) + return "" + + +def step_urls(traj: dict[str, Any]) -> list[str]: + return [str(s.get("url", "")) for s in traj.get("steps", []) if isinstance(s, dict)] + + +def trajectory_urls(traj: dict[str, Any]) -> list[str]: + """start_url, every step URL (url / url_before / url_after) and final_url, in order.""" + urls: list[str] = [] + if traj.get("start_url"): + urls.append(str(traj["start_url"])) + for step in traj.get("steps") or []: + if not isinstance(step, dict): + continue + for key in ("url", "url_before", "url_after"): + if step.get(key): + urls.append(str(step[key])) + if traj.get("final_url"): + urls.append(str(traj["final_url"])) + return urls + + +def is_site_url(url: str) -> bool: + """HTTP(S) on a loopback host, any port (runs use alt ports like 41024 or 45003).""" + parsed = urlparse(str(url or "")) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + return False + host = parsed.hostname.casefold() + if host == "localhost": + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def site_urls(traj: dict[str, Any]) -> list[str]: + return [u for u in trajectory_urls(traj) if is_site_url(u)] + + +def normalized_url_path(url: str) -> str: + path = urlparse(str(url or "")).path or "/" + return path.rstrip("/") or "/" + + +def navigated_to_path(traj: dict[str, Any], expected_path: str) -> bool: + expected = normalized_url_path(expected_path) + return any(normalized_url_path(u) == expected for u in site_urls(traj)) + + +def navigated_to(traj: dict[str, Any], substr: str, times: int = 1) -> bool: + """merriam_webster parity: at least `times` recorded site URLs contain substr.""" + return sum(1 for u in site_urls(traj) if substr in u) >= times + + +def pet_visited(traj: dict[str, Any], slug: str) -> bool: + return navigated_to_path(traj, f"/pet/{slug}") + + +def shelter_visited(traj: dict[str, Any], shelter_id: int) -> bool: + return navigated_to_path(traj, f"/shelter/{int(shelter_id)}") + + +def _tokens(text: Any) -> set[str]: + return set(re.findall(r"[a-z0-9]+", normalize_text(text))) + + +def _query_of(url: str) -> dict[str, list[str]]: + return parse_qs(urlparse(url).query, keep_blank_values=True) + + +def _location_ok(query: dict[str, list[str]], location_any: Sequence[set[str]] | None) -> bool: + if not location_any: + return True + recorded = _tokens(" ".join(query.get("location") or [])) + return any(set(alt) <= recorded for alt in location_any) + + +def _param_ok(query: dict[str, list[str]], key: str, expected: Any) -> bool: + values = [normalize_text(v) for v in (query.get(key) or [])] + if key == "breed": + return any(normalize_text(expected) in v for v in values) + if isinstance(expected, (tuple, list, set, frozenset)): + return any(normalize_text(alt) in values for alt in expected) + return normalize_text(expected) in values + + +def search_visits(traj: dict[str, Any]) -> list[str]: + return [u for u in site_urls(traj) if normalized_url_path(u) == "/search"] + + +def search_visited(traj: dict[str, Any], location_any: Sequence[set[str]] | None = None, **params: Any) -> bool: + """Some /search visit carries the requested location tokens AND every exact param. + + ``location_any`` is a list of token sets; the recorded ``location`` value must contain + every token of at least one set ("Phoenix, AZ" -> {phoenix, az}). ``breed`` is a + normalized substring; other params (species, sex, age, size, page) are exact values + (a tuple means any of the alternatives). + """ + for url in search_visits(traj): + query = _query_of(url) + if _location_ok(query, location_any) and all(_param_ok(query, k, v) for k, v in params.items()): + return True + return False + + +def shelters_search_visited(traj: dict[str, Any], q_any: Sequence[set[str]]) -> bool: + for url in site_urls(traj): + if normalized_url_path(url) != "/shelters": + continue + recorded = _tokens(" ".join(_query_of(url).get("q") or [])) + if any(set(alt) <= recorded for alt in q_any): + return True + return False + + +def trajectory_task_matches(traj: dict[str, Any], task_id: str) -> bool: + return str(traj.get("task_id") or "").strip() == task_id + + +def trajectory_input_texts(traj: dict[str, Any]) -> list[str]: + values: list[str] = [] + for step in traj.get("steps") or []: + if not isinstance(step, dict) or normalize_text(step.get("action")) != "input": + continue + params = step.get("params") + if isinstance(params, dict) and params.get("text") is not None: + values.append(str(params["text"])) + return values + + +def trajectory_input_contains(traj: dict[str, Any], expected_text: str) -> bool: + expected = normalize_text(expected_text) + return any(normalize_text(v) == expected for v in trajectory_input_texts(traj)) + + +def trajectory_last_email(traj: dict[str, Any]) -> str: + emails = [normalize_text(v) for v in trajectory_input_texts(traj) + if re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", v.strip())] + return emails[-1] if emails else "" + + +def trajectory_emails(traj: dict[str, Any]) -> list[str]: + return [normalize_text(v) for v in trajectory_input_texts(traj) + if re.fullmatch(r"[^@\s]+@[^@\s]+\.[^@\s]+", v.strip())] + + +# --------------------------------------------------------------------------- # +# Text normalization and answer matchers (affirmative, negation-aware) +# --------------------------------------------------------------------------- # +def normalize_text(value: Any) -> str: + text = unicodedata.normalize("NFKC", str(value or "")) + text = text.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"') + return re.sub(r"\s+", " ", text).strip().casefold() + + +def norm(s: Any) -> str: # merriam_webster parity + return normalize_text(s) + + +NEGATION = r"\b(?:not|no|never|without|wrong|incorrect|isn't|wasn't|isnt|wasnt|aren't|arent|neither|nor)\b" + + +def _match_is_affirmative(text: str, match: re.Match[str]) -> bool: + before = re.split(r"[.!?;:\n]+|\b(?:but|however|instead|whereas|while)\b", text[:match.start()], flags=re.I)[-1] + after = text[match.end():] + return not re.search(NEGATION, before, re.I) and not re.match( + r"\s*(?:is|was|are|were)?\s*(?:not|wrong|incorrect)\b", after, re.I) + + +def _affirmative_search(pattern: str, text: str, flags: int = 0) -> bool: + return any(_match_is_affirmative(text, m) for m in re.finditer(pattern, text, flags)) + + +def contains_all(text: Any, expected: Iterable[Any]) -> bool: + normalized = normalize_text(text) + return all(bool(v) and _affirmative_search(r"(? bool: + normalized = normalize_text(text) + return any(bool(v) and _affirmative_search(r"(? bool: + return normalize_text(final) == normalize_text(expected) + + +def contains_phrase_loose(text: Any, phrase: Any) -> bool: + """Phrase match ignoring punctuation (titles quoted with/without '?', quotes, dashes).""" + def squash(s: Any) -> str: + return re.sub(r"[^a-z0-9]+", " ", normalize_text(s)).strip() + return squash(phrase) in squash(text) + + +def digits_only(value: Any) -> str: + return re.sub(r"\D", "", str(value or "")) + + +def contains_money(text: Any, amount: int) -> bool: + """``$165``, ``165 dollars``, ``USD 165``, ``$165.00``; not ``$1650`` or ``165 months``.""" + normalized = normalize_text(text) + amount = int(amount) + patterns = [ + rf"(? bool: + """``36 months``, ``36-month-old``, ``36 mo``, ``36 mos.``; not ``136 months``.""" + normalized = normalize_text(text) + months = int(months) + return _affirmative_search(rf"(? bool: + normalized = normalize_text(text) + words = {1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten"} + if _affirmative_search(rf"(? bool: + """``602-555-0141`` with any separators (dash, dot, space, parentheses) or none.""" + d = digits_only(phone) + raw = unicodedata.normalize("NFKC", str(text or "")) + groups = [d[:3], d[3:6], d[6:]] if len(d) == 10 else [d] + sep = r"[\s.\-()]*" + pattern = r"(? bool: + return _affirmative_search(r"(? bool: + """Does the answer state ``keyword`` (e.g. "good with children" / "house-trained") as + yes (expected_yes=True) or no? Handles "X: Yes", "X - No", "X? No", "not X", + "she is X", "X and Y: No", "neither X nor Y". With conflicting statements the one + closest to ``anchor_name`` wins. + """ + normalized = normalize_text(text) + verdicts: list[tuple[int, bool | None]] = [] + for m in re.finditer(keyword_pattern, normalized): + after = normalized[m.end():] + clause_after = re.split(r"[.;\n!]", after, maxsplit=1)[0] + head = re.match(rf"\s*[:=\-–—?()\"']*\s*(?:is|was|are|:)?\s*[:=\-–—]?\s*({YES_WORDS}|{NO_WORDS})\b", after) + verdict: bool | None = None + if head: + verdict = bool(re.fullmatch(YES_WORDS, head.group(1))) + else: + first = re.search(rf"\b({YES_WORDS}|{NO_WORDS})\b", clause_after) + if first and re.search(r"\b(?:and|or|,|&)\b|,", clause_after[:first.start()]): + verdict = bool(re.fullmatch(YES_WORDS, first.group(1))) + elif first and re.fullmatch(NO_WORDS, first.group(1)): + verdict = False + else: + before = re.split(r"[.;\n!,]|\b(?:but|however|whereas|while|although)\b", normalized[:m.start()])[-1] + if re.search(NEGATION, before): + verdict = False + else: + verdict = True + verdicts.append((m.start(), verdict)) + values = [v for _, v in verdicts if v is not None] + if not values: + return False + if all(v == values[0] for v in values): + return values[0] == expected_yes + if anchor_name: + anchors = [a.start() for a in re.finditer(re.escape(normalize_text(anchor_name)), normalized)] + if anchors: + nearest = min((v for v in verdicts if v[1] is not None), + key=lambda item: min(abs(item[0] - a) for a in anchors)) + return nearest[1] == expected_yes + return expected_yes in values + + +def _first_name_in(segment: str, names: Sequence[str]) -> tuple[int, str] | None: + best: tuple[int, str] | None = None + for n in names: + hit = re.search(r"(? tuple[int, str] | None: + best: tuple[int, str] | None = None + for n in names: + for hit in re.finditer(r"(? best[0]: + best = (hit.start(), n) + return best + + +def name_nearest_keyword(text: Any, keyword_pattern: str, names: Sequence[str]) -> str | None: + """Which pet does the answer attach to the comparison keyword ("lowest", "cheaper", + "youngest", ...)? Resolution order inside the keyword's clause: + 1. an explicit assignment after the keyword ("lowest fee is Batman", "lowest: Batman", + "cheapest ... belongs to Batman"); + 2. " than/among/of/between ..." -> the name just BEFORE the keyword + ("Pepper is cheaper than Daisy", "Batman has the lowest fee among ..."); + 3. the first name after the keyword, else the last name before it. + None when the keyword never occurs.""" + normalized = normalize_text(text) + lowered = [normalize_text(n) for n in names] + alternation = "|".join(re.escape(n) for n in lowered) + for m in re.finditer(keyword_pattern, normalized): + after = re.split(r"[.;\n!]", normalized[m.end():], maxsplit=1)[0] + before = re.split(r"[.;\n!]", normalized[:m.start()])[-1] + explicit = re.search( + rf"(?:\bis|\bwas|\bare|\bwould be|\bbelongs to|\bgoes to|\bis for|:|=|→|->|—|–)\s*" + rf"(?:the\s+|a\s+)?(?:pet\s+|one\s+|dog\s+|cat\s+|kitten\s+)?(?:is\s+)?(? bool: + """True when the answer singles out ``winner`` for the comparison. With the keyword + present the name attached to it must be the winner; with only the inverse keyword + present ("Daisy is more expensive than Pepper") the attached name must be a loser + and the winner must be named; with neither, the winner must be the only candidate + named (a bare report such as "Batman, Chihuahua / Yorkshire Terrier, $165").""" + names = [winner, *others] + w = normalize_text(winner) + normalized = normalize_text(text) + winner_named = bool(re.search(r"(? bool: + return self.ok + + def check(self, name: str, cond: Any, evidence: str = "", llm: bool = False) -> bool: + if llm and self.no_llm: + self.evidence.append(f"[SKIP] {name} (--no_llm)") + return True + if cond: + self.evidence.append(f"[PASS] {name}: {evidence}") + else: + self.ok = False + if not self.reason: + self.reason = name + self.evidence.append(f"[FAIL] {name}: {evidence}") + return bool(cond) + + def emit(self) -> None: + print(json.dumps({"task_id": self.task_id, "pass": self.ok, + "reason": self.reason or "all checks passed", + "evidence": self.evidence}, ensure_ascii=False, indent=2)) + raise SystemExit(0 if self.ok else 1) + + +def fail_closed(task_id: str, reason: str, detail: str) -> None: + print(json.dumps({"task_id": task_id, "pass": False, "infra_error": True, "reason": reason, + "evidence": [f"[FAIL] {reason}: {detail}"]}, ensure_ascii=False, indent=2)) + raise SystemExit(1) + + +def _same_local_origin(url: str, start_url: str) -> bool: + try: + o, s = urlparse(str(url or "")), urlparse(str(start_url or "")) + return (o.scheme == s.scheme == "http" and o.hostname is not None and s.hostname is not None + and not o.username and not o.password and o.port == s.port + and o.hostname.casefold() == s.hostname.casefold() and is_site_url(url)) + except ValueError: + return False + + +def _png_ok(path: Path) -> bool: + try: + from PIL import Image # optional; present in agent_demo's env + with Image.open(path) as im: + im.load() + return im.format == "PNG" and im.width > 0 and im.height > 0 + except ImportError: + pass + except Exception: + return False + try: + with open(path, "rb") as fh: + head = fh.read(24) + if head[:8] != b"\x89PNG\r\n\x1a\n" or head[12:16] != b"IHDR": + return False + width, height = struct.unpack(">II", head[16:24]) + return width > 0 and height > 0 + except OSError: + return False + + +def _screenshots_decode(traj: dict[str, Any]) -> tuple[bool, str]: + root = Path(str(traj.get("_run_dir") or "")) + steps = traj.get("steps") + if not root.is_dir() or not isinstance(steps, list) or not steps: + return False, "run directory or steps are missing" + checked = 0 + for index, step in enumerate(steps): + if not isinstance(step, dict): + return False, f"step {index} is not an object" + for key in ("screenshot_before", "screenshot_after"): + name = step.get(key) + rel = Path(str(name or "")) + if not name or rel.is_absolute() or ".." in rel.parts: + return False, f"step {index} has unsafe {key}" + path = next((c for c in (root / "screenshots" / rel, root / rel) if c.is_file()), None) + if path is None: + return False, f"step {index} is missing {key}={name!r}" + if not _png_ok(path): + return False, f"step {index} {key} is not a decodable PNG" + checked += 1 + return True, f"decoded {checked} PNG screenshots" + + +def check_trajectory_identity(judge: Judge, traj: dict[str, Any], task_id: str) -> None: + judge.check("final_answer_nonempty", bool(final_answer(traj)), f"final_answer={final_answer(traj)!r}") + judge.check("trajectory_task_matches", trajectory_task_matches(traj, task_id), + f"expected_task_id={task_id!r}, observed_task_id={traj.get('task_id')!r}") + judge.check("trajectory_completed", + traj.get("terminated") is True and traj.get("termination_reason") == "agent_done", + f"terminated={traj.get('terminated')!r}, reason={traj.get('termination_reason')!r}") + steps = traj.get("steps") + judge.check("trajectory_has_steps", isinstance(steps, list) and bool(steps), + f"steps={len(steps) if isinstance(steps, list) else 'invalid'}") + recorded = trajectory_urls(traj) + judge.check("all_urls_match_local_origin", + bool(recorded) and all(_same_local_origin(u, traj.get("start_url", "")) for u in recorded), + f"start_url={traj.get('start_url')!r}, recorded_urls={recorded!r}") + ok, ev = _screenshots_decode(traj) + judge.check("screenshots_decode", ok, ev) + + +def check_visited_path(judge: Judge, traj: dict[str, Any], name: str, path: str) -> bool: + return judge.check(name, navigated_to_path(traj, path), f"required_path={path}") + + +def check_visited_pet(judge: Judge, traj: dict[str, Any], slug: str) -> bool: + return judge.check(f"visited_pet_{slug}", pet_visited(traj, slug), f"required_path=/pet/{slug}") + + +def check_visited_pets(judge: Judge, traj: dict[str, Any], slugs: Iterable[str]) -> None: + for slug in slugs: + check_visited_pet(judge, traj, slug) + + +def check_search_visited(judge: Judge, traj: dict[str, Any], name: str, + location_any: Sequence[set[str]] | None = None, **params: Any) -> bool: + ok = search_visited(traj, location_any, **params) + described = f"location~{[sorted(a) for a in location_any] if location_any else 'any'} & {params}" + return judge.check(name, ok, f"required=/search?{described}; observed_search_urls={search_visits(traj)!r}") + + +def check_signed_in_as(judge: Judge, traj: dict[str, Any], email: str) -> None: + judge.check("visited_login_page", navigated_to_path(traj, "/login"), "required_path=/login") + judge.check("entered_expected_account_email", trajectory_last_email(traj) == normalize_text(email), + f"expected_email={email!r}, last_entered_email={trajectory_last_email(traj)!r}") + + +def check_paths_in_order(judge: Judge, traj: dict[str, Any], name: str, + requirements: Sequence[tuple[str, dict[str, Any]]]) -> bool: + """Every (path, params) requirement must be met by recorded site URLs in order.""" + urls = site_urls(traj) + cursor = 0 + for expected_path, params in requirements: + expected = normalized_url_path(expected_path) + for index in range(cursor, len(urls)): + url = urls[index] + query = _query_of(url) + if normalized_url_path(url) == expected and all(_param_ok(query, k, v) for k, v in params.items()): + cursor = index + 1 + break + else: + return judge.check(name, False, f"requirements={requirements!r}, observed={urls!r}") + return judge.check(name, True, f"requirements={requirements!r}") + + +# --------------------------------------------------------------------------- # +# SQLite state +# --------------------------------------------------------------------------- # +def db_query(db_path: str | os.PathLike[str], sql: str, params: Sequence[Any] = ()) -> list[sqlite3.Row]: + con = sqlite3.connect(str(db_path)) + con.row_factory = sqlite3.Row + try: + return con.execute(sql, params).fetchall() + finally: + con.close() + + +def fetch_db(container: str, kind: str) -> str: + if kind not in {"instance", "instance_seed"}: + raise ValueError(f"unsupported DB kind: {kind}") + handle, dest = tempfile.mkstemp(prefix=f"{SITE}_{kind}_", suffix=".db") + os.close(handle) + src = f"{container}:/opt/WebSyn/{SITE}/{kind}/{SITE}.db" + r = subprocess.run(["docker", "cp", src, dest], capture_output=True, text=True) + if r.returncode: + Path(dest).unlink(missing_ok=True) + raise RuntimeError(f"could not copy {src}: {r.stderr.strip() or r.stdout.strip()}") + atexit.register(Path(dest).unlink, missing_ok=True) + return dest + + +def resolve_db(explicit_path: str | None, container: str, kind: str) -> str | None: + if explicit_path: + p = Path(explicit_path) + return str(p) if p.is_file() else None + try: + return fetch_db(container, kind) + except (OSError, RuntimeError): + return None + + +def table_columns(db_path: str, table: str) -> tuple[str, ...]: + if not re.fullmatch(r"[a-z_]+", table): + raise ValueError(f"unsupported table: {table}") + return tuple(row["name"] for row in db_query(db_path, f"PRAGMA table_info({table})")) + + +def table_rows(db_path: str, table: str) -> list[tuple[Any, ...]]: + if not re.fullmatch(r"[a-z_]+", table): + raise ValueError(f"unsupported table: {table}") + return [tuple(row) for row in db_query(db_path, f"SELECT * FROM {table} ORDER BY 1")] + + +def table_dicts(db_path: str, table: str) -> list[dict[str, Any]]: + if not re.fullmatch(r"[a-z_]+", table): + raise ValueError(f"unsupported table: {table}") + return [dict(row) for row in db_query(db_path, f"SELECT * FROM {table} ORDER BY 1")] + + +def _tables(db_path: str) -> set[str]: + return {row["name"] for row in db_query( + db_path, "SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'")} + + +def _schema_objects(db_path: str) -> list[tuple[Any, ...]]: + return [tuple(row) for row in db_query( + db_path, "SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE sql IS NOT NULL " + "AND name NOT LIKE 'sqlite_%' ORDER BY type, name")] + + +def catalog_drift(db_path: str) -> list[str]: + """Differences between the snapshot's pet/shelter/user rows and ground_truth.py.""" + problems: list[str] = [] + pets = {row["slug"]: row for row in table_dicts(db_path, "pet")} + if set(pets) != {p["slug"] for p in GT.PETS}: + problems.append(f"pet slugs differ: {sorted(set(pets) ^ {p['slug'] for p in GT.PETS})}") + for frozen in GT.PETS: + row = pets.get(frozen["slug"]) + if not row: + continue + for field in GT.PET_FIELDS: + observed = row.get(field) + expected = frozen[field] + if field in ("house_trained", "good_dogs", "good_cats", "good_children"): + observed = int(bool(observed)) + if observed != expected: + problems.append(f"pet {frozen['slug']}.{field}: expected {expected!r}, observed {observed!r}") + shelters = {row["id"]: row for row in table_dicts(db_path, "shelter")} + for frozen in GT.SHELTERS: + row = shelters.get(frozen["id"]) + if not row: + problems.append(f"shelter {frozen['id']} missing") + continue + for field in GT.SHELTER_FIELDS: + if row.get(field) != frozen[field]: + problems.append(f"shelter {frozen['id']}.{field}: expected {frozen[field]!r}, observed {row.get(field)!r}") + users = {row["id"]: row for row in table_dicts(db_path, "user")} + for frozen in GT.USERS: + row = users.get(frozen["id"]) + if not row or normalize_text(row.get("email")) != frozen["email"] or row.get("name") != frozen["name"]: + problems.append(f"user {frozen['id']}: expected {frozen!r}, observed {dict(row) if row else None!r}") + favorites = [(r["id"], r["user_id"], r["pet_id"]) for r in table_dicts(db_path, "favorite")] + if favorites != GT.SEED_FAVORITES: + problems.append(f"seed favorites differ: expected {GT.SEED_FAVORITES!r}, observed {favorites!r}") + return problems + + +def _validate_snapshot_contract(initial_db: str, after_db: str) -> None: + expected_tables = set(EXPECTED_COLUMNS) + if _tables(initial_db) != expected_tables or _tables(after_db) != expected_tables: + raise ValueError(f"unexpected tables: initial={sorted(_tables(initial_db))}, after={sorted(_tables(after_db))}") + for table, columns in EXPECTED_COLUMNS.items(): + for db in (initial_db, after_db): + if table_columns(db, table) != columns: + raise ValueError(f"unexpected columns for {table}: {table_columns(db, table)}") + if _schema_objects(initial_db) != _schema_objects(after_db): + raise ValueError("initial and after database schemas differ") + observed = {t: len(table_rows(initial_db, t)) for t in INITIAL_COUNTS} + if observed != INITIAL_COUNTS: + raise ValueError(f"initial database counts differ: expected={INITIAL_COUNTS}, observed={observed}") + drift = catalog_drift(initial_db) + if drift: + raise ValueError("initial snapshot does not match the frozen catalog: " + "; ".join(drift[:5])) + changed = [t for t in IMMUTABLE_TABLES if table_rows(initial_db, t) != table_rows(after_db, t)] + if changed: + raise ValueError(f"immutable catalog tables changed: {changed}") + + +def resolve_snapshots(args, task_id: str) -> tuple[str, str]: + """Return validated (initial_db, after_db) or fail closed.""" + initial_db = resolve_db(args.initial_db or None, args.container, "instance_seed") + after_db = resolve_db(args.after_db or None, args.container, "instance") + if not initial_db or not after_db: + fail_closed(task_id, "database_unavailable", + "both initial and after adopt_a_pet database snapshots are required " + "(--initial_db/--after_db, /initial.db + after.db, or docker cp from $WH_CONTAINER)") + try: + _validate_snapshot_contract(str(initial_db), str(after_db)) + except (OSError, sqlite3.Error, ValueError) as exc: + fail_closed(task_id, "snapshot_contract_invalid", str(exc)) + return str(initial_db), str(after_db) + + +def user_row(db_path: str, email: str) -> dict[str, Any] | None: + rows = db_query(db_path, "SELECT id, email, name, password_hash FROM user WHERE lower(email)=lower(?) ORDER BY id LIMIT 1", (email,)) + return dict(rows[0]) if rows else None + + +def user_id_for_email(db_path: str, email: str) -> int | None: + row = user_row(db_path, email) + return int(row["id"]) if row else None + + +def pet_slug_by_id(db_path: str, pet_id: int) -> str: + rows = db_query(db_path, "SELECT slug FROM pet WHERE id=?", (int(pet_id),)) + return str(rows[0]["slug"]) if rows else f"" + + +def favorite_pairs(db_path: str) -> set[tuple[int, int]]: + return {(int(r["user_id"]), int(r["pet_id"])) for r in db_query(db_path, "SELECT user_id, pet_id FROM favorite")} + + +def favorite_slugs(db_path: str, email: str) -> set[str]: + uid = user_id_for_email(db_path, email) + if uid is None: + return set() + return {pet_slug_by_id(db_path, pid) for (u, pid) in favorite_pairs(db_path) if u == uid} + + +def describe_favorites(db_path: str) -> list[tuple[str, str]]: + out = [] + for uid, pid in sorted(favorite_pairs(db_path)): + rows = db_query(db_path, "SELECT email FROM user WHERE id=?", (uid,)) + out.append((normalize_text(rows[0]["email"]) if rows else f"", pet_slug_by_id(db_path, pid))) + return out + + +def table_delta(initial_db: str, after_db: str, table: str) -> dict[str, list[Any]]: + before = {int(r["id"]): r for r in table_dicts(initial_db, table)} + after = {int(r["id"]): r for r in table_dicts(after_db, table)} + common = before.keys() & after.keys() + return {"added": [after[k] for k in sorted(after.keys() - before.keys())], + "removed": [before[k] for k in sorted(before.keys() - after.keys())], + "changed": [(before[k], after[k]) for k in sorted(common) if before[k] != after[k]]} + + +def tables_unchanged(initial_db: str, after_db: str, tables: Iterable[str]) -> dict[str, bool]: + return {t: table_rows(initial_db, t) == table_rows(after_db, t) for t in tables} + + +def check_tables_unchanged(judge: Judge, initial_db: str, after_db: str, tables: Iterable[str], prefix: str = "") -> None: + for table, same in tables_unchanged(initial_db, after_db, tables).items(): + judge.check(f"{prefix}{table}_unchanged", same, + f"table={table}, initial_rows={len(table_rows(initial_db, table))}, " + f"after_rows={len(table_rows(after_db, table))}, identical={same}") + + +def check_read_only(judge: Judge, initial_db: str, after_db: str) -> None: + """Read-only tasks: user, favorite, application and pet_alert must be row-identical.""" + check_tables_unchanged(judge, initial_db, after_db, MUTABLE_TABLES, prefix="read_only_") + + +def check_exact_favorite_delta(judge: Judge, initial_db: str, after_db: str, + added: Iterable[tuple[str, str]] = (), removed: Iterable[tuple[str, str]] = ()) -> None: + """The favorite table must change by EXACTLY the given (email, slug) pairs.""" + def resolve(pairs: Iterable[tuple[str, str]], db: str) -> set[tuple[int, int]]: + out = set() + for email, slug in pairs: + uid = user_id_for_email(db, email) + out.add((uid if uid is not None else -1, GT.pet(slug)["id"])) + return out + before, after = favorite_pairs(initial_db), favorite_pairs(after_db) + exp_added, exp_removed = resolve(added, after_db), resolve(removed, initial_db) + judge.check("favorites_added_exactly", after - before == exp_added, + f"expected_added={sorted(added)!r}, observed_added={[p for p in describe_favorites(after_db) if p not in describe_favorites(initial_db)]!r}") + judge.check("favorites_removed_exactly", before - after == exp_removed, + f"expected_removed={sorted(removed)!r}, observed_removed={[p for p in describe_favorites(initial_db) if p not in describe_favorites(after_db)]!r}") + delta = table_delta(initial_db, after_db, "favorite") + judge.check("favorites_no_rewritten_rows", not delta["changed"], f"changed={delta['changed']!r}") + + +def password_matches(stored_hash: Any, password: str) -> bool: + """Verify a werkzeug ``scrypt:N:r:p$salt$hex`` (or ``pbkdf2:sha256:iters$salt$hex``) hash with stdlib.""" + try: + method, salt, digest = str(stored_hash or "").split("$", 2) + except ValueError: + return False + try: + if method.startswith("scrypt:"): + _, n, r, p = method.split(":") + computed = hashlib.scrypt(password.encode("utf-8"), salt=salt.encode("utf-8"), n=int(n), r=int(r), p=int(p), + maxmem=132 * 1024 * 1024, dklen=64).hex() + elif method.startswith("pbkdf2:"): + parts = method.split(":") + algo = parts[1] + iters = int(parts[2]) if len(parts) > 2 else 600000 + computed = hashlib.pbkdf2_hmac(algo, password.encode("utf-8"), salt.encode("utf-8"), iters).hex() + else: + return False + except (ValueError, TypeError): + return False + return hmac.compare_digest(computed, digest) + + +# --------------------------------------------------------------------------- # +# LLM utilities (parity with merriam_webster; never on the verdict path) +# --------------------------------------------------------------------------- # +def _llm_config(): + return os.environ.get("OPENAI_API_KEY", ""), os.environ.get("OPENAI_BASE_URL", ""), os.environ.get("JUDGE_MODEL", "") + + +def _chat(messages, max_tokens=1024): + if _NO_LLM: + return None + key, base, model = _llm_config() + if not (key and base and model): + return None + payload = {"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": 1.0} + req = urllib.request.Request(base, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}) + try: + data = json.loads(urllib.request.urlopen(req, timeout=180).read()) + return data["choices"][0]["message"]["content"] + except Exception: + return None + + +def _verdict(out): + if not out: + return False, "" + s = out.strip() + return s.upper().startswith("PASS"), s + + +def llm_text_match(agent_answer, ground_truth, question): + if _NO_LLM: + return False, "[skipped: --no_llm]" + out = _chat([{"role": "user", "content": + f"You are a STRICT binary grader.\nQuestion: {question}\n" + f"Ground-truth answer (ANCHOR — judge against THIS, never use your own knowledge): {ground_truth}\n" + f"Agent's answer: {agent_answer}\nDecide PASS or FAIL ignoring case/punctuation/word order/surrounding prose. " + f"PASS only if the agent's answer is consistent with the ground truth AND actually answers the question. " + f"Line 1: PASS or FAIL. Line 2: one-sentence reason."}]) + return _verdict(out) + + +def llm_screenshot_shows(shot_path, must_show, question=""): + if _NO_LLM: + return False, "[skipped: --no_llm]" + b64 = base64.b64encode(Path(shot_path).read_bytes()).decode() + out = _chat([{"role": "user", "content": [ + {"type": "text", "text": f"You are a STRICT binary grader. Only what is VISIBLY rendered in this screenshot counts.\n" + f"Question the page should answer: {question}\nExpected content to verify PRESENCE of: {must_show}\n" + f"PASS only if the expected content is visibly shown. Do NOT use prior knowledge.\n" + f"Line 1: PASS or FAIL. Line 2: quote the visible evidence."}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}]) + return _verdict(out) + + +# --------------------------------------------------------------------------- # +# Verifier entry point shared by verify_N.py +# --------------------------------------------------------------------------- # +def run_verifier(task_id: str, run_checks) -> None: + """Common main(): load run, resolve snapshots, run checks, emit; any exception fails closed.""" + args = parse_args() + try: + traj = load_run(args.run_dir) + except (OSError, ValueError) as exc: + fail_closed(task_id, "trajectory_unavailable", str(exc)) + initial_db, after_db = resolve_snapshots(args, task_id) + judge = Judge(task_id, no_llm=True if args.no_llm else True) # verdicts never depend on an LLM + try: + run_checks(judge, traj, initial_db, after_db) + except Exception as exc: # noqa: BLE001 — any verifier error fails closed + fail_closed(task_id, "verifier_error", f"{type(exc).__name__}: {exc}") + judge.emit() diff --git a/websyn_start.sh b/websyn_start.sh index f575dbc2c..0f434dc47 100644 --- a/websyn_start.sh +++ b/websyn_start.sh @@ -7,7 +7,8 @@ SITES=(allrecipes amazon apple arxiv bbc_news booking github google_flights google_map google_search huggingface wolfram_alpha cambridge_dictionary coursera espn merriam_webster ikea phys_org target ted osu rotten_tomatoes compass walmart_careers - fedex webmd_doctor healthline kaggle) + fedex webmd_doctor healthline kaggle + adopt_a_pet) BASE_PORT=40000 SITE_COUNT=${#SITES[@]} PID_DIR=/tmp/websyn_pids