[test] fact-check eval: qwen/qwen3.8-27b:free - DO NOT MERGE

.github/workflows/agentics-maintenance.yml

@@ -37,7 +37,7 @@ name: Agentic Maintenance
 
 on:
   schedule:
-    - cron: "37 0 * * *"  # Daily (based on minimum expires: 7 days)
+    - cron: "37 */6 * * *"  # Every 6 hours (based on minimum expires: 2 days)
   workflow_dispatch:
     inputs:
       operation:

.github/workflows/til-fact-check.md

@@ -0,0 +1,155 @@
+---
+name: TIL Fact Check
+description: |
+  Required merge gate for TIL posts (gh-aw governance pattern: a ruleset
+  requires the check this workflow produces). Verifies every factual claim in
+  a new content/blog post against its primary source and posts the
+  `fact-check` check run. Non-TIL PRs get an immediate success.
+
+on:
+  pull_request:
+    # labeled: a PR that gains `til` after a NOT_TIL pass must be re-verified.
+    types: [opened, synchronize, reopened, labeled]
+  bots: ["dependabot[bot]"]
+
+permissions: read-all
+
+inlined-imports: true
+
+network:
+  allowed:
+    - defaults
+    - github
+    - python
+    - openrouter.ai
+    - openai.com
+    - anthropic.com
+    - blog.google
+    - ai.google.dev
+    - mistral.ai
+    - deepseek.com
+    - huggingface.co
+    - z.ai
+    - qwencloud.com
+    - arxiv.org
+
+models:
+  default-ai-credits-pricing:
+    input: 0.000001
+    output: 0.000001
+
+engine:
+  id: copilot
+  max-continuations: 8
+  args: ["--allow-all-urls"]
+  env:
+    COPILOT_PROVIDER_BASE_URL: https://openrouter.ai/api/v1
+    COPILOT_MODEL: "qwen/qwen3.8-27b:free"
+    COPILOT_PROVIDER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+
+sandbox:
+  agent:
+    id: awf
+    model-fallback: false
+    token-steering: false
+
+safe-outputs:
+  threat-detection:
+    continue-on-error: false
+    engine:
+      id: copilot
+      env:
+        COPILOT_PROVIDER_BASE_URL: https://openrouter.ai/api/v1
+        COPILOT_MODEL: "nvidia/nemotron-3-super-120b-a12b:free"
+        COPILOT_PROVIDER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+  # No `target`: the check attaches to the triggering event's head SHA, i.e. the
+  # commit the agent actually read, never a newer unverified head.
+  create-check-run:
+    name: fact-check
+
+steps:
+  - name: Prepare fact-check inputs
+    run: |
+      mkdir -p /tmp/gh-aw/agent/factcheck
+      gh pr view "$PR" --json labels,files > /tmp/gh-aw/agent/factcheck/pr.json
+      jq -r '.files[].path | select(startswith("content/blog/"))' \
+        /tmp/gh-aw/agent/factcheck/pr.json > /tmp/gh-aw/agent/factcheck/posts.txt
+      if jq -e '[.labels[].name] | index("til")' /tmp/gh-aw/agent/factcheck/pr.json > /dev/null \
+        && [ -s /tmp/gh-aw/agent/factcheck/posts.txt ]; then
+        echo "TIL" > /tmp/gh-aw/agent/factcheck/gate.txt
+      else
+        echo "NOT_TIL" > /tmp/gh-aw/agent/factcheck/gate.txt
+      fi
+      # proxy.js negotiates markdown only for /blog/[\w-]+ slugs; a dotted
+      # slug (#101: qwen-image-2.1-...) would silently lose its markdown twin.
+      grep -vE '^content/blog/[a-z0-9-]+\.mdx$' /tmp/gh-aw/agent/factcheck/posts.txt \
+        > /tmp/gh-aw/agent/factcheck/bad-slugs.txt || true
+      cat /tmp/gh-aw/agent/factcheck/gate.txt /tmp/gh-aw/agent/factcheck/posts.txt
+      # DeterministicOps: fetch every cited source once, so the agent reads files
+      # instead of re-fetching (run 36388743650 burned 250/250 invocations on
+      # repeated `curl README | grep`). Hugging Face model links also get the raw
+      # README and config.json, the primary spec sources.
+      cd /tmp/gh-aw/agent/factcheck && mkdir -p sources && : > sources/index.txt
+      while read -r f; do [ -f "$GITHUB_WORKSPACE/$f" ] && grep -oE '\]\(https?://[^) ]+' "$GITHUB_WORKSPACE/$f" | cut -c3-; done < posts.txt | sort -u > urls.txt
+      sed -nE 's#^https://huggingface\.co/([^/]+/[^/?#]+)/?$#\1#p' urls.txt | while read -r m; do
+        echo "https://huggingface.co/$m/raw/main/README.md"; echo "https://huggingface.co/$m/raw/main/config.json"
+      done >> urls.txt
+      n=0
+      sort -u urls.txt | head -60 | while read -r u; do
+        n=$((n+1))
+        if curl -sfL --max-time 20 "$u" -o "sources/$n.txt"; then echo "sources/$n.txt $u" >> sources/index.txt
+        else echo "UNREACHABLE $u" >> sources/index.txt; fi
+      done
+      cat sources/index.txt
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      PR: ${{ github.event.pull_request.number }}
+
+tools:
+  web-fetch:
+  bash: ["cat", "ls", "head", "tail", "wc", "grep", "jq"]
+  github:
+    toolsets: [default]
+timeout-minutes: 45
+max-turns: 250
+
+---
+
+# TIL Fact Check
+
+RUN CONTRACT — read first: every run MUST end with exactly one `create_check_run` call. Ending without it is a failed run and blocks the merge.
+
+## Gate
+
+Read `/tmp/gh-aw/agent/factcheck/gate.txt` first. If it says `NOT_TIL`, immediately call `create_check_run` with `conclusion: success`, `title: "Not a TIL post"`, `summary: "No til-labeled blog post in this PR; nothing to fact-check."` and stop.
+
+## Job (gate says `TIL`)
+
+`/tmp/gh-aw/agent/factcheck/posts.txt` lists the post files in this PR. For each, `cat` it and verify it.
+
+Every source the post links is already downloaded: `/tmp/gh-aw/agent/factcheck/sources/index.txt` maps each local file to its URL (Hugging Face model links also have their raw `README.md` and `config.json`), and marks `UNREACHABLE` links. Read these files with `cat` / `grep` / `jq`; never re-fetch a URL that is in the index. Use `web-fetch` only for a primary source the post does not link (for example a pricing page or an arXiv abstract), once per URL. You are the only check between this post and publication: it auto-merges the moment you pass it. Be strict.
+
+1. List every checkable claim: numbers (parameters, context length, benchmark scores, prices, dates, sizes), specs (license, architecture, modality, organization), citations (paper titles and IDs), and every "X is not published / not specified" statement.
+2. Verify each against a primary source you fetch in this run:
+   - Model specs: `https://huggingface.co/<org>/<model>/raw/main/config.json` (for example `max_position_embeddings` is the context window) and the model card. A model card's pipeline tag is not a spec.
+   - Benchmark tables: fetch the card and compare cell by cell; confirm each number comes from the column of the model the post names.
+   - Prices: the vendor's pricing page or `https://openrouter.ai/api/v1/models`. A price derived by arithmetic is wrong.
+   - Papers: `https://arxiv.org/abs/<id>`; the title must name this model and version.
+   - "Not specified" claims: wrong if the source does publish it.
+   - Links: every `UNREACHABLE` entry in the index is a failed claim.
+   - A post that links no primary source at all fails: its claims are unsupported by definition.
+   - Every path listed in `/tmp/gh-aw/agent/factcheck/bad-slugs.txt` is a failed claim ("slug must be lowercase letters, digits, and hyphens").
+3. A claim is **wrong** if the source contradicts it and **unsupported** if you cannot find it in a fetched source. Descriptions of what a benchmark measures, or advice, need a source too.
+
+## Verdict
+
+- `conclusion: success` only if every checkable claim is verified correct.
+- `conclusion: failure` if any claim is wrong or unsupported, or if you could not fetch a source a claim depends on. When in doubt, fail: a false negative costs one day's post, and a false positive publishes an error.
+- `title`: `N claims verified` or `N claims failed of M`.
+- `summary`: one line per failed claim: file:line, what the post says, what the source says, the source URL. On success, list the claims checked with their source URLs.
+
+Never edit files. Never approve based on the PR description's own evidence list: re-fetch every source yourself.
+
+## Final reminder
+
+Whatever happened above, your last action MUST be exactly one `create_check_run` call. If you run short of budget or a source is unreachable, call it with `conclusion: failure` and say why.

.github/workflows/til-writer.md

@@ -3,8 +3,8 @@ description: |
   TIL factory writer. Triggered by the '/til' command on an issue containing a
   topic, URL, repo, or rough note. Researches the topic with web fetch and
   repository inspection, verifies claims against primary sources, then opens a
-  pull request adding a draft MDX post to content/blog/. A human merges;
-  normal CI and Vercel preview handle everything deterministic.
+  pull request adding an MDX post to content/blog/. It auto-merges once CI
+  and the fact-check workflow pass.
 
 on:
   slash_command:
@@ -16,7 +16,19 @@ permissions: read-all
 network:
   allowed:
     - defaults
+    - github
+    - python
     - openrouter.ai
+    - openai.com
+    - anthropic.com
+    - blog.google
+    - ai.google.dev
+    - mistral.ai
+    - deepseek.com
+    - huggingface.co
+    - z.ai
+    - qwencloud.com
+    - arxiv.org
 
 models:
   default-ai-credits-pricing:
@@ -38,6 +50,7 @@ sandbox:
 
 safe-outputs:
   threat-detection:
+    continue-on-error: false
     engine:
       id: copilot
       env:
@@ -47,8 +60,11 @@ safe-outputs:
   create-pull-request:
     title-prefix: "[til] "
     labels: [til, automated]
+    draft: false
+    auto-merge: squash
+    expires: 2d
     allowed-files:
-      - "content/blog/**"
+      - "content/blog/*.mdx"
 
 steps:
   - name: Install content linters (same toolchain as CI)
@@ -80,16 +96,16 @@ The capture note is: "${{ steps.sanitized.outputs.text }}"
 
 3. **Verify.** Every factual claim in the draft must trace to a source you actually fetched, or to a command you actually ran in this repository. Never write "I tested X" unless you ran it here. Prefer exact version numbers, dates, and quoted behavior over generalities.
 
-4. **Write the draft.** Create one new file in `content/blog/` named `<kebab-case-slug>.mdx`, matching the existing posts' format exactly — YAML frontmatter with `title`, `description`, `date` (today, as a QUOTED string like `date: "2026-08-26"` — an unquoted date fails validation), `author: "Sameer Khan"`, `tags` (inline list, 3–6 items), `category` (one of the existing categories: AI, Developer Tools, Web Development, Projects), `published: true` — merging the PR IS the publish approval. CI validates the PR with `npm run validate:content` (frontmatter schema + markdownlint + MDX compile); step 6 has you run the same check yourself before opening the PR. Study 2–3 existing posts first for MDX conventions, then follow this measured TIL register (derived from analyzing all 579 posts in simonw/til):
+4. **Write the draft.** Create one new file in `content/blog/` named `<kebab-case-slug>.mdx` — lowercase letters, digits, and hyphens only, never a dot (version `2.1` becomes `2-1`), because a dotted slug breaks the site's markdown negotiation and fails the fact-check gate — matching the existing posts' format exactly — YAML frontmatter with `title`, `description`, `date` (today, as a QUOTED string like `date: "2026-08-26"` — an unquoted date fails validation), `author: "Sameer Khan"`, `tags` (inline list, 3–6 items), `category` (one of the existing categories: AI, Developer Tools, Web Development, Projects), `published: true` — the PR auto-merges once CI and the `fact-check` workflow pass, with no human review. CI validates the PR with `npm run validate:content` (frontmatter schema + markdownlint + MDX compile); step 6 has you run the same check yourself before opening the PR. Study 2–3 existing posts first for MDX conventions, then follow this measured TIL register (derived from analyzing all 579 posts in simonw/til):
    - **Title**: gerund-led sentence case, ~7 words, naming the task — "Running X inside Y", "Fixing X when Y". Never "How to…", never clickbait, no first-person in titles.
-   - **Opening**: first sentence states the concrete first-person trigger — what you were doing and what forced the learning ("I needed…", "I noticed…", "For X I found…") — with a link to the real project or issue. The first paragraph doubles as the summary; no throat-clearing.
+   - **Opening**: first sentence states the concrete trigger from the capture note — what forced the learning; write it in first person only when the capture note itself says what Sameer was doing, never invent one — with a link to the real project or issue. The first paragraph doubles as the summary; no throat-clearing.
    - **Length**: target the 150–900 word range, median ~320. Go longer (up to ~1,500) only when the material genuinely demands a deep-dive.
    - **Body**: prose interleaved with at least one fenced code block containing real commands and real output (88% of Simon's posts have code). 3–5 external links to primary sources. H2 subheads only if the post runs long. End with the working final version or a pointer to it. A `<` followed by a letter or digit in prose (`<50ms`, `<model>`) is JSX to MDX and fails the compile (so is a bare `{`) — write `under 50ms`, escape it as `\<50ms` / `\{`, or put it in backticks.
    - **Never**: "In today's fast-paced world" intros, unsupported superlatives, invented anecdotes, "what is X" boilerplate, emoji anywhere (headings, tables, lists — use "Yes"/"No" in comparison tables, plain words everywhere else).
 
 5. **Evidence bundle.** Write a JSON evidence file to cache-memory named after the slug: source URLs with access dates, versions of any tools referenced, commands run with outputs, and the list of factual claims mapped to sources. Repeat the evidence summary in the PR description — that is its permanent record.
 
-6. **Lint gate, then open the PR.** **Lint gate (MANDATORY, before the PR call).** Run `npm run fix-content`. It auto-fixes formatting, then prints every remaining error as `file:line:col rule message` (a table row with the wrong number of cells, an unquoted frontmatter value, a missing field, and so on). Fix each reported line in the file and run it again until it prints no errors. CI runs the identical check and rejects the PR otherwise, so never call `create_pull_request` while it still reports an error. This step is MANDATORY and is the entire point of the run: you MUST finish by calling the `create_pull_request` safe-output tool with the new MDX file. Call it with exactly these arguments: `title`, `body`, and `branch` (use `til/<slug>`). Do NOT pass `temporary_id` — if you include it, it must match `^aw_[A-Za-z0-9_]{3,12}$` (e.g. `aw_til1`) or the whole PR is rejected by validation. A run that researches but never calls `create_pull_request` is a failed run — if you truly cannot produce the post, call `report_incomplete` with the reason instead of ending silently. The PR description must contain: the angle chosen and why, the evidence summary (sources with dates), and a checklist of claims verified. Set `published: true` in frontmatter: the human merge is the publish decision.
+6. **Lint gate, then open the PR.** **Lint gate (MANDATORY, before the PR call).** Run `npm run fix-content`. It auto-fixes formatting, then prints every remaining error as `file:line:col rule message` (a table row with the wrong number of cells, an unquoted frontmatter value, a missing field, and so on). Fix each reported line in the file and run it again until it prints no errors. CI runs the identical check and rejects the PR otherwise, so never call `create_pull_request` while it still reports an error. This step is MANDATORY and is the entire point of the run: you MUST finish by calling the `create_pull_request` safe-output tool with the new MDX file. Call it with exactly these arguments: `title`, `body`, and `branch` (use `til/<slug>`). Do NOT pass `temporary_id` — if you include it, it must match `^aw_[A-Za-z0-9_]{3,12}$` (e.g. `aw_til1`) or the whole PR is rejected by validation. A run that researches but never calls `create_pull_request` is a failed run — if you truly cannot produce the post, call `report_incomplete` with the reason instead of ending silently. The PR description must contain: the angle chosen and why, the evidence summary (sources with dates), and a checklist of claims verified. Set `published: true` in frontmatter: CI plus fact-check passing is the publish decision.
 
 ## Hard rules