FRED from GitHub runners: no custom User-Agent; canary one leg per builder

.github/workflows/refresh-snapshots.yml

@@ -70,11 +70,18 @@ jobs:
           DATASET: ${{ inputs.dataset }}
           FORCE: ${{ inputs.force }}
         run: |
-          flags=""
-          [ -n "$DATASET" ] && flags="$flags --dataset $DATASET"
-          [ "$FORCE" = "true" ] && flags="$flags --all"
-          all=$(python scripts/snapshots.py list | jq -c .)
-          due=$(python scripts/snapshots.py due $flags | jq -c .)
+          # Arrays, not string concatenation: the dataset input is a single
+          # argument however it is spelled.
+          due_flags=()
+          list_flags=()
+          if [ -n "$DATASET" ]; then
+            due_flags+=(--dataset "$DATASET")
+            list_flags+=(--dataset "$DATASET")
+          fi
+          [ "$FORCE" = "true" ] && due_flags+=(--all)
+          # canary: one leg per BUILDER (a set-writing builder fetches once)
+          all=$(python scripts/snapshots.py list --by-builder "${list_flags[@]}" | jq -c .)
+          due=$(python scripts/snapshots.py due "${due_flags[@]}" | jq -c .)
           echo "all=$all" >> "$GITHUB_OUTPUT"
           echo "due=$due" >> "$GITHUB_OUTPUT"
           echo "canary: $(echo "$all" | jq -r '.[].dataset' | tr '\n' ' ')"

builders/_fred.py

@@ -21,7 +21,14 @@
   DatetimeIndex named `DATE`, so committed files keep the header the
   lectures expect
 - FRED's `.` for a missing observation -> NaN
-- a User-Agent header, which fred.stlouisfed.org has been seen to require
+- NO CUSTOM User-Agent by default, deliberately: the request goes out with
+  urllib's own `Python-urllib/x.y`. Measured 2026-09-01 from a GitHub-hosted
+  runner (data-lectures#115): FRED's edge answers `Python-urllib/3.12` and
+  `curl/8.5.0` in ~50 ms, and STALLS `qeld-builder` and even `Mozilla/5.0`
+  until the read times out. The same requests all succeed from a
+  workstation, which is how the custom agent survived local testing. Pass
+  `user_agent=` only if you have measured that it works from where the
+  builder will actually run
 - one series per request, aligned with an outer join in frame(), so a
   series that starts later is simply empty before its first observation
 
@@ -39,14 +46,15 @@
 
 
 class Fred:
-    def __init__(self, user_agent='qeld-builder', timeout=60):
-        self.user_agent = user_agent
+    def __init__(self, user_agent=None, timeout=60):
+        self.user_agent = user_agent      # None -> no custom header; urllib
+                                          # sends Python-urllib/x.y (see above)
         self.timeout = timeout
 
     def _get(self, params):
         query = urllib.parse.urlencode({k: v for k, v in params.items() if v is not None})
-        request = urllib.request.Request(f'{FREDGRAPH}?{query}',
-                                         headers={'User-Agent': self.user_agent})
+        headers = {'User-Agent': self.user_agent} if self.user_agent else {}
+        request = urllib.request.Request(f'{FREDGRAPH}?{query}', headers=headers)
         with urllib.request.urlopen(request, timeout=self.timeout) as response:
             return response.read()
 

builders/fred_data.py

@@ -61,7 +61,9 @@ def _fetch_series(code):
     url = f'{FRED_CSV}?id={code}&cosd={START}&coed={END}'
     if code in DAILY_AVERAGED:
         url += '&fq=Monthly&fam=avg'
-    request = urllib.request.Request(url, headers={'User-Agent': 'qeld-builder'})
+    # No custom User-Agent: FRED's edge stalls unfamiliar agents from GitHub
+    # runners and answers urllib's default at once (data-lectures#115).
+    request = urllib.request.Request(url)
     with urllib.request.urlopen(request) as response:
         payload = response.read()
     frame = pd.read_csv(io.BytesIO(payload), index_col=0, parse_dates=True,

scripts/snapshots.py

@@ -197,7 +197,20 @@ def sha256(path: pathlib.Path) -> str:
 # ---------------------------------------------------------------------------
 
 def cmd_list(args) -> int:
-    print(json.dumps(snapshots(load_manifests()), indent=1))
+    rows = snapshots(load_manifests())
+    if args.dataset:
+        rows = [r for r in rows if r["dataset"] == args.dataset]
+    if args.by_builder:
+        # One leg per builder for the canary: a builder that writes a set
+        # fetches once for all of them, so running it per dataset only
+        # repeats the same fetch. One pass, first-seen order; the leg is
+        # named for the builder's first dataset.
+        by_builder: dict[str, dict] = {}
+        for r in rows:
+            leg = by_builder.setdefault(r["builder"], {**r, "datasets": []})
+            leg["datasets"].append(r["dataset"])
+        rows = list(by_builder.values())
+    print(json.dumps(rows, indent=1))
     return 0
 
 
@@ -331,7 +344,8 @@ def cmd_pr_body(args) -> int:
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
     sub = ap.add_subparsers(dest="cmd", required=True)
-    sub.add_parser("list").set_defaults(fn=cmd_list)
+    p = sub.add_parser("list"); p.add_argument("--dataset"); p.add_argument("--by-builder", action="store_true")
+    p.set_defaults(fn=cmd_list)
     p = sub.add_parser("due"); p.add_argument("--all", action="store_true"); p.add_argument("--dataset")
     p.set_defaults(fn=cmd_due)
     p = sub.add_parser("stamp"); p.add_argument("dataset"); p.add_argument("--summary", required=True)