From 43f33a627151640f5f30ed479e842782b347141e Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Mon, 22 Jun 2026 18:23:01 +0300 Subject: [PATCH 1/4] feat(release): end-of-run fleet inventory report (completeness + freshness) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release reported only per-leaf size/build-time, so stale, missing, or failed images stayed invisible — a published-but-stale combo could hide behind green stages (v0.1.0's combos were ~all 06-18/06-19 and nothing flagged it; it was found by hand in GHCR). Add a step to the report job that cross-checks the registry against the run: - every container package bucketed (base/leaf/per-task/combo/standalone) and split built-this-run vs older (push-time vs run start) -> step summary + a durable fleet-inventory.tsv artifact; - failed jobs bucketed by stage; - ::warning annotations for consumable layers (base/combo/standalone) that did not refresh this run. Adds actions:read so the report job can read this run's job results. Signed-off-by: Elron Bandel --- .github/.secrets.baseline | 4 +- .github/workflows/release-images.yml | 67 ++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/.github/.secrets.baseline b/.github/.secrets.baseline index c03b7cfb..bcba219a 100644 --- a/.github/.secrets.baseline +++ b/.github/.secrets.baseline @@ -145,7 +145,7 @@ "filename": ".github/workflows/release-images.yml", "hashed_secret": "6e0da5f85a202cf018708adc9db4b5c04ac093e6", "is_verified": false, - "line_number": 151 + "line_number": 152 } ], ".github/workflows/release.yml": [ @@ -194,5 +194,5 @@ } ] }, - "generated_at": "2026-06-22T08:37:48Z" + "generated_at": "2026-06-22T15:22:03Z" } diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index 7068e7b6..7bbf5aeb 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -81,6 +81,7 @@ concurrency: permissions: contents: read packages: write # push to GHCR; Actions-pushed packages auto-link to this repo + actions: read # report job reads this run's jobs (build time + failure summary) env: REGISTRY: ghcr.io/exgentic @@ -751,6 +752,71 @@ jobs: echo "" echo "_size-report.json / .csv are uploaded as the \`fleet-size-report\` artifact (durable — feeds audits + README). Combos + per-task sizes are omitted here (hundreds) — query \`ghcr.io/exgentic/evals/*\` / \`benchmarks/*-\`._" } >> "$GITHUB_STEP_SUMMARY" + - name: Fleet inventory — completeness + freshness + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + # The registry is the source of truth for what the fleet ACTUALLY + # shipped. Cross-check every published image against this run so stale + # or failed images can't hide behind green stages. + ORG="${REGISTRY##*/}" + run_started=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq '.run_started_at' 2>/dev/null || echo "1970-01-01T00:00:00Z") + gh api "/orgs/${ORG}/packages?package_type=container&per_page=100" --paginate \ + --jq '.[]|[.name,.updated_at]|@tsv' > /tmp/pkgs.tsv 2>/dev/null || true + total=$(wc -l < /tmp/pkgs.tsv | tr -d ' ') + # Bucket each by category; split fresh (pushed during THIS run) vs older. + awk -F'\t' -v rs="$run_started" ' + { n=$1; ts=$2; d=substr(ts,1,10) + if (n~/^core\// || n~/^gateways\//) c="base" + else if (n~/^benchmarks\/(swe-bench|terminal-bench|skills-bench)-./) c="per-task" + else if (n~/^(benchmarks|agents|models)\//) c="leaf" + else if (n~/-standalone$/) c="standalone" + else if (n~/^evals\//) c="combo" + else next + tot[c]++ + if (ts>=rs) fr[c]++; else { old[c]++; if(oldest[c]==""||dno[c]) no[c]=d } } + END { split("base leaf per-task combo standalone", o, " ") + for (i=1;i<=5;i++){ c=o[i]; if(!(c in tot)) continue + printf "| %s | %d | %d | %d | %s |\n", c, tot[c], fr[c]+0, old[c]+0, (old[c]?oldest[c]" → "no[c]:"—") } }' \ + /tmp/pkgs.tsv > /tmp/inv.md + gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs" --paginate \ + --jq '.jobs[]|select(.conclusion=="failure")|.name' > /tmp/failed.txt 2>/dev/null || true + nfail=$(wc -l < /tmp/failed.txt | tr -d ' ') + { + echo "" + echo "## Fleet inventory — every published image, by freshness" + if [ "${total:-0}" -gt 0 ]; then + echo "_${total} container packages in \`${ORG}\`. **built this run** = pushed since \`${run_started}\`; **older** = not rebuilt this run (stale if its inputs changed since)._" + echo "| category | total | built this run | older | older span |" + echo "|---|--:|--:|--:|---|" + cat /tmp/inv.md + else + echo "_Inventory unavailable (empty package list — check the job's \`packages: read\` scope)._" + fi + echo "" + echo "## Failures this run: ${nfail}" + if [ "${nfail:-0}" -gt 0 ]; then + echo "| bucket | count |"; echo "|---|--:|" + sed -E 's/^([A-Za-z][A-Za-z-]*).*/\1/' /tmp/failed.txt | sort | uniq -c | sort -rn \ + | awk '{printf "| %s | %d |\n",$2,$1}' + echo "" + echo "
failed job names" + echo "" + sed 's/^/- /' /tmp/failed.txt + echo "" + echo "
" + fi + } >> "$GITHUB_STEP_SUMMARY" + # Loud annotation for consumable layers that did NOT refresh this run. + while IFS='|' read -r _ cat _t _f old _rest; do + cat="$(echo "$cat" | xargs)"; old="$(echo "$old" | xargs)" + case "$cat" in + combo|standalone|base) + [ "${old:-0}" -gt 0 ] && echo "::warning title=Stale ${cat}::${old} ${cat} image(s) not rebuilt this run — refresh if inputs changed." ;; + esac + done < /tmp/inv.md + cp /tmp/pkgs.tsv fleet-inventory.tsv 2>/dev/null || true - name: Upload durable size report if: ${{ !inputs.dry_run }} uses: actions/upload-artifact@v4 @@ -759,4 +825,5 @@ jobs: path: | size-report.json size-report.csv + fleet-inventory.tsv if-no-files-found: ignore From b21ddbc8bd1738282f02e07e173b843703f8928a Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Tue, 23 Jun 2026 08:36:57 +0300 Subject: [PATCH 2/4] feat(release): fleet-inventory summary notice (package count + window + failures) Signed-off-by: Elron Bandel --- .github/workflows/release-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index 7bbf5aeb..fb03e14f 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -783,6 +783,7 @@ jobs: gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs" --paginate \ --jq '.jobs[]|select(.conclusion=="failure")|.name' > /tmp/failed.txt 2>/dev/null || true nfail=$(wc -l < /tmp/failed.txt | tr -d ' ') + echo "::notice title=Fleet inventory::${total} packages indexed (window since ${run_started}); ${nfail} failed job(s) this run." { echo "" echo "## Fleet inventory — every published image, by freshness" From dd4ff7eba3f0c39e1a9bf338964b001bbcbc1663 Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Tue, 23 Jun 2026 09:05:01 +0300 Subject: [PATCH 3/4] chore(release): temp debug echo for inventory total (remove before merge) Signed-off-by: Elron Bandel --- .github/workflows/release-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index fb03e14f..b504e774 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -784,6 +784,7 @@ jobs: --jq '.jobs[]|select(.conclusion=="failure")|.name' > /tmp/failed.txt 2>/dev/null || true nfail=$(wc -l < /tmp/failed.txt | tr -d ' ') echo "::notice title=Fleet inventory::${total} packages indexed (window since ${run_started}); ${nfail} failed job(s) this run." + echo "DBG_FLEET total=${total} run_started=${run_started} nfail=${nfail}" { echo "" echo "## Fleet inventory — every published image, by freshness" From 3ebb70f213e304f0391df6391293b672834a0e4c Mon Sep 17 00:00:00 2001 From: Elron Bandel Date: Wed, 24 Jun 2026 09:47:16 +0300 Subject: [PATCH 4/4] feat(release): list packages via read:packages PAT (GHCR_READ_TOKEN), GITHUB_TOKEN fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GITHUB_TOKEN can't list org packages (confirmed in CI: total=0; the failure summary + freshness window DO work on it). Use a GHCR_READ_TOKEN secret (a read:packages PAT) for the org-packages listing call only, falling back to GITHUB_TOKEN when unset — inventory then shows 'unavailable', never failing the release. Drops the temporary debug line. Signed-off-by: Elron Bandel --- .github/workflows/release-images.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml index b504e774..16f2cc47 100644 --- a/.github/workflows/release-images.yml +++ b/.github/workflows/release-images.yml @@ -762,6 +762,10 @@ jobs: # or failed images can't hide behind green stages. ORG="${REGISTRY##*/}" run_started=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq '.run_started_at' 2>/dev/null || echo "1970-01-01T00:00:00Z") + # GITHUB_TOKEN can't list org packages (confirmed: returns empty) — use a + # read:packages PAT if provided (secret GHCR_READ_TOKEN), else fall back + # (inventory stays empty → "unavailable" note, never fails the release). + GH_TOKEN="${{ secrets.GHCR_READ_TOKEN || github.token }}" \ gh api "/orgs/${ORG}/packages?package_type=container&per_page=100" --paginate \ --jq '.[]|[.name,.updated_at]|@tsv' > /tmp/pkgs.tsv 2>/dev/null || true total=$(wc -l < /tmp/pkgs.tsv | tr -d ' ') @@ -784,7 +788,6 @@ jobs: --jq '.jobs[]|select(.conclusion=="failure")|.name' > /tmp/failed.txt 2>/dev/null || true nfail=$(wc -l < /tmp/failed.txt | tr -d ' ') echo "::notice title=Fleet inventory::${total} packages indexed (window since ${run_started}); ${nfail} failed job(s) this run." - echo "DBG_FLEET total=${total} run_started=${run_started} nfail=${nfail}" { echo "" echo "## Fleet inventory — every published image, by freshness"