fix: poll crates.io after publishing crates

.github/workflows/ci.yml

@@ -64,6 +64,9 @@ jobs:
       - name: Check release version consistency
         run: bash scripts/check_release_versions.sh
 
+      - name: Check the release waits for crates.io to index each crate
+        run: bash tests/release/crates-index-poll.test.sh
+
       - name: Test internal dependency pin coverage
         run: bash tests/release/release-version-pins.test.sh
 

.github/workflows/release.yml

@@ -207,22 +207,24 @@ jobs:
         #   proto + core -> types -> brain -> daemon -> cli
         # Every generated crate archive is verified before upload. Existing
         # versions are skipped so a partially completed release can be resumed.
-        # sleep 30 lets crates.io index each crate before its dependents publish.
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
           VERSION: ${{ github.ref_name }}
         run: |
           test -n "${CARGO_REGISTRY_TOKEN:-}" || { echo '::error::CARGO_REGISTRY_TOKEN is not configured'; exit 1; }
+          source scripts/crates-index-poll.sh
           VERSION="${VERSION#v}"
           for crate in sysknife-proto sysknife-core sysknife-types \
                        sysknife-brain sysknife-daemon sysknife-cli; do
-            if curl --fail --silent --retry 3 --retry-connrefused \
-              --user-agent 'sysknife-release/1.0 (https://github.com/lacs-project/sysknife)' \
-              "https://crates.io/api/v1/crates/${crate}/${VERSION}" >/dev/null; then
+            if crate_version_status "$crate" "$VERSION"; then
               echo "${crate} ${VERSION} already exists; skipping"
             else
+              status=$?
+              if ((status != 1)); then
+                exit "$status"
+              fi
               cargo publish -p "$crate" --locked --token "$CARGO_REGISTRY_TOKEN"
-              sleep 30
+              wait_for_crate_version "$crate" "$VERSION"
             fi
           done
 

scripts/crates-index-poll.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+crate_version_status() {
+    local crate="$1"
+    local version="$2"
+    local curl_cmd="${CRATES_IO_CURL:-curl}"
+    local http_code
+
+    if ! http_code="$(
+        "$curl_cmd" \
+            --silent \
+            --show-error \
+            --retry 3 \
+            --retry-connrefused \
+            --output /dev/null \
+            --write-out '%{http_code}' \
+            --user-agent 'sysknife-release/1.0 (https://github.com/lacs-project/sysknife)' \
+            "https://crates.io/api/v1/crates/${crate}/${version}"
+    )"; then
+        printf 'ERROR: could not reach crates.io while checking %s %s\n' \
+            "$crate" "$version" >&2
+        return 2
+    fi
+
+    case "$http_code" in
+        200)
+            return 0
+            ;;
+        404)
+            return 1
+            ;;
+        *)
+            printf 'ERROR: crates.io returned HTTP %s while checking %s %s\n' \
+                "$http_code" "$crate" "$version" >&2
+            return 2
+            ;;
+    esac
+}
+
+wait_for_crate_version() {
+    local crate="$1"
+    local version="$2"
+    local timeout="${CRATES_IO_POLL_TIMEOUT:-300}"
+    local interval="${CRATES_IO_POLL_INTERVAL:-15}"
+    local sleep_cmd="${CRATES_IO_SLEEP:-sleep}"
+    local elapsed=0
+    local status
+
+    while :; do
+        if crate_version_status "$crate" "$version"; then
+            printf '%s %s is available on crates.io\n' "$crate" "$version"
+            return 0
+        else
+            status=$?
+        fi
+
+        if ((status != 1)); then
+            return "$status"
+        fi
+
+        if ((elapsed >= timeout)); then
+            printf 'ERROR: timed out after %s seconds waiting for %s %s to appear on crates.io\n' \
+                "$timeout" "$crate" "$version" >&2
+            return 1
+        fi
+
+        "$sleep_cmd" "$interval"
+        elapsed=$((elapsed + interval))
+    done
+}

tests/release/crates-index-poll.test.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$repo_root"
+
+helper="scripts/crates-index-poll.sh"
+workflow=".github/workflows/release.yml"
+
+[[ -f "$helper" ]] || {
+    printf 'FAIL: missing %s\n' "$helper" >&2
+    exit 1
+}
+
+source scripts/crates-index-poll.sh
+
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+count_file="${tmp}/count"
+printf '0\n' > "$count_file"
+
+stub="${tmp}/curl-stub"
+cat > "$stub" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+count="$(cat "$CRATES_IO_TEST_COUNT_FILE")"
+count=$((count + 1))
+printf '%s\n' "$count" > "$CRATES_IO_TEST_COUNT_FILE"
+case "$count" in
+    1|2|3) printf '404' ;;
+    *) printf '200' ;;
+esac
+EOF
+chmod +x "$stub"
+
+CRATES_IO_CURL="$stub" \
+CRATES_IO_TEST_COUNT_FILE="$count_file" \
+CRATES_IO_POLL_TIMEOUT=3 \
+CRATES_IO_POLL_INTERVAL=1 \
+CRATES_IO_SLEEP=true \
+wait_for_crate_version sysknife-daemon 0.15.0
+
+[[ "$(cat "$count_file")" == "4" ]] || {
+    printf 'FAIL: expected 4 poll requests, got %s\n' "$(cat "$count_file")" >&2
+    exit 1
+}
+
+printf '0\n' > "$count_file"
+
+cat > "$stub" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+count="$(cat "$CRATES_IO_TEST_COUNT_FILE")"
+count=$((count + 1))
+printf '%s\n' "$count" > "$CRATES_IO_TEST_COUNT_FILE"
+printf '404'
+EOF
+chmod +x "$stub"
+
+if output="$(
+    CRATES_IO_CURL="$stub" \
+    CRATES_IO_TEST_COUNT_FILE="$count_file" \
+    CRATES_IO_POLL_TIMEOUT=2 \
+    CRATES_IO_POLL_INTERVAL=1 \
+    CRATES_IO_SLEEP=true \
+    wait_for_crate_version sysknife-daemon 0.15.0 2>&1
+)"; then
+    printf 'FAIL: perpetual 404 unexpectedly succeeded\n' >&2
+    exit 1
+fi
+
+grep -Fq 'sysknife-daemon 0.15.0' <<<"$output"
+grep -Fq 'timed out after 2 seconds' <<<"$output"
+[[ "$(cat "$count_file")" == "3" ]]
+
+printf '0\n' > "$count_file"
+
+if output="$(
+    CRATES_IO_CURL="$stub" \
+    CRATES_IO_TEST_COUNT_FILE="$count_file" \
+    CRATES_IO_POLL_INTERVAL=15 \
+    CRATES_IO_SLEEP=true \
+    wait_for_crate_version sysknife-daemon 0.15.0 2>&1
+)"; then
+    printf 'FAIL: default timeout unexpectedly succeeded\n' >&2
+    exit 1
+fi
+
+grep -Fq 'timed out after 300 seconds' <<<"$output"
+[[ "$(cat "$count_file")" == "21" ]]
+
+cat > "$stub" <<'EOF'
+#!/usr/bin/env bash
+exit 7
+EOF
+chmod +x "$stub"
+
+if output="$(
+    CRATES_IO_CURL="$stub" \
+    CRATES_IO_POLL_TIMEOUT=2 \
+    CRATES_IO_POLL_INTERVAL=1 \
+    CRATES_IO_SLEEP=true \
+    wait_for_crate_version sysknife-daemon 0.15.0 2>&1
+)"; then
+    printf 'FAIL: network failure unexpectedly succeeded\n' >&2
+    exit 1
+fi
+
+grep -Fq 'could not reach crates.io' <<<"$output"
+grep -Fq 'sysknife-daemon 0.15.0' <<<"$output"
+
+grep -Fq 'source scripts/crates-index-poll.sh' "$workflow" || {
+    printf 'FAIL: release workflow does not load the polling helper\n' >&2
+    exit 1
+}
+
+grep -Fq 'wait_for_crate_version "$crate" "$VERSION"' "$workflow" || {
+    printf 'FAIL: release workflow does not wait for the index after publishing\n' >&2
+    exit 1
+}
+
+if grep -Fq 'sleep 30' "$workflow"; then
+    printf 'FAIL: release workflow still contains fixed sleep 30\n' >&2
+    exit 1
+fi
+
+printf 'crates index polling contract passed.\n'