Report scheduled full-stack failures

.github/workflows/canary.yml

@@ -1,10 +1,7 @@
 name: Canary — plugin vs opencode latest
 
 on:
-  schedule:
-    # Twice daily; opencode ships stable releases every day or two.
-    - cron: "23 3 * * *"
-    - cron: "23 15 * * *"
+  workflow_call:
   workflow_dispatch:
 
 jobs:
@@ -55,21 +52,3 @@ jobs:
             @opencode-ai/sdk@1.17.18 @opencode-ai/plugin@1.17.18
       - run: bash "$GITHUB_WORKSPACE/tests/ci/npm_with_retry.sh" install -g opencode-ai@latest
       - run: bash scripts/smoke-loader.sh
-
-  notify:
-    needs: [contract, loader-smoke]
-    if: >
-      always() && github.event_name == 'schedule' &&
-      (needs.contract.result != 'success' || needs.loader-smoke.result != 'success')
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    steps:
-      - name: Notify Google Chat on scheduled failure
-        env:
-          WEBHOOK_URL: ${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }}
-        run: |
-          [ -n "$WEBHOOK_URL" ] || { echo "GOOGLE_CHAT_WEBHOOK_URL is missing"; exit 1; }
-          curl --fail-with-body --retry 3 --retry-all-errors --max-time 20 \
-            -sS -X POST -H 'Content-Type: application/json' \
-            -d "{\"text\": \"🚨 opencode-plugin canary FAILED: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" \
-            "$WEBHOOK_URL"

.github/workflows/live-stack.yml

@@ -1,12 +1,12 @@
 name: Full stack e2e
 
 on:
+  schedule:
+    - cron: "23 3 * * *"
+    - cron: "23 15 * * *"
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
   workflow_dispatch:
-  workflow_run:
-    workflows: ["Canary — plugin vs opencode latest"]
-    types: [completed]
 
 permissions:
   contents: read
@@ -17,13 +17,22 @@ concurrency:
   queue: max
 
 jobs:
-  channels:
+  canary:
     if: >
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'pull_request' && github.event.pull_request.draft == false &&
+       github.event.pull_request.head.repo.full_name == github.repository)
+    uses: ./.github/workflows/canary.yml
+
+  channels:
+    needs: canary
+    if: >
+      !cancelled() && needs.canary.result == 'success' &&
+      ((github.event_name == 'pull_request' && github.event.pull_request.draft == false &&
        github.event.pull_request.head.repo.full_name == github.repository) ||
       github.event_name == 'workflow_dispatch' ||
-      (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' &&
-       github.event.workflow_run.head_branch == 'main')
+      github.event_name == 'schedule')
     uses: ./.github/workflows/live-channels.yml
     with:
       orchestrated: true
@@ -36,8 +45,7 @@ jobs:
       ((github.event_name == 'pull_request' && github.event.pull_request.draft == false &&
         github.event.pull_request.head.repo.full_name == github.repository) ||
        github.event_name == 'workflow_dispatch' ||
-       (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' &&
-        github.event.workflow_run.head_branch == 'main'))
+       github.event_name == 'schedule')
     uses: ./.github/workflows/live-a2a.yml
     with:
       orchestrated: true
@@ -50,8 +58,7 @@ jobs:
       ((github.event_name == 'pull_request' && github.event.pull_request.draft == false &&
         github.event.pull_request.head.repo.full_name == github.repository) ||
        github.event_name == 'workflow_dispatch' ||
-       (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' &&
-        github.event.workflow_run.head_branch == 'main'))
+       github.event_name == 'schedule')
     uses: ./.github/workflows/live-voice.yml
     with:
       orchestrated: true
@@ -67,8 +74,7 @@ jobs:
       ((github.event_name == 'pull_request' && github.event.pull_request.draft == false &&
         github.event.pull_request.head.repo.full_name == github.repository) ||
        github.event_name == 'workflow_dispatch' ||
-       (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' &&
-        github.event.workflow_run.head_branch == 'main'))
+       github.event_name == 'schedule')
     uses: ./.github/workflows/live-external-events.yml
     with:
       orchestrated: true
@@ -82,8 +88,7 @@ jobs:
       ((github.event_name == 'pull_request' && github.event.pull_request.draft == false &&
         github.event.pull_request.head.repo.full_name == github.repository) ||
        github.event_name == 'workflow_dispatch' ||
-       (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' &&
-        github.event.workflow_run.head_branch == 'main'))
+       github.event_name == 'schedule')
     runs-on: ubuntu-latest
     steps:
       - name: Require every live suite to pass
@@ -101,26 +106,3 @@ jobs:
             if [ "$result" != "success" ]; then failed=1; fi
           done
           exit "$failed"
-
-  notify:
-    needs: [channels, a2a, voice, external-events, full-stack]
-    if: >
-      always() && github.event_name == 'workflow_run' &&
-      github.event.workflow_run.event == 'schedule' &&
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.head_branch == 'main' &&
-      (needs.channels.result != 'success' || needs.a2a.result != 'success' ||
-       needs.voice.result != 'success' ||
-       needs.external-events.result != 'success' || needs.full-stack.result != 'success')
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    steps:
-      - name: Notify Google Chat
-        env:
-          WEBHOOK_URL: ${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }}
-        run: |
-          [ -n "$WEBHOOK_URL" ] || { echo "GOOGLE_CHAT_WEBHOOK_URL is missing"; exit 1; }
-          curl --fail-with-body --retry 3 --retry-all-errors --max-time 20 \
-            -sS -X POST -H 'Content-Type: application/json' \
-            -d "{\"text\": \"🚨 opencode-plugin full-stack e2e FAILED (chained off the canary): ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" \
-            "$WEBHOOK_URL"

.github/workflows/scheduled-failure-report.yml

@@ -0,0 +1,64 @@
+name: Scheduled failure reporting
+
+on:
+  workflow_run:
+    workflows: ["Full stack e2e"]
+    types: [completed]
+
+permissions:
+  actions: read
+
+jobs:
+  report:
+    if: >-
+      ${{
+        github.event.workflow_run.event == 'schedule' &&
+        (
+          github.event.workflow_run.conclusion == 'failure' ||
+          github.event.workflow_run.conclusion == 'timed_out' ||
+          github.event.workflow_run.conclusion == 'startup_failure'
+        )
+      }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Post failure notification
+        env:
+          NOTIFICATION_URL: ${{ secrets.SCHEDULED_FAILURE_NOTIFICATION_URL }}
+          REPOSITORY: ${{ github.repository }}
+          WORKFLOW_NAME: ${{ github.event.workflow_run.name }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+          RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
+          RUN_URL: ${{ github.event.workflow_run.html_url }}
+        run: |
+          set -euo pipefail
+          [ -n "${NOTIFICATION_URL}" ] || { echo "::error::Missing failure notification URL"; exit 1; }
+          repo_slug="${REPOSITORY//\//-}"
+          thread_key="ci-${repo_slug}-${RUN_ID}-${RUN_ATTEMPT}"
+          payload="$(jq -n --arg repository "${REPOSITORY}" --arg workflow "${WORKFLOW_NAME}" --arg run_url "${RUN_URL}" '{text: ("Scheduled integration checks failed\n\nRepository: " + $repository + "\nWorkflow: " + $workflow + "\nRun: " + $run_url)}')"
+          curl --fail-with-body --silent --show-error --retry 3 --retry-connrefused --retry-delay 2 --retry-max-time 90 --connect-timeout 10 --max-time 30 -X POST "${NOTIFICATION_URL}&threadKey=${thread_key}&messageReplyOption=REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD" -H 'Content-Type: application/json' --data "${payload}"
+
+      - name: Send signed failure event
+        if: always()
+        env:
+          RECEIVER_URL: ${{ secrets.SCHEDULED_FAILURE_RECEIVER_URL }}
+          SIGNING_SECRET: ${{ secrets.SCHEDULED_FAILURE_SIGNING_SECRET }}
+          FAILURE_ENVIRONMENT: ${{ secrets.SCHEDULED_FAILURE_ENVIRONMENT }}
+          REPOSITORY: ${{ github.repository }}
+          WORKFLOW_NAME: ${{ github.event.workflow_run.name }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+          RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
+          RUN_URL: ${{ github.event.workflow_run.html_url }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          set -euo pipefail
+          [ -n "${RECEIVER_URL}" ] || { echo "::error::Missing failure receiver URL"; exit 1; }
+          [ -n "${SIGNING_SECRET}" ] || { echo "::error::Missing failure signing secret"; exit 1; }
+          [ -n "${FAILURE_ENVIRONMENT}" ] || { echo "::error::Missing failure environment"; exit 1; }
+          echo "::add-mask::${SIGNING_SECRET}"
+          repo_slug="${REPOSITORY//\//-}"
+          thread_key="ci-${repo_slug}-${RUN_ID}-${RUN_ATTEMPT}"
+          payload="$(jq -c -n --arg event_type "scheduled_ci_failure" --arg source "${REPOSITORY}" --arg repository "${REPOSITORY}" --arg workflow "${WORKFLOW_NAME}" --arg source_job "${WORKFLOW_NAME}" --arg environment "${FAILURE_ENVIRONMENT}" --argjson run_id "${RUN_ID}" --argjson run_attempt "${RUN_ATTEMPT}" --arg run_url "${RUN_URL}" --arg head_sha "${HEAD_SHA}" --arg chat_thread_key "${thread_key}" '{$event_type, $source, $repository, $workflow, $source_job, $environment, $run_id, $run_attempt, $run_url, $head_sha, $chat_thread_key}')"
+          signature="$(printf '%s' "${payload}" | openssl dgst -sha256 -hmac "${SIGNING_SECRET}" -binary | xxd -p -c 256)"
+          request_id="ci:${REPOSITORY}:${RUN_ID}:${RUN_ATTEMPT}:${FAILURE_ENVIRONMENT}"
+          curl --fail-with-body --silent --show-error --retry 3 --retry-connrefused --retry-delay 2 --retry-max-time 90 --connect-timeout 10 --max-time 30 -X POST "${RECEIVER_URL}" -H 'Content-Type: application/json' -H 'X-GitHub-Event: workflow_run' -H "X-Hub-Signature-256: sha256=${signature}" -H "X-Inkbox-Request-Id: ${request_id}" --data-binary "${payload}"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@inkbox/opencode-plugin",
-  "version": "0.2.10",
+  "version": "0.2.11",
   "private": true,
   "description": "Inkbox for opencode \u2014 give your agent an email address, a phone number (SMS/MMS + voice), iMessage, contacts, notes, and an encrypted credential vault.",
   "license": "MIT",

tests/scheduled-failure-reporting.test.ts

@@ -0,0 +1,32 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const readWorkflow = (name: string) =>
+  readFileSync(path.join(root, ".github", "workflows", name), "utf8");
+
+describe("scheduled failure reporting", () => {
+  it("uses one scheduled and signed reporting path", () => {
+    const canary = readWorkflow("canary.yml");
+    const stack = readWorkflow("live-stack.yml");
+    const report = readWorkflow("scheduled-failure-report.yml");
+
+    expect(canary).toContain("workflow_call:");
+    expect(canary).not.toContain("schedule:");
+    expect(canary).not.toContain("notify:");
+    expect(stack).toContain("schedule:");
+    expect(stack).toContain("uses: ./.github/workflows/canary.yml");
+    expect(stack).not.toContain("workflow_run:");
+    expect(stack).not.toContain("notify:");
+    expect(report).toContain('workflows: ["Full stack e2e"]');
+    expect(report).toContain("github.event.workflow_run.event == 'schedule'");
+    expect(report).toContain("timed_out");
+    expect(report).toContain("startup_failure");
+    expect(report).toContain("if: always()");
+    expect(report).toContain("actions: read");
+    expect(report).toContain("X-Hub-Signature-256");
+    expect(report).toContain("chat_thread_key");
+  });
+});