diff --git a/.github/AGENTS.md b/.github/AGENTS.md
new file mode 100644
index 00000000000..728455bb029
--- /dev/null
+++ b/.github/AGENTS.md
@@ -0,0 +1,43 @@
+CI/CD for the Chainlink Go monorepo.
+
+
+Prefer runs-on runners: https://runs-on.com/docs/
+Resolve smartcontractkit/.github actions and workflows from the local checkout first. Ask the user for a path if missing. Fetch the web only for a specific version, commit, or when local behavior disagrees with docs.
+Use [octometrics-action](https://github.com/kalverra/octometrics-action) for debugging resource usage.
+```yaml
+example-job:
+ name: Example Job
+ runs-on: ubuntu-latest
+ steps:
+ - name: Monitor
+ uses: kalverra/octometrics-action
+ with:
+ job_name: Example Job
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Optional, but highly recommended to prevent rate limiting
+ - name: Checkout code
+ uses: actions/checkout@v4
+ - name: Run rest of workflow
+ run: |
+ echo "Hello World!"
+```
+
+
+
+When changing workflows or actions, rank goals in this order:
+1. Maintainability: Prefer composable Actions and small scripts over large inline YAML or bash.
+2. Security: Apply least privilege and sound secret handling.
+3. Reliability: Fail clearly; handle transient errors where appropriate.
+4. Speed: Reduce wall-clock time.
+5. Cost: Reduce runner spend.
+
+
+
+Pin 3rd party action versions to commit hashes, not version tags.
+```yaml
+- name: Enable S3 Cache for Self-Hosted Runners
+ uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
+ with:
+ metrics: cpu,network,memory,disk
+```
+
diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
index e22615a6edf..c9fe3122f79 100644
--- a/.github/workflows/ci-core.yml
+++ b/.github/workflows/ci-core.yml
@@ -1,5 +1,5 @@
name: CI Core
-run-name: CI Core ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }}
+run-name: CI Core
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
@@ -17,6 +17,11 @@ on:
- cron: "0 0,6,12,18 * * *"
workflow_dispatch:
+# Debugging: set to "1" for per-package CPU/mem pprof under go_test_profiles/ (optional trace). Revert before merge.
+env:
+ CI_GO_TEST_PROFILE: "1"
+ CI_GO_TEST_PROFILE_TRACE: "1"
+
jobs:
filter:
name: Detect Changes
@@ -201,6 +206,7 @@ jobs:
# We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests
# when they should not be using it, while still allowing us to DRY up the setup
DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
+ CI_GO_TEST_PROFILE_DIR: ${{ github.workspace }}/go_test_profiles
strategy:
fail-fast: false
matrix:
@@ -247,6 +253,13 @@ jobs:
contents: read
actions: read
steps:
+ # DEBUG: Monitor resource usage
+ - name: Monitor
+ uses: kalverra/octometrics-action@main
+ with:
+ job_name: Core Tests (${{ matrix.type.cmd }})
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Enable S3 Cache for Self-Hosted Runners
uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
with:
@@ -340,7 +353,7 @@ jobs:
# See: https://github.com/golang/go/issues/69179
- name: Analyze and upload test results
- if: ${{ matrix.type.should-run == 'true' && matrix.type.trunk-auto-quarantine == 'true' && !cancelled() }}
+ if: ${{ matrix.type.should-run == 'true' && matrix.type.trunk-auto-quarantine == 'true' && !cancelled() && env.CI_GO_TEST_PROFILE != '1' }}
uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/v1
with:
junit-file-path: "./junit.xml"
@@ -389,6 +402,15 @@ jobs:
./deployment/junit.xml
retention-days: 7
+ - name: Upload Go test profiles
+ if: ${{ always() && matrix.type.should-run == 'true' && env.CI_GO_TEST_PROFILE == '1' }}
+ uses: actions/upload-artifact@v7
+ with:
+ name: ${{ matrix.type.cmd }}_go_test_profiles
+ path: go_test_profiles/
+ if-no-files-found: ignore
+ retention-days: 7
+
- name: Notify Slack on Race Test Failure
if: |
failure() &&
diff --git a/core/capabilities/vault/capability_test.go b/core/capabilities/vault/capability_test.go
index 81069729a84..89f609055b4 100644
--- a/core/capabilities/vault/capability_test.go
+++ b/core/capabilities/vault/capability_test.go
@@ -324,7 +324,7 @@ func TestCapability_CapabilityCall_SecretIdentifierOwnerMismatch(t *testing.T) {
gsr := &vault.GetSecretsRequest{
WorkflowOwner: tc.workflowOwner,
- Requests: reqs,
+ Requests: reqs,
}
anyproto, err := anypb.New(gsr)
require.NoError(t, err)
diff --git a/core/cmd/app.go b/core/cmd/app.go
index 3045e71c2a9..37c9aab0287 100644
--- a/core/cmd/app.go
+++ b/core/cmd/app.go
@@ -20,6 +20,8 @@ import (
"github.com/smartcontractkit/chainlink/v2/core/utils"
)
+// DEBUG: Go changes to trigger test runs
+
func removeHidden(cmds ...cli.Command) []cli.Command {
var ret []cli.Command
for _, cmd := range cmds {
diff --git a/core/cmd/solana_transaction_commands_test.go b/core/cmd/solana_transaction_commands_test.go
index 447c9ae4418..b7623840095 100644
--- a/core/cmd/solana_transaction_commands_test.go
+++ b/core/cmd/solana_transaction_commands_test.go
@@ -64,7 +64,6 @@ func TestShell_SolanaSendSol(t *testing.T) {
{amount: "0", expErr: "amount must be greater than zero"},
{amount: "asdf", expErr: "invalid amount:"},
} {
-
t.Run(tt.amount, func(t *testing.T) {
startBal, err := balance(from.PublicKey(), url)
require.NoError(t, err)
diff --git a/core/services/workflows/v2/config.go b/core/services/workflows/v2/config.go
index ed883b01c00..e310bee0519 100644
--- a/core/services/workflows/v2/config.go
+++ b/core/services/workflows/v2/config.go
@@ -111,7 +111,7 @@ type EngineLimiters struct {
UserMetricLabelsPerMetric limits.BoundLimiter[int]
UserMetricLabelValueLength limits.BoundLimiter[int]
- ExecutionTimestampsEnabled limits.GateLimiter
+ ExecutionTimestampsEnabled limits.GateLimiter
VaultOrgIDAsSecretOwnerEnabled limits.GateLimiter
}
diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests
index 1677ec6442e..2e0899e3cc1 100755
--- a/tools/bin/go_core_tests
+++ b/tools/bin/go_core_tests
@@ -3,6 +3,8 @@ set -o pipefail
set +e
SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"`
+# shellcheck source=go_test_package_parallelism.bash
+source "$SCRIPT_PATH/go_test_package_parallelism.bash"
OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"}
GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"25m"}
@@ -39,18 +41,69 @@ echo "Using JUNIT_FLAG: $JUNIT_FLAG"
echo "Test execution results: ---------------------"
echo ""
-# For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
-# For matching `go test -v` output - use only --format='standard-verbose'
-gotestsum \
- --format='standard-quiet' \
- --hide-summary=skipped \
- $RERUN_FLAGS \
- --packages='./...' \
- --jsonfile "$OUTPUT_FILE" \
- "$JUNIT_FLAG" \
- -- $GO_TEST_FLAGS
+if [[ "${CI_GO_TEST_PROFILE:-}" == "1" ]]; then
+ # go test only allows -cpuprofile/-memprofile/-trace when testing a single package per invocation.
+ echo "CI_GO_TEST_PROFILE: running gotestsum once per package with CPU and mem profiles"
+ PROFILE_DIR="${CI_GO_TEST_PROFILE_DIR:-./go_test_profiles}"
+ mkdir -p "$PROFILE_DIR"
+ PROFILE_PARALLEL="$(go_test_package_parallelism)"
+ # Strip coverage flags (single shared coverprofile path conflicts with per-package profiles)
+ GO_TEST_FLAGS=$(echo " $GO_TEST_FLAGS " | sed -E 's/ -coverprofile=[^ ]+ / /g;s/ -covermode=[^ ]+ / /g;s/ -coverpkg=[^ ]+ / /g' | tr -s ' ' | sed 's/^ //;s/ $//')
+ RERUN_FLAGS=""
+ JUNIT_FLAG=""
+ echo "Profile mode: rerun and junit disabled for this run; merge JSON events into $OUTPUT_FILE"
+ export GO_TEST_FLAGS_FOR_PROFILE="$GO_TEST_FLAGS"
+ export PROFILE_DIR
+ export CI_GO_TEST_PROFILE_TRACE_VALUE="${CI_GO_TEST_PROFILE_TRACE:-0}"
-EXITCODE=${PIPESTATUS[0]}
+ mapfile -t PKGS < <(go list -e -f '{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' ./... 2>/dev/null | grep -v '^$' || true)
+ if [[ ${#PKGS[@]} -eq 0 ]]; then
+ echo "No packages with tests found"
+ exit 1
+ fi
+ echo "Profiling ${#PKGS[@]} packages (parallelism=${PROFILE_PARALLEL})"
+ : >"$OUTPUT_FILE"
+ if printf '%s\0' "${PKGS[@]}" | xargs -0 -n1 -P"${PROFILE_PARALLEL}" bash -c '
+ set -euo pipefail
+ pkg="$1"
+ safe=$(echo "$pkg" | sed "s/[^a-zA-Z0-9._-]/_/g")
+ TRACE_FLAGS=()
+ if [[ "${CI_GO_TEST_PROFILE_TRACE_VALUE}" == "1" ]]; then
+ TRACE_FLAGS=(-trace="${PROFILE_DIR}/trace_${safe}.out")
+ fi
+ gotestsum \
+ --format=standard-quiet \
+ --hide-summary=skipped \
+ --packages="$pkg" \
+ --jsonfile="${PROFILE_DIR}/events_${safe}.jsonl" \
+ -- ${GO_TEST_FLAGS_FOR_PROFILE} \
+ -cpuprofile="${PROFILE_DIR}/cpu_${safe}.prof" \
+ -memprofile="${PROFILE_DIR}/mem_${safe}.prof" \
+ "${TRACE_FLAGS[@]}"
+ ' bash; then
+ EXITCODE=0
+ else
+ EXITCODE=1
+ fi
+ shopt -s nullglob
+ for f in "$PROFILE_DIR"/events_*.jsonl; do
+ cat "$f"
+ done >"$OUTPUT_FILE"
+ shopt -u nullglob
+else
+ # For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
+ # For matching `go test -v` output - use only --format='standard-verbose'
+ gotestsum \
+ --format='standard-quiet' \
+ --hide-summary=skipped \
+ $RERUN_FLAGS \
+ --packages='./...' \
+ --jsonfile "$OUTPUT_FILE" \
+ "$JUNIT_FLAG" \
+ -- $GO_TEST_FLAGS
+
+ EXITCODE=${PIPESTATUS[0]}
+fi
# Assert no known sensitive strings present in test logger output
printf "\n----------------------------------------------\n\n"
diff --git a/tools/bin/go_core_tests_integration b/tools/bin/go_core_tests_integration
index 01f09ff2df3..b8aeab2c437 100755
--- a/tools/bin/go_core_tests_integration
+++ b/tools/bin/go_core_tests_integration
@@ -3,6 +3,8 @@ set -o pipefail
set +e
SCRIPT_PATH=$(dirname "$0"); SCRIPT_PATH=$(eval "cd \"$SCRIPT_PATH\" && pwd")
+# shellcheck source=go_test_package_parallelism.bash
+source "$SCRIPT_PATH/go_test_package_parallelism.bash"
OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"}
GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"25m"}
@@ -57,17 +59,66 @@ echo "Using TEST_DIRS: $INTEGRATION_TEST_DIRS_SPACE_DELIMITED"
echo "Test execution results: ---------------------"
echo ""
-# For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
-# For matching `go test -v` output - use only --format='standard-verbose'
-gotestsum \
- --format='standard-quiet' \
- --hide-summary=skipped \
- $RERUN_FLAGS \
- --jsonfile "$OUTPUT_FILE" \
- "$JUNIT_FLAG" \
- -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED
+if [[ "${CI_GO_TEST_PROFILE:-}" == "1" ]]; then
+ echo "CI_GO_TEST_PROFILE: running gotestsum once per integration package with CPU and mem profiles"
+ PROFILE_DIR="${CI_GO_TEST_PROFILE_DIR:-./go_test_profiles}"
+ mkdir -p "$PROFILE_DIR"
+ PROFILE_PARALLEL="$(go_test_package_parallelism)"
+ GO_TEST_FLAGS=$(echo " $GO_TEST_FLAGS " | sed -E 's/ -coverprofile=[^ ]+ / /g;s/ -covermode=[^ ]+ / /g;s/ -coverpkg=[^ ]+ / /g' | tr -s ' ' | sed 's/^ //;s/ $//')
+ RERUN_FLAGS=""
+ JUNIT_FLAG=""
+ echo "Profile mode: rerun and junit disabled for this run; merge JSON events into $OUTPUT_FILE"
+ export GO_TEST_FLAGS_FOR_PROFILE="$GO_TEST_FLAGS"
+ export PROFILE_DIR
+ export CI_GO_TEST_PROFILE_TRACE_VALUE="${CI_GO_TEST_PROFILE_TRACE:-0}"
-EXITCODE=${PIPESTATUS[0]}
+ mapfile -t PKGS < <(go list -tags integration -e -f '{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' $INTEGRATION_TEST_DIRS_SPACE_DELIMITED 2>/dev/null | grep -v '^$' || true)
+ if [[ ${#PKGS[@]} -eq 0 ]]; then
+ echo "No integration packages with tests found"
+ exit 1
+ fi
+ echo "Profiling ${#PKGS[@]} integration packages (parallelism=${PROFILE_PARALLEL})"
+ : >"$OUTPUT_FILE"
+ if printf '%s\0' "${PKGS[@]}" | xargs -0 -n1 -P"${PROFILE_PARALLEL}" bash -c '
+ set -euo pipefail
+ pkg="$1"
+ safe=$(echo "$pkg" | sed "s/[^a-zA-Z0-9._-]/_/g")
+ TRACE_FLAGS=()
+ if [[ "${CI_GO_TEST_PROFILE_TRACE_VALUE}" == "1" ]]; then
+ TRACE_FLAGS=(-trace="${PROFILE_DIR}/trace_${safe}.out")
+ fi
+ gotestsum \
+ --format=standard-quiet \
+ --hide-summary=skipped \
+ --packages="$pkg" \
+ --jsonfile="${PROFILE_DIR}/events_${safe}.jsonl" \
+ -- --tags integration ${GO_TEST_FLAGS_FOR_PROFILE} \
+ -cpuprofile="${PROFILE_DIR}/cpu_${safe}.prof" \
+ -memprofile="${PROFILE_DIR}/mem_${safe}.prof" \
+ "${TRACE_FLAGS[@]}"
+ ' bash; then
+ EXITCODE=0
+ else
+ EXITCODE=1
+ fi
+ shopt -s nullglob
+ for f in "$PROFILE_DIR"/events_*.jsonl; do
+ cat "$f"
+ done >"$OUTPUT_FILE"
+ shopt -u nullglob
+else
+ # For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
+ # For matching `go test -v` output - use only --format='standard-verbose'
+ gotestsum \
+ --format='standard-quiet' \
+ --hide-summary=skipped \
+ $RERUN_FLAGS \
+ --jsonfile "$OUTPUT_FILE" \
+ "$JUNIT_FLAG" \
+ -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED
+
+ EXITCODE=${PIPESTATUS[0]}
+fi
# Assert no known sensitive strings present in test logger output
printf "\n----------------------------------------------\n\n"
@@ -83,5 +134,5 @@ if [[ $EXITCODE != 0 ]]; then
else
echo "All tests passed!"
fi
-echo "go_core_tests exiting with code $EXITCODE"
+echo "go_core_tests_integration exiting with code $EXITCODE"
exit $EXITCODE
diff --git a/tools/bin/go_test_package_parallelism.bash b/tools/bin/go_test_package_parallelism.bash
new file mode 100644
index 00000000000..4a40281d54f
--- /dev/null
+++ b/tools/bin/go_test_package_parallelism.bash
@@ -0,0 +1,28 @@
+# Default package parallelism for `go test ./...` matches `go help build`:
+# -p n ... The default for -p is GOMAXPROCS, normally the number of CPUs available.
+# Honor the same inputs: optional CI_GO_TEST_PROFILE_PARALLEL, then $GOMAXPROCS, then go env, then OS CPU count.
+go_test_package_parallelism() {
+ local n
+ if [[ -n "${CI_GO_TEST_PROFILE_PARALLEL:-}" ]] && [[ "${CI_GO_TEST_PROFILE_PARALLEL}" =~ ^[0-9]+$ ]] && [[ "${CI_GO_TEST_PROFILE_PARALLEL}" -gt 0 ]]; then
+ echo "${CI_GO_TEST_PROFILE_PARALLEL}"
+ return
+ fi
+ if [[ -n "${GOMAXPROCS:-}" ]] && [[ "${GOMAXPROCS}" =~ ^[0-9]+$ ]] && [[ "${GOMAXPROCS}" -gt 0 ]]; then
+ echo "${GOMAXPROCS}"
+ return
+ fi
+ n=$(go env GOMAXPROCS 2>/dev/null | tr -d ' \r\n\t' || true)
+ if [[ -n "$n" ]] && [[ "$n" =~ ^[0-9]+$ ]] && [[ "$n" -gt 0 ]]; then
+ echo "$n"
+ return
+ fi
+ if command -v nproc >/dev/null 2>&1; then
+ nproc
+ elif command -v getconf >/dev/null 2>&1; then
+ getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1
+ elif [[ "$(uname -s)" == "Darwin" ]]; then
+ sysctl -n hw.logicalcpu 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1
+ else
+ echo 1
+ fi
+}