diff --git a/.github/AGENTS.md b/.github/AGENTS.md new file mode 100644 index 00000000000..728455bb029 --- /dev/null +++ b/.github/AGENTS.md @@ -0,0 +1,43 @@ +CI/CD for the Chainlink Go monorepo. + + +Prefer runs-on runners: https://runs-on.com/docs/ +Resolve smartcontractkit/.github actions and workflows from the local checkout first. Ask the user for a path if missing. Fetch the web only for a specific version, commit, or when local behavior disagrees with docs. +Use [octometrics-action](https://github.com/kalverra/octometrics-action) for debugging resource usage. +```yaml +example-job: + name: Example Job + runs-on: ubuntu-latest + steps: + - name: Monitor + uses: kalverra/octometrics-action + with: + job_name: Example Job + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Optional, but highly recommended to prevent rate limiting + - name: Checkout code + uses: actions/checkout@v4 + - name: Run rest of workflow + run: | + echo "Hello World!" +``` + + + +When changing workflows or actions, rank goals in this order: +1. Maintainability: Prefer composable Actions and small scripts over large inline YAML or bash. +2. Security: Apply least privilege and sound secret handling. +3. Reliability: Fail clearly; handle transient errors where appropriate. +4. Speed: Reduce wall-clock time. +5. Cost: Reduce runner spend. + + + +Pin 3rd party action versions to commit hashes, not version tags. +```yaml +- name: Enable S3 Cache for Self-Hosted Runners + uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 + with: + metrics: cpu,network,memory,disk +``` + diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index e22615a6edf..c9fe3122f79 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -1,5 +1,5 @@ name: CI Core -run-name: CI Core ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }} +run-name: CI Core concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} @@ -17,6 +17,11 @@ on: - cron: "0 0,6,12,18 * * *" workflow_dispatch: +# Debugging: set to "1" for per-package CPU/mem pprof under go_test_profiles/ (optional trace). Revert before merge. +env: + CI_GO_TEST_PROFILE: "1" + CI_GO_TEST_PROFILE_TRACE: "1" + jobs: filter: name: Detect Changes @@ -201,6 +206,7 @@ jobs: # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests # when they should not be using it, while still allowing us to DRY up the setup DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + CI_GO_TEST_PROFILE_DIR: ${{ github.workspace }}/go_test_profiles strategy: fail-fast: false matrix: @@ -247,6 +253,13 @@ jobs: contents: read actions: read steps: + # DEBUG: Monitor resource usage + - name: Monitor + uses: kalverra/octometrics-action@main + with: + job_name: Core Tests (${{ matrix.type.cmd }}) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Enable S3 Cache for Self-Hosted Runners uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 with: @@ -340,7 +353,7 @@ jobs: # See: https://github.com/golang/go/issues/69179 - name: Analyze and upload test results - if: ${{ matrix.type.should-run == 'true' && matrix.type.trunk-auto-quarantine == 'true' && !cancelled() }} + if: ${{ matrix.type.should-run == 'true' && matrix.type.trunk-auto-quarantine == 'true' && !cancelled() && env.CI_GO_TEST_PROFILE != '1' }} uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/v1 with: junit-file-path: "./junit.xml" @@ -389,6 +402,15 @@ jobs: ./deployment/junit.xml retention-days: 7 + - name: Upload Go test profiles + if: ${{ always() && matrix.type.should-run == 'true' && env.CI_GO_TEST_PROFILE == '1' }} + uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.type.cmd }}_go_test_profiles + path: go_test_profiles/ + if-no-files-found: ignore + retention-days: 7 + - name: Notify Slack on Race Test Failure if: | failure() && diff --git a/core/capabilities/vault/capability_test.go b/core/capabilities/vault/capability_test.go index 81069729a84..89f609055b4 100644 --- a/core/capabilities/vault/capability_test.go +++ b/core/capabilities/vault/capability_test.go @@ -324,7 +324,7 @@ func TestCapability_CapabilityCall_SecretIdentifierOwnerMismatch(t *testing.T) { gsr := &vault.GetSecretsRequest{ WorkflowOwner: tc.workflowOwner, - Requests: reqs, + Requests: reqs, } anyproto, err := anypb.New(gsr) require.NoError(t, err) diff --git a/core/cmd/app.go b/core/cmd/app.go index 3045e71c2a9..37c9aab0287 100644 --- a/core/cmd/app.go +++ b/core/cmd/app.go @@ -20,6 +20,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/utils" ) +// DEBUG: Go changes to trigger test runs + func removeHidden(cmds ...cli.Command) []cli.Command { var ret []cli.Command for _, cmd := range cmds { diff --git a/core/cmd/solana_transaction_commands_test.go b/core/cmd/solana_transaction_commands_test.go index 447c9ae4418..b7623840095 100644 --- a/core/cmd/solana_transaction_commands_test.go +++ b/core/cmd/solana_transaction_commands_test.go @@ -64,7 +64,6 @@ func TestShell_SolanaSendSol(t *testing.T) { {amount: "0", expErr: "amount must be greater than zero"}, {amount: "asdf", expErr: "invalid amount:"}, } { - t.Run(tt.amount, func(t *testing.T) { startBal, err := balance(from.PublicKey(), url) require.NoError(t, err) diff --git a/core/services/workflows/v2/config.go b/core/services/workflows/v2/config.go index ed883b01c00..e310bee0519 100644 --- a/core/services/workflows/v2/config.go +++ b/core/services/workflows/v2/config.go @@ -111,7 +111,7 @@ type EngineLimiters struct { UserMetricLabelsPerMetric limits.BoundLimiter[int] UserMetricLabelValueLength limits.BoundLimiter[int] - ExecutionTimestampsEnabled limits.GateLimiter + ExecutionTimestampsEnabled limits.GateLimiter VaultOrgIDAsSecretOwnerEnabled limits.GateLimiter } diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests index 1677ec6442e..2e0899e3cc1 100755 --- a/tools/bin/go_core_tests +++ b/tools/bin/go_core_tests @@ -3,6 +3,8 @@ set -o pipefail set +e SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"` +# shellcheck source=go_test_package_parallelism.bash +source "$SCRIPT_PATH/go_test_package_parallelism.bash" OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"} GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"25m"} @@ -39,18 +41,69 @@ echo "Using JUNIT_FLAG: $JUNIT_FLAG" echo "Test execution results: ---------------------" echo "" -# For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped -# For matching `go test -v` output - use only --format='standard-verbose' -gotestsum \ - --format='standard-quiet' \ - --hide-summary=skipped \ - $RERUN_FLAGS \ - --packages='./...' \ - --jsonfile "$OUTPUT_FILE" \ - "$JUNIT_FLAG" \ - -- $GO_TEST_FLAGS +if [[ "${CI_GO_TEST_PROFILE:-}" == "1" ]]; then + # go test only allows -cpuprofile/-memprofile/-trace when testing a single package per invocation. + echo "CI_GO_TEST_PROFILE: running gotestsum once per package with CPU and mem profiles" + PROFILE_DIR="${CI_GO_TEST_PROFILE_DIR:-./go_test_profiles}" + mkdir -p "$PROFILE_DIR" + PROFILE_PARALLEL="$(go_test_package_parallelism)" + # Strip coverage flags (single shared coverprofile path conflicts with per-package profiles) + GO_TEST_FLAGS=$(echo " $GO_TEST_FLAGS " | sed -E 's/ -coverprofile=[^ ]+ / /g;s/ -covermode=[^ ]+ / /g;s/ -coverpkg=[^ ]+ / /g' | tr -s ' ' | sed 's/^ //;s/ $//') + RERUN_FLAGS="" + JUNIT_FLAG="" + echo "Profile mode: rerun and junit disabled for this run; merge JSON events into $OUTPUT_FILE" + export GO_TEST_FLAGS_FOR_PROFILE="$GO_TEST_FLAGS" + export PROFILE_DIR + export CI_GO_TEST_PROFILE_TRACE_VALUE="${CI_GO_TEST_PROFILE_TRACE:-0}" -EXITCODE=${PIPESTATUS[0]} + mapfile -t PKGS < <(go list -e -f '{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' ./... 2>/dev/null | grep -v '^$' || true) + if [[ ${#PKGS[@]} -eq 0 ]]; then + echo "No packages with tests found" + exit 1 + fi + echo "Profiling ${#PKGS[@]} packages (parallelism=${PROFILE_PARALLEL})" + : >"$OUTPUT_FILE" + if printf '%s\0' "${PKGS[@]}" | xargs -0 -n1 -P"${PROFILE_PARALLEL}" bash -c ' + set -euo pipefail + pkg="$1" + safe=$(echo "$pkg" | sed "s/[^a-zA-Z0-9._-]/_/g") + TRACE_FLAGS=() + if [[ "${CI_GO_TEST_PROFILE_TRACE_VALUE}" == "1" ]]; then + TRACE_FLAGS=(-trace="${PROFILE_DIR}/trace_${safe}.out") + fi + gotestsum \ + --format=standard-quiet \ + --hide-summary=skipped \ + --packages="$pkg" \ + --jsonfile="${PROFILE_DIR}/events_${safe}.jsonl" \ + -- ${GO_TEST_FLAGS_FOR_PROFILE} \ + -cpuprofile="${PROFILE_DIR}/cpu_${safe}.prof" \ + -memprofile="${PROFILE_DIR}/mem_${safe}.prof" \ + "${TRACE_FLAGS[@]}" + ' bash; then + EXITCODE=0 + else + EXITCODE=1 + fi + shopt -s nullglob + for f in "$PROFILE_DIR"/events_*.jsonl; do + cat "$f" + done >"$OUTPUT_FILE" + shopt -u nullglob +else + # For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped + # For matching `go test -v` output - use only --format='standard-verbose' + gotestsum \ + --format='standard-quiet' \ + --hide-summary=skipped \ + $RERUN_FLAGS \ + --packages='./...' \ + --jsonfile "$OUTPUT_FILE" \ + "$JUNIT_FLAG" \ + -- $GO_TEST_FLAGS + + EXITCODE=${PIPESTATUS[0]} +fi # Assert no known sensitive strings present in test logger output printf "\n----------------------------------------------\n\n" diff --git a/tools/bin/go_core_tests_integration b/tools/bin/go_core_tests_integration index 01f09ff2df3..b8aeab2c437 100755 --- a/tools/bin/go_core_tests_integration +++ b/tools/bin/go_core_tests_integration @@ -3,6 +3,8 @@ set -o pipefail set +e SCRIPT_PATH=$(dirname "$0"); SCRIPT_PATH=$(eval "cd \"$SCRIPT_PATH\" && pwd") +# shellcheck source=go_test_package_parallelism.bash +source "$SCRIPT_PATH/go_test_package_parallelism.bash" OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"} JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"} GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"25m"} @@ -57,17 +59,66 @@ echo "Using TEST_DIRS: $INTEGRATION_TEST_DIRS_SPACE_DELIMITED" echo "Test execution results: ---------------------" echo "" -# For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped -# For matching `go test -v` output - use only --format='standard-verbose' -gotestsum \ - --format='standard-quiet' \ - --hide-summary=skipped \ - $RERUN_FLAGS \ - --jsonfile "$OUTPUT_FILE" \ - "$JUNIT_FLAG" \ - -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED +if [[ "${CI_GO_TEST_PROFILE:-}" == "1" ]]; then + echo "CI_GO_TEST_PROFILE: running gotestsum once per integration package with CPU and mem profiles" + PROFILE_DIR="${CI_GO_TEST_PROFILE_DIR:-./go_test_profiles}" + mkdir -p "$PROFILE_DIR" + PROFILE_PARALLEL="$(go_test_package_parallelism)" + GO_TEST_FLAGS=$(echo " $GO_TEST_FLAGS " | sed -E 's/ -coverprofile=[^ ]+ / /g;s/ -covermode=[^ ]+ / /g;s/ -coverpkg=[^ ]+ / /g' | tr -s ' ' | sed 's/^ //;s/ $//') + RERUN_FLAGS="" + JUNIT_FLAG="" + echo "Profile mode: rerun and junit disabled for this run; merge JSON events into $OUTPUT_FILE" + export GO_TEST_FLAGS_FOR_PROFILE="$GO_TEST_FLAGS" + export PROFILE_DIR + export CI_GO_TEST_PROFILE_TRACE_VALUE="${CI_GO_TEST_PROFILE_TRACE:-0}" -EXITCODE=${PIPESTATUS[0]} + mapfile -t PKGS < <(go list -tags integration -e -f '{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' $INTEGRATION_TEST_DIRS_SPACE_DELIMITED 2>/dev/null | grep -v '^$' || true) + if [[ ${#PKGS[@]} -eq 0 ]]; then + echo "No integration packages with tests found" + exit 1 + fi + echo "Profiling ${#PKGS[@]} integration packages (parallelism=${PROFILE_PARALLEL})" + : >"$OUTPUT_FILE" + if printf '%s\0' "${PKGS[@]}" | xargs -0 -n1 -P"${PROFILE_PARALLEL}" bash -c ' + set -euo pipefail + pkg="$1" + safe=$(echo "$pkg" | sed "s/[^a-zA-Z0-9._-]/_/g") + TRACE_FLAGS=() + if [[ "${CI_GO_TEST_PROFILE_TRACE_VALUE}" == "1" ]]; then + TRACE_FLAGS=(-trace="${PROFILE_DIR}/trace_${safe}.out") + fi + gotestsum \ + --format=standard-quiet \ + --hide-summary=skipped \ + --packages="$pkg" \ + --jsonfile="${PROFILE_DIR}/events_${safe}.jsonl" \ + -- --tags integration ${GO_TEST_FLAGS_FOR_PROFILE} \ + -cpuprofile="${PROFILE_DIR}/cpu_${safe}.prof" \ + -memprofile="${PROFILE_DIR}/mem_${safe}.prof" \ + "${TRACE_FLAGS[@]}" + ' bash; then + EXITCODE=0 + else + EXITCODE=1 + fi + shopt -s nullglob + for f in "$PROFILE_DIR"/events_*.jsonl; do + cat "$f" + done >"$OUTPUT_FILE" + shopt -u nullglob +else + # For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped + # For matching `go test -v` output - use only --format='standard-verbose' + gotestsum \ + --format='standard-quiet' \ + --hide-summary=skipped \ + $RERUN_FLAGS \ + --jsonfile "$OUTPUT_FILE" \ + "$JUNIT_FLAG" \ + -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED + + EXITCODE=${PIPESTATUS[0]} +fi # Assert no known sensitive strings present in test logger output printf "\n----------------------------------------------\n\n" @@ -83,5 +134,5 @@ if [[ $EXITCODE != 0 ]]; then else echo "All tests passed!" fi -echo "go_core_tests exiting with code $EXITCODE" +echo "go_core_tests_integration exiting with code $EXITCODE" exit $EXITCODE diff --git a/tools/bin/go_test_package_parallelism.bash b/tools/bin/go_test_package_parallelism.bash new file mode 100644 index 00000000000..4a40281d54f --- /dev/null +++ b/tools/bin/go_test_package_parallelism.bash @@ -0,0 +1,28 @@ +# Default package parallelism for `go test ./...` matches `go help build`: +# -p n ... The default for -p is GOMAXPROCS, normally the number of CPUs available. +# Honor the same inputs: optional CI_GO_TEST_PROFILE_PARALLEL, then $GOMAXPROCS, then go env, then OS CPU count. +go_test_package_parallelism() { + local n + if [[ -n "${CI_GO_TEST_PROFILE_PARALLEL:-}" ]] && [[ "${CI_GO_TEST_PROFILE_PARALLEL}" =~ ^[0-9]+$ ]] && [[ "${CI_GO_TEST_PROFILE_PARALLEL}" -gt 0 ]]; then + echo "${CI_GO_TEST_PROFILE_PARALLEL}" + return + fi + if [[ -n "${GOMAXPROCS:-}" ]] && [[ "${GOMAXPROCS}" =~ ^[0-9]+$ ]] && [[ "${GOMAXPROCS}" -gt 0 ]]; then + echo "${GOMAXPROCS}" + return + fi + n=$(go env GOMAXPROCS 2>/dev/null | tr -d ' \r\n\t' || true) + if [[ -n "$n" ]] && [[ "$n" =~ ^[0-9]+$ ]] && [[ "$n" -gt 0 ]]; then + echo "$n" + return + fi + if command -v nproc >/dev/null 2>&1; then + nproc + elif command -v getconf >/dev/null 2>&1; then + getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1 + elif [[ "$(uname -s)" == "Darwin" ]]; then + sysctl -n hw.logicalcpu 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1 + else + echo 1 + fi +}