[DX-3764] Experiment with Core Unit Test Postgres To Reduce Bottlenecks

.github/AGENTS.md

@@ -0,0 +1,43 @@
+CI/CD for the Chainlink Go monorepo.
+
+<resources>
+Prefer runs-on runners: https://runs-on.com/docs/
+Resolve smartcontractkit/.github actions and workflows from the local checkout first. Ask the user for a path if missing. Fetch the web only for a specific version, commit, or when local behavior disagrees with docs.
+Use [octometrics-action](https://github.com/kalverra/octometrics-action) for debugging resource usage.
+```yaml
+example-job:
+  name: Example Job
+  runs-on: ubuntu-latest
+  steps:
+    - name: Monitor
+      uses: kalverra/octometrics-action
+      with:
+        job_name: Example Job
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Optional, but highly recommended to prevent rate limiting
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Run rest of workflow
+      run: |
+        echo "Hello World!"
+```
+</resources>
+
+<priorities>
+When changing workflows or actions, rank goals in this order:
+1. Maintainability: Prefer composable Actions and small scripts over large inline YAML or bash.
+2. Security: Apply least privilege and sound secret handling.
+3. Reliability: Fail clearly; handle transient errors where appropriate.
+4. Speed: Reduce wall-clock time.
+5. Cost: Reduce runner spend.
+</priorities>
+
+<rules>
+Pin 3rd party action versions to commit hashes, not version tags.
+```yaml
+- name: Enable S3 Cache for Self-Hosted Runners
+  uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
+  with:
+    metrics: cpu,network,memory,disk
+```
+</rules>

.github/workflows/ci-core.yml

@@ -1,5 +1,5 @@
 name: CI Core
-run-name: CI Core ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }}
+run-name: CI Core
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
@@ -17,6 +17,11 @@
     - cron: "0 0,6,12,18 * * *"
   workflow_dispatch:
 
+# Debugging: set to "1" for per-package CPU/mem pprof under go_test_profiles/ (optional trace). Revert before merge.
+env:
+  CI_GO_TEST_PROFILE: "1"
+  CI_GO_TEST_PROFILE_TRACE: "1"
+
 jobs:
   filter:
     name: Detect Changes
@@ -201,6 +206,7 @@
       # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests
       # when they should not be using it, while still allowing us to DRY up the setup
       DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
+      CI_GO_TEST_PROFILE_DIR: ${{ github.workspace }}/go_test_profiles
     strategy:
       fail-fast: false
       matrix:
@@ -247,6 +253,13 @@
       contents: read
       actions: read
     steps:
+      # DEBUG: Monitor resource usage
+      - name: Monitor
+        uses: kalverra/octometrics-action@main
+        with:
+          job_name: Core Tests (${{ matrix.type.cmd }})
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Enable S3 Cache for Self-Hosted Runners
         uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0
         with:
@@ -340,7 +353,7 @@
           # See: https://github.com/golang/go/issues/69179
 
       - name: Analyze and upload test results
-        if: ${{ matrix.type.should-run == 'true' && matrix.type.trunk-auto-quarantine == 'true' && !cancelled() }}
+        if: ${{ matrix.type.should-run == 'true' && matrix.type.trunk-auto-quarantine == 'true' && !cancelled() && env.CI_GO_TEST_PROFILE != '1' }}
         uses: smartcontractkit/.github/actions/branch-out-upload@branch-out-upload/v1
         with:
           junit-file-path: "./junit.xml"
@@ -389,6 +402,15 @@
             ./deployment/junit.xml
           retention-days: 7
 
+      - name: Upload Go test profiles
+        if: ${{ always() && matrix.type.should-run == 'true' && env.CI_GO_TEST_PROFILE == '1' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: ${{ matrix.type.cmd }}_go_test_profiles
+          path: go_test_profiles/
+          if-no-files-found: ignore
+          retention-days: 7
+
       - name: Notify Slack on Race Test Failure
         if: |
           failure() &&

core/capabilities/vault/capability_test.go

@@ -324,7 +324,7 @@ func TestCapability_CapabilityCall_SecretIdentifierOwnerMismatch(t *testing.T) {
 
 			gsr := &vault.GetSecretsRequest{
 				WorkflowOwner: tc.workflowOwner,
-				Requests: reqs,
+				Requests:      reqs,
 			}
 			anyproto, err := anypb.New(gsr)
 			require.NoError(t, err)

core/cmd/app.go

@@ -20,6 +20,8 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 )
 
+// DEBUG: Go changes to trigger test runs
+
 func removeHidden(cmds ...cli.Command) []cli.Command {
 	var ret []cli.Command
 	for _, cmd := range cmds {

core/cmd/solana_transaction_commands_test.go

@@ -64,7 +64,6 @@ func TestShell_SolanaSendSol(t *testing.T) {
 		{amount: "0", expErr: "amount must be greater than zero"},
 		{amount: "asdf", expErr: "invalid amount:"},
 	} {
-
 		t.Run(tt.amount, func(t *testing.T) {
 			startBal, err := balance(from.PublicKey(), url)
 			require.NoError(t, err)

core/services/workflows/v2/config.go

@@ -111,7 +111,7 @@ type EngineLimiters struct {
 	UserMetricLabelsPerMetric  limits.BoundLimiter[int]
 	UserMetricLabelValueLength limits.BoundLimiter[int]
 
-	ExecutionTimestampsEnabled limits.GateLimiter
+	ExecutionTimestampsEnabled     limits.GateLimiter
 	VaultOrgIDAsSecretOwnerEnabled limits.GateLimiter
 }
 

tools/bin/go_core_tests

@@ -3,6 +3,8 @@ set -o pipefail
 set +e
 
 SCRIPT_PATH=`dirname "$0"`; SCRIPT_PATH=`eval "cd \"$SCRIPT_PATH\" && pwd"`
+# shellcheck source=go_test_package_parallelism.bash
+source "$SCRIPT_PATH/go_test_package_parallelism.bash"
 OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
 JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"}
 GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"25m"}
@@ -39,18 +41,69 @@ echo "Using JUNIT_FLAG: $JUNIT_FLAG"
 echo "Test execution results: ---------------------"
 echo ""
 
-# For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
-# For matching `go test -v` output - use only --format='standard-verbose'
-gotestsum \
-  --format='standard-quiet' \
-  --hide-summary=skipped \
-  $RERUN_FLAGS \
-  --packages='./...' \
-  --jsonfile "$OUTPUT_FILE" \
-  "$JUNIT_FLAG" \
-  -- $GO_TEST_FLAGS
+if [[ "${CI_GO_TEST_PROFILE:-}" == "1" ]]; then
+  # go test only allows -cpuprofile/-memprofile/-trace when testing a single package per invocation.
+  echo "CI_GO_TEST_PROFILE: running gotestsum once per package with CPU and mem profiles"
+  PROFILE_DIR="${CI_GO_TEST_PROFILE_DIR:-./go_test_profiles}"
+  mkdir -p "$PROFILE_DIR"
+  PROFILE_PARALLEL="$(go_test_package_parallelism)"
+  # Strip coverage flags (single shared coverprofile path conflicts with per-package profiles)
+  GO_TEST_FLAGS=$(echo " $GO_TEST_FLAGS " | sed -E 's/ -coverprofile=[^ ]+ / /g;s/ -covermode=[^ ]+ / /g;s/ -coverpkg=[^ ]+ / /g' | tr -s ' ' | sed 's/^ //;s/ $//')
+  RERUN_FLAGS=""
+  JUNIT_FLAG=""
+  echo "Profile mode: rerun and junit disabled for this run; merge JSON events into $OUTPUT_FILE"
+  export GO_TEST_FLAGS_FOR_PROFILE="$GO_TEST_FLAGS"
+  export PROFILE_DIR
+  export CI_GO_TEST_PROFILE_TRACE_VALUE="${CI_GO_TEST_PROFILE_TRACE:-0}"
 
-EXITCODE=${PIPESTATUS[0]}
+  mapfile -t PKGS < <(go list -e -f '{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' ./... 2>/dev/null | grep -v '^$' || true)
+  if [[ ${#PKGS[@]} -eq 0 ]]; then
+    echo "No packages with tests found"
+    exit 1
+  fi
+  echo "Profiling ${#PKGS[@]} packages (parallelism=${PROFILE_PARALLEL})"
+  : >"$OUTPUT_FILE"
+  if printf '%s\0' "${PKGS[@]}" | xargs -0 -n1 -P"${PROFILE_PARALLEL}" bash -c '
+    set -euo pipefail
+    pkg="$1"
+    safe=$(echo "$pkg" | sed "s/[^a-zA-Z0-9._-]/_/g")
+    TRACE_FLAGS=()
+    if [[ "${CI_GO_TEST_PROFILE_TRACE_VALUE}" == "1" ]]; then
+      TRACE_FLAGS=(-trace="${PROFILE_DIR}/trace_${safe}.out")
+    fi
+    gotestsum \
+      --format=standard-quiet \
+      --hide-summary=skipped \
+      --packages="$pkg" \
+      --jsonfile="${PROFILE_DIR}/events_${safe}.jsonl" \
+      -- ${GO_TEST_FLAGS_FOR_PROFILE} \
+        -cpuprofile="${PROFILE_DIR}/cpu_${safe}.prof" \
+        -memprofile="${PROFILE_DIR}/mem_${safe}.prof" \
+        "${TRACE_FLAGS[@]}"
+  ' bash; then
+    EXITCODE=0
+  else
+    EXITCODE=1
+  fi
+  shopt -s nullglob
+  for f in "$PROFILE_DIR"/events_*.jsonl; do
+    cat "$f"
+  done >"$OUTPUT_FILE"
+  shopt -u nullglob
+else
+  # For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
+  # For matching `go test -v` output - use only --format='standard-verbose'
+  gotestsum \
+    --format='standard-quiet' \
+    --hide-summary=skipped \
+    $RERUN_FLAGS \
+    --packages='./...' \
+    --jsonfile "$OUTPUT_FILE" \
+    "$JUNIT_FLAG" \
+    -- $GO_TEST_FLAGS
+
+  EXITCODE=${PIPESTATUS[0]}
+fi
 
 # Assert no known sensitive strings present in test logger output
 printf "\n----------------------------------------------\n\n"

tools/bin/go_core_tests_integration

@@ -3,6 +3,8 @@ set -o pipefail
 set +e
 
 SCRIPT_PATH=$(dirname "$0"); SCRIPT_PATH=$(eval "cd \"$SCRIPT_PATH\" && pwd")
+# shellcheck source=go_test_package_parallelism.bash
+source "$SCRIPT_PATH/go_test_package_parallelism.bash"
 OUTPUT_FILE=${OUTPUT_FILE:-"./output.txt"}
 JUNIT_FILE=${JUNIT_FILE:-"./junit.xml"}
 GO_TEST_TIMEOUT=${GO_TEST_TIMEOUT:-"25m"}
@@ -57,17 +59,66 @@ echo "Using TEST_DIRS: $INTEGRATION_TEST_DIRS_SPACE_DELIMITED"
 echo "Test execution results: ---------------------"
 echo ""
 
-# For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
-# For matching `go test -v` output - use only --format='standard-verbose'
-gotestsum \
-  --format='standard-quiet' \
-  --hide-summary=skipped \
-  $RERUN_FLAGS \
-  --jsonfile "$OUTPUT_FILE" \
-  "$JUNIT_FLAG" \
-  -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED
+if [[ "${CI_GO_TEST_PROFILE:-}" == "1" ]]; then
+  echo "CI_GO_TEST_PROFILE: running gotestsum once per integration package with CPU and mem profiles"
+  PROFILE_DIR="${CI_GO_TEST_PROFILE_DIR:-./go_test_profiles}"
+  mkdir -p "$PROFILE_DIR"
+  PROFILE_PARALLEL="$(go_test_package_parallelism)"
+  GO_TEST_FLAGS=$(echo " $GO_TEST_FLAGS " | sed -E 's/ -coverprofile=[^ ]+ / /g;s/ -covermode=[^ ]+ / /g;s/ -coverpkg=[^ ]+ / /g' | tr -s ' ' | sed 's/^ //;s/ $//')
+  RERUN_FLAGS=""
+  JUNIT_FLAG=""
+  echo "Profile mode: rerun and junit disabled for this run; merge JSON events into $OUTPUT_FILE"
+  export GO_TEST_FLAGS_FOR_PROFILE="$GO_TEST_FLAGS"
+  export PROFILE_DIR
+  export CI_GO_TEST_PROFILE_TRACE_VALUE="${CI_GO_TEST_PROFILE_TRACE:-0}"
 
-EXITCODE=${PIPESTATUS[0]}
+  mapfile -t PKGS < <(go list -tags integration -e -f '{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' $INTEGRATION_TEST_DIRS_SPACE_DELIMITED 2>/dev/null | grep -v '^$' || true)
+  if [[ ${#PKGS[@]} -eq 0 ]]; then
+    echo "No integration packages with tests found"
+    exit 1
+  fi
+  echo "Profiling ${#PKGS[@]} integration packages (parallelism=${PROFILE_PARALLEL})"
+  : >"$OUTPUT_FILE"
+  if printf '%s\0' "${PKGS[@]}" | xargs -0 -n1 -P"${PROFILE_PARALLEL}" bash -c '
+    set -euo pipefail
+    pkg="$1"
+    safe=$(echo "$pkg" | sed "s/[^a-zA-Z0-9._-]/_/g")
+    TRACE_FLAGS=()
+    if [[ "${CI_GO_TEST_PROFILE_TRACE_VALUE}" == "1" ]]; then
+      TRACE_FLAGS=(-trace="${PROFILE_DIR}/trace_${safe}.out")
+    fi
+    gotestsum \
+      --format=standard-quiet \
+      --hide-summary=skipped \
+      --packages="$pkg" \
+      --jsonfile="${PROFILE_DIR}/events_${safe}.jsonl" \
+      -- --tags integration ${GO_TEST_FLAGS_FOR_PROFILE} \
+        -cpuprofile="${PROFILE_DIR}/cpu_${safe}.prof" \
+        -memprofile="${PROFILE_DIR}/mem_${safe}.prof" \
+        "${TRACE_FLAGS[@]}"
+  ' bash; then
+    EXITCODE=0
+  else
+    EXITCODE=1
+  fi
+  shopt -s nullglob
+  for f in "$PROFILE_DIR"/events_*.jsonl; do
+    cat "$f"
+  done >"$OUTPUT_FILE"
+  shopt -u nullglob
+else
+  # For matching `go test` output - use --format='standard-quiet' and --hide-summary=skipped
+  # For matching `go test -v` output - use only --format='standard-verbose'
+  gotestsum \
+    --format='standard-quiet' \
+    --hide-summary=skipped \
+    $RERUN_FLAGS \
+    --jsonfile "$OUTPUT_FILE" \
+    "$JUNIT_FLAG" \
+    -- --tags integration $GO_TEST_FLAGS $INTEGRATION_TEST_DIRS_SPACE_DELIMITED
+
+  EXITCODE=${PIPESTATUS[0]}
+fi
 
 # Assert no known sensitive strings present in test logger output
 printf "\n----------------------------------------------\n\n"
@@ -83,5 +134,5 @@ if [[ $EXITCODE != 0 ]]; then
 else
   echo "All tests passed!"
 fi
-echo "go_core_tests exiting with code $EXITCODE"
+echo "go_core_tests_integration exiting with code $EXITCODE"
 exit $EXITCODE

tools/bin/go_test_package_parallelism.bash

@@ -0,0 +1,28 @@
+# Default package parallelism for `go test ./...` matches `go help build`:
+#   -p n  ... The default for -p is GOMAXPROCS, normally the number of CPUs available.
+# Honor the same inputs: optional CI_GO_TEST_PROFILE_PARALLEL, then $GOMAXPROCS, then go env, then OS CPU count.
+go_test_package_parallelism() {
+  local n
+  if [[ -n "${CI_GO_TEST_PROFILE_PARALLEL:-}" ]] && [[ "${CI_GO_TEST_PROFILE_PARALLEL}" =~ ^[0-9]+$ ]] && [[ "${CI_GO_TEST_PROFILE_PARALLEL}" -gt 0 ]]; then
+    echo "${CI_GO_TEST_PROFILE_PARALLEL}"
+    return
+  fi
+  if [[ -n "${GOMAXPROCS:-}" ]] && [[ "${GOMAXPROCS}" =~ ^[0-9]+$ ]] && [[ "${GOMAXPROCS}" -gt 0 ]]; then
+    echo "${GOMAXPROCS}"
+    return
+  fi
+  n=$(go env GOMAXPROCS 2>/dev/null | tr -d ' \r\n\t' || true)
+  if [[ -n "$n" ]] && [[ "$n" =~ ^[0-9]+$ ]] && [[ "$n" -gt 0 ]]; then
+    echo "$n"
+    return
+  fi
+  if command -v nproc >/dev/null 2>&1; then
+    nproc
+  elif command -v getconf >/dev/null 2>&1; then
+    getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1
+  elif [[ "$(uname -s)" == "Darwin" ]]; then
+    sysctl -n hw.logicalcpu 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1
+  else
+    echo 1
+  fi
+}