diff --git a/.github/workflows/cre-soak-memory-leak.yml b/.github/workflows/cre-soak-memory-leak.yml index c5f68be2c6b..2493e1415dd 100644 --- a/.github/workflows/cre-soak-memory-leak.yml +++ b/.github/workflows/cre-soak-memory-leak.yml @@ -12,6 +12,10 @@ on: description: "Chainlink image tag to use" required: true type: string + chainlink_version: + description: "Test / orchestration code reference to use (defaults to github.sha)" + required: false + type: string workflow_call: inputs: ecr_name: @@ -21,6 +25,10 @@ on: chainlink_image_tag: required: true type: string + chainlink_version: + description: "Test / orchestration code reference to use (defaults to github.sha)" + required: false + type: string jobs: soak: @@ -51,7 +59,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 with: - ref: ${{ github.sha }} + ref: ${{ inputs.chainlink_version || github.sha }} persist-credentials: false - name: Set up Go @@ -129,12 +137,14 @@ jobs: -count=1 \ github.com/smartcontractkit/chainlink/system-tests/tests/soak/cre - - name: Upload alloc.pprof + - name: Upload performance reports if: always() uses: actions/upload-artifact@v7 with: - name: alloc.pprof - path: system-tests/tests/soak/cre/alloc.pprof + name: pprof + path: | + system-tests/tests/soak/cre/*.pprof + system-tests/tests/soak/cre/admin-profiles retention-days: 7 - name: Upload Docker logs diff --git a/core/cmd/admin_commands.go b/core/cmd/admin_commands.go index 7d6bbbe2bbd..522a7f0e117 100644 --- a/core/cmd/admin_commands.go +++ b/core/cmd/admin_commands.go @@ -375,10 +375,10 @@ func (s *Shell) Profile(c *cli.Context) error { errs[0] = s.profile(ctx, genDir, "", vitals, seconds) }() for i, name := range names { - go func() { + go func(i int, name string) { defer wg.Done() - errs[i] = s.profile(ctx, genDir, name, vitals, seconds) - }() + errs[i+1] = s.profile(ctx, genDir, name, vitals, seconds) + }(i, name) } wg.Wait() diff --git a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml index b94d80787b6..dc851e083dc 100644 --- a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml +++ b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don.toml @@ -43,7 +43,7 @@ # because bootstrap job for capability DON will be created on the boostrap node from this DON supported_evm_chains = [1337, 2337] - env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" } + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", GODEBUG="memprofilerate=1024" } capabilities = ["ocr3", "custom-compute", "web-api-trigger", "cron", "http-action", "http-trigger", "consensus", "don-time", "write-evm-1337", "read-contract-1337", "evm-1337"] registry_based_launch_allowlist = ["cron-trigger@1.0.0"] diff --git a/core/scripts/cre/environment/configs/workflow-gateway-don.toml b/core/scripts/cre/environment/configs/workflow-gateway-don.toml index 197853faa28..81095103958 100644 --- a/core/scripts/cre/environment/configs/workflow-gateway-don.toml +++ b/core/scripts/cre/environment/configs/workflow-gateway-don.toml @@ -40,7 +40,7 @@ http_port_range_start = 10000 p2p_port_range_start = 12000 - env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node" } + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", GODEBUG="memprofilerate=1" } capabilities = ["ocr3", "custom-compute", "web-api-target", "web-api-trigger", "vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "write-evm-1337", "write-evm-2337", "evm-1337", "evm-2337", "read-contract-1337", "read-contract-2337"] registry_based_launch_allowlist = ["cron-trigger@1.0.0"] diff --git a/core/scripts/go.mod b/core/scripts/go.mod index d3a2d8cb42a..f680717f32c 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -54,7 +54,7 @@ require ( github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260119171452-39c98c3b33cd github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260409211238-5b99921cbc7c github.com/smartcontractkit/chainlink-protos/job-distributor v0.18.0 - github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15 + github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834 github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.20 github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.5 github.com/smartcontractkit/chainlink-testing-framework/seth v1.51.5 diff --git a/core/scripts/go.sum b/core/scripts/go.sum index c49c7452924..06a2ee0559a 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1711,8 +1711,8 @@ github.com/smartcontractkit/chainlink-sui v0.0.0-20260409162403-8bca8f7c9de5 h1: github.com/smartcontractkit/chainlink-sui v0.0.0-20260409162403-8bca8f7c9de5/go.mod h1:YQDu2RcdoAzI5xlhtpbjvaQQZwkUt/Q+IhLbP25M614= github.com/smartcontractkit/chainlink-sui/deployment v0.0.0-20260304150206-c64e48eb0cb0 h1:5NdsaclAfx+p8lZUZ3WIqMW3M9Cze1ZVPENOQhha1pk= github.com/smartcontractkit/chainlink-sui/deployment v0.0.0-20260304150206-c64e48eb0cb0/go.mod h1:IfeW6t5Yc5293H5ixuooAft+wYBMSFQWKjbBTwYiKr4= -github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15 h1:6wIGZ2DNbjCGayndMJMwwFTfzJcWoSPRQ+q5xxQpgg0= -github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15/go.mod h1:BALK9cj8sk12e15UF6uDhifHgIApa+6N11TcQfInEro= +github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834 h1:BVcc1ondsNzErmAHngzengb9UiQgp4F6jH+5fJ93kUk= +github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834/go.mod h1:BALK9cj8sk12e15UF6uDhifHgIApa+6N11TcQfInEro= github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.20 h1:8D2DUnn7mLUZOLhPDGGFKKvBrgU6LQd00tq2VOprvfI= github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.20/go.mod h1:98jNYBOPuKWJw9a8x0LgQuudp5enrHhQQP5Hq0YwRB8= github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.0 h1:PWAMYu0WaAMBfbpxCpFJGRIDHmcgmYin6a+UQC0OdtY= diff --git a/core/web/loop_registry.go b/core/web/loop_registry.go index 0cc7e1ffbcc..a61679e8e61 100644 --- a/core/web/loop_registry.go +++ b/core/web/loop_registry.go @@ -127,7 +127,7 @@ func (l *LoopRegistryServer) pluginMetricHandler(gc *gin.Context) { gc.Data(http.StatusOK, "text/plain", b) } -const PPROFOverheadSeconds = 30 +const PPROFOverheadSeconds = 360 func pprofURLVals(gc *gin.Context) (urlVals url.Values, timeout time.Duration) { urlVals = make(url.Values) diff --git a/system-tests/lib/cre/don/secrets/secrets.go b/system-tests/lib/cre/don/secrets/secrets.go index e974c349234..05d928cfa26 100644 --- a/system-tests/lib/cre/don/secrets/secrets.go +++ b/system-tests/lib/cre/don/secrets/secrets.go @@ -20,7 +20,7 @@ import ( type nodeSecret struct { EthKeys nodeEthKeyWrapper `toml:"EVM"` SolKeys nodeSolKeyWrapper `toml:"Solana"` - AptosKeys nodeAptosKeyWrapper `toml:"Aptos"` + AptosKeys nodeAptosKeyWrapper `toml:"-"` // TODO set back to `toml:"Aptos"` before merging P2PKey nodeP2PKey `toml:"P2PKey"` DKGRecipientKey nodeDKGRecipientKey `toml:"DKGRecipientKey"` diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 81d7fe8c6e0..5fb3e7e23db 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -40,7 +40,7 @@ require ( github.com/smartcontractkit/chainlink-protos/job-distributor v0.18.0 github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997 github.com/smartcontractkit/chainlink-solana v1.1.2-0.20260407162454-407b2a207dcc - github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15 + github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834 github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.15 github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.5 diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 75a5b3c27ef..378f3f128ea 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1678,8 +1678,8 @@ github.com/smartcontractkit/chainlink-sui v0.0.0-20260409162403-8bca8f7c9de5 h1: github.com/smartcontractkit/chainlink-sui v0.0.0-20260409162403-8bca8f7c9de5/go.mod h1:YQDu2RcdoAzI5xlhtpbjvaQQZwkUt/Q+IhLbP25M614= github.com/smartcontractkit/chainlink-sui/deployment v0.0.0-20260304150206-c64e48eb0cb0 h1:5NdsaclAfx+p8lZUZ3WIqMW3M9Cze1ZVPENOQhha1pk= github.com/smartcontractkit/chainlink-sui/deployment v0.0.0-20260304150206-c64e48eb0cb0/go.mod h1:IfeW6t5Yc5293H5ixuooAft+wYBMSFQWKjbBTwYiKr4= -github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15 h1:6wIGZ2DNbjCGayndMJMwwFTfzJcWoSPRQ+q5xxQpgg0= -github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15/go.mod h1:BALK9cj8sk12e15UF6uDhifHgIApa+6N11TcQfInEro= +github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834 h1:BVcc1ondsNzErmAHngzengb9UiQgp4F6jH+5fJ93kUk= +github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834/go.mod h1:BALK9cj8sk12e15UF6uDhifHgIApa+6N11TcQfInEro= github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.15 h1:usf6YCNmSO8R1/rU28wUfIdp7zXlqGGOAttXW5mgkXU= github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.15/go.mod h1:YqrpawYGRkT/jcvXcmaZeZPOtu0erIenrHl5Mb8+U/c= github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.0 h1:PWAMYu0WaAMBfbpxCpFJGRIDHmcgmYin6a+UQC0OdtY= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index fc696cb67c2..86dd26324b1 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -68,7 +68,7 @@ require ( github.com/smartcontractkit/chainlink-protos/job-distributor v0.18.0 github.com/smartcontractkit/chainlink-protos/ring/go v0.0.0-20260331131315-f08a616d8dcd github.com/smartcontractkit/chainlink-protos/workflows/go v0.0.0-20260323124644-faea187e6997 - github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15 + github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834 github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.0 github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.7 github.com/smartcontractkit/chainlink-testing-framework/lib v1.54.5 diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 30292e865ba..b24e390a423 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1862,8 +1862,8 @@ github.com/smartcontractkit/chainlink-sui v0.0.0-20260409162403-8bca8f7c9de5 h1: github.com/smartcontractkit/chainlink-sui v0.0.0-20260409162403-8bca8f7c9de5/go.mod h1:YQDu2RcdoAzI5xlhtpbjvaQQZwkUt/Q+IhLbP25M614= github.com/smartcontractkit/chainlink-sui/deployment v0.0.0-20260304150206-c64e48eb0cb0 h1:5NdsaclAfx+p8lZUZ3WIqMW3M9Cze1ZVPENOQhha1pk= github.com/smartcontractkit/chainlink-sui/deployment v0.0.0-20260304150206-c64e48eb0cb0/go.mod h1:IfeW6t5Yc5293H5ixuooAft+wYBMSFQWKjbBTwYiKr4= -github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15 h1:6wIGZ2DNbjCGayndMJMwwFTfzJcWoSPRQ+q5xxQpgg0= -github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.15/go.mod h1:BALK9cj8sk12e15UF6uDhifHgIApa+6N11TcQfInEro= +github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834 h1:BVcc1ondsNzErmAHngzengb9UiQgp4F6jH+5fJ93kUk= +github.com/smartcontractkit/chainlink-testing-framework/framework v0.15.16-0.20260415064325-cd082d2e5834/go.mod h1:BALK9cj8sk12e15UF6uDhifHgIApa+6N11TcQfInEro= github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.18 h1:1ng+p/+85zcVLHB050PiWUAjOcxyd4KjwkUlJy34rgE= github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose v0.1.18/go.mod h1:2+OrSz56pdgtY0Oc20nCS9LH/bEksFDBQjoR82De5PI= github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake v0.10.0 h1:PWAMYu0WaAMBfbpxCpFJGRIDHmcgmYin6a+UQC0OdtY=