Debug logs + docker logs for compat pipeline, fix Automation nightly (smartcontractkit#21911)

* debug

* fix pprof in nightly

* more debug

* set log levels everywhere

Author: skudasov

.github/workflows/devenv-compat.yml

@@ -230,10 +230,19 @@ jobs:
             echo "ci_summary.txt not found" >> $GITHUB_STEP_SUMMARY
           fi
 
-      - name: Upload Logs
+      - name: Collect Docker container logs
         if: always()
-        uses: actions/upload-artifact@v7
+        working-directory: ${{ env.WORKING_DIR }}
+        run: |
+          mkdir -p logs
+          for container in $(docker ps -q); do
+            container_name=$(docker inspect --format='{{.Name}}' $container | sed 's/^\///')
+            docker logs $container > "logs/${container_name}.log" 2>&1
+          done
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
           name: container-logs-smoke-${{ job.check_run_id }}
-          path: ${{ env.LOGS_DIR }}
-          retention-days: 3
+          path: ${{ env.LOGS_DIR }}

.github/workflows/devenv-core-compat.yml

@@ -13,9 +13,6 @@ permissions:
   contents: read
 
 jobs:
-  # CREcluster compatibility and upgrade test
-  # TODO
-
   # Data Feeds v1 upgrade test
   # this verifies that all the versions existing in RANE SOT for Data Feeds v1 can be upgraded to the latest version
   # TODO: disabled for now until we have connectivity
@@ -33,7 +30,7 @@ jobs:
   #     no-git-rollback: true
   #     working-directory: "devenv"
   #     logs-directory: "./devenv/tests/ocr2/logs"
-  #     ctf-log-level: "info"
+  #     ctf-log-level: "debug" # trace, debug, info
   #     strip-image-suffix: "v"
   #   secrets: inherit
 
@@ -63,9 +60,9 @@ jobs:
       # directory with your developer environment to run buildcmd and envcmd
       working-directory: "devenv"
       # directory for executing your tests, to run testcmd
-      logs-directory: "./devenv/tests/ocr2/logs"
+      logs-directory: "./devenv/logs"
       # CTF log level
-      ctf-log-level: "info"
+      ctf-log-level: "debug"
       # this is needed to remove 'v' from Git tags to match SDLC ECR tags
       strip-image-suffix: "v"
     secrets: inherit
@@ -95,7 +92,7 @@ jobs:
       # directory for executing your tests, to run testcmd
       logs-directory: "./system-tests/tests/smoke/cre/logs"
       # CTF log level
-      ctf-log-level: "info"
+      ctf-log-level: "debug"
       # this is needed to remove 'v' from Git tags to match SDLC ECR tags
       strip-image-suffix: "v"
     secrets: inherit

.github/workflows/legacy-system-tests-nightly.yml

@@ -354,10 +354,10 @@ jobs:
           path: devenv/tests/${{ matrix.tests_dir }}/logs
           retention-days: 3
 
-      - name: Upload DF1 soak alloc.pprof
+      - name: Upload Go pprof
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: alloc.pprof
+          name: alloc.pprof-${{ job.check_run_id }}
           path: devenv/tests/${{ matrix.tests_dir }}/alloc.pprof
           retention-days: 7

.github/workflows/post-build.yml

@@ -41,7 +41,7 @@ jobs:
             exclude-refs: "ccip"
             working-directory: "devenv"
             logs-directory: "./devenv/tests/ocr2/logs"
-            ctf-log-level: "info"
+            ctf-log-level: "debug"
             strip-image-suffix: "v"
         secrets: inherit
 
@@ -61,6 +61,6 @@ jobs:
             exclude-refs: "ccip"
             working-directory: "./system-tests/tests"
             logs-directory: "./system-tests/tests/smoke/cre/logs"
-            ctf-log-level: "info"
+            ctf-log-level: "debug"
             strip-image-suffix: "v"
         secrets: inherit