PIC-5667 - postgres migration

.github/workflows/newman-build.yaml

@@ -0,0 +1,65 @@
+name: Build newman and publish artifacts
+
+on:
+  push:
+    branches: [ PIC-5667-postgres-migration ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Java 8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+
+      - name: Install Maven 3.6.3
+        uses: stCarolas/setup-maven@v4
+        with:
+          maven-version: 3.6.3
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build all modules
+        run: mvn clean package
+
+      - name: Install GitHub CLI
+        run: |
+          sudo apt update
+          sudo apt install -y gh
+
+      - name: Zip all target directories
+        run: |
+          mkdir -p build-zips
+          for dir in */target; do
+            parent=$(dirname "$dir")
+            zipfile="build-zips/${parent}.zip"
+            echo "Zipping $dir -> $zipfile"
+            (cd "$dir" && zip -r "../../$zipfile" .)
+          done
+
+      - name: Generate checksums
+        run: |
+          cd build-zips
+          sha256sum *.zip > checksums.sha256
+          cat checksums.sha256
+
+      - name: Create or update GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+
+          # Create release if not exists
+          gh release view "v$VERSION" >/dev/null 2>&1 || \
+            gh release create "v$VERSION" --title "v$VERSION" --notes "SHA256 checksums below:"
+
+          # Upload all files including checksums
+          for file in build-zips/*.zip build-zips/checksums.sha256; do
+            echo "Uploading $file"
+            gh release upload "v$VERSION" "$file" --clobber
+          done

docker/Dockerfile → docker/_Dockerfile

@@ -8,10 +8,10 @@ ARG gid=1001
 RUN apt-get update -y
 RUN apt-get install coreutils wget -y
 
-ENV MAVEN_VERSION 3.6.3
-ENV MAVEN_FILE apache-maven-${MAVEN_VERSION}-bin.tar.gz
-ENV NODE_VERSION 22.11.0
-ENV NODE_FILE node-v${NODE_VERSION}-linux-x64.tar.gz
+ENV MAVEN_VERSION=3.6.3
+ENV MAVEN_FILE=apache-maven-${MAVEN_VERSION}-bin.tar.gz
+ENV NODE_VERSION=22.11.0
+ENV NODE_FILE=node-v${NODE_VERSION}-linux-x64.tar.gz
 
 # Jenkins is run with user `newman`, uid = 1000
 # If you bind mount a volume from the host or a data container,
@@ -21,7 +21,7 @@ RUN groupadd -g ${gid} ${group} \
 
 # Install Java.
 RUN apt-get install openjdk-8-jdk -y
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
 
 # Install Maven
 RUN wget --no-verbose -O /tmp/${MAVEN_FILE} \
@@ -66,3 +66,5 @@ USER root
 RUN apt-get install sudo curl -y netbase
 
 USER ${user}
+
+CMD ["sh", "/newman/bin/newman-server.sh"]

docker/docker-compose.yml

@@ -0,0 +1,32 @@
+version: "3.8"
+
+services:
+  newman-server:
+    image: newman-server:1.0
+    container_name: newman-server
+    ports:
+      - "8080:8080"
+      - "8443:8443"
+    volumes:
+      - /home/newman/server:/newman
+    entrypoint: ["/newman/bin/newman-server.sh"]
+    networks:
+      - newman-network
+
+  newman-postgres:
+    image: postgres
+    container_name: newman-postgres
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_DB: newman-db
+      POSTGRES_USER: admin
+      POSTGRES_PASSWORD: password
+    volumes:
+      - /home/newman/postgres_data:/var/lib/postgresql/data
+    networks:
+      - newman-network
+
+networks:
+  newman-network:
+    driver: bridge

files/branch_list.txt

@@ -0,0 +1 @@
+master

files/newman-submitter-loop.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+while true; do
+        source submitter-env.sh
+
+        # take branches from file
+        branch_list=`cat ${BRANCH_FILE_PATH}`
+        IFS=',' read -a branch_array <<< "${branch_list}"
+        echo "starting loop over branches ..."
+
+        # loop over all branches
+        for branch in "${branch_array[@]}"
+        do
+                echo "Start submitting XAP jobs. Date is [`date`]"
+                # current hour
+                HOURS=$(date +%H)
+
+                if [ $HOURS -ge 20 -a $HOURS -le 23 ]; then
+                        echo "running in nightly mode, will trigger new jobs even if no changes where made, date is `date`"
+                        export NEWMAN_MODE="NIGHTLY"
+                        export NEWMAN_BUILD_TAGS="XAP,DOTNET"
+                else
+                        echo "running in daily mode, will trigger new jobs only if changes in build were made, date is `date`"
+                        export NEWMAN_MODE="DAILY"
+                        export NEWMAN_BUILD_TAGS="XAP"
+                fi
+
+                export NEWMAN_BUILD_BRANCH=${branch}
+
+                echo "NEWMAN_BUILD_BRANCH=${NEWMAN_BUILD_BRANCH}"
+                echo "NEWMAN_BUILD_TAGS=${NEWMAN_BUILD_TAGS}"
+                echo "NEWMAN_MODE=${NEWMAN_MODE}"
+
+                #checking future job
+                java -jar newman-submitter-2.0.jar
+                HAS_FUTURE_JOBS=$?
+                echo "FINISHED SUBMIT LOOP!! [HAS_FUTURE_JOBS=${HAS_FUTURE_JOBS}]. Date is [`date`]"
+                sleep 60
+        done
+done

files/submitter-env.ini

@@ -0,0 +1,41 @@
+[main]
+DISCONNECT = 59f25af5b3859424cac590ad
+MANAGER = 5c88ea4bb38594355214b205
+XAP_CORE = 59f25d19b3859424cac590bf
+OFF_HEAP = 59f25d5bb3859424cac590c2
+ROCKS_DB = 59f25d5ab3859424cac590c1
+JETTY9 = 59f25af6b3859424cac590b1
+WAN = 59f25af7b3859424cac590b9
+I9E_SGTEST = 59f59592b3859424cac690fa
+MONGO_DB = 59f25af7b3859424cac590b4
+ESM_SECURITY = 59f25af7b3859424cac590b7
+SERVICE_GRID = 59f25af7b3859424cac590b5
+SECURITY = 59f25af7b3859424cac590b8
+ESM = 59f25af7b3859424cac590b6
+DOTNET = 59f25af6b3859424cac590ae
+WEB_UI = 59f25e0f6d98c14e1daa2008
+MANAGER_SECURITY = 58907cbb29f67f0c0af7eee1
+INSIGHTEDGE_PREMIUM = 59f25af6b3859424cac590b0
+MX_OFF_HEAP = 5a96b14268c479caafb1162f
+MX_PMEM = 5be44547b385947afe427786
+FOOTPRINT = 5b1ce25cb385946642f31fac
+I9E_SGTEST_CLI = 5b4c8771b38594390d56cccf
+DEMOTE = 5b6c24e7b385941b3b8ce4bf
+DEMOTE_ROCKSDB = 5b82bae131f99308478e101b
+AX_SPARK = 5d2b2d1c4cedfd000ca050d8
+K8S_I9E = 5c88efc7b38594355214b36b
+K8S_XAP = 5c88efb2b38594355214b36a
+OPS_UI = 5ef32df04cedfd00099271ae
+OPS_UI_SECURED = 5ef327854cedfd000992719e
+ELASTICGRID = 5f9fe92cf10dba603d82bb9e
+PERFORMANCE = 59f25e0f6d98c14e1daa5000
+TIERED_STORAGE_COLD_ONLY = 61dae61c4cedfd000c4bf3b2
+DAILY_SUITES_LIST = ${XAP_CORE},${SERVICE_GRID},${MANAGER},${ROCKS_DB},${OFF_HEAP},${DISCONNECT},${ESM_SECURITY},${WAN},${MONGO_DB},${ESM},${SECURITY},${JETTY9},${WEB_UI},${MANAGER_SECURITY},${MX_OFF_HEAP},${FOOTPRINT},${DEMOTE},${OPS_UI},${OPS_UI_SECURED},${TIERED_STORAGE_COLD_ONLY}
+NIGHTLY_SUITES_LIST = ${XAP_CORE},${SERVICE_GRID},${MANAGER},${ROCKS_DB},${DISCONNECT},${WAN},${MONGO_DB},${SECURITY},${JETTY9},${WEB_UI},${MANAGER_SECURITY},${MX_OFF_HEAP},${OFF_HEAP},${FOOTPRINT},${DEMOTE},${DEMOTE_ROCKSDB},${OPS_UI},${OPS_UI_SECURED},${PERFORMANCE},${K8S_XAP},${TIERED_STORAGE_COLD_ONLY},${DOTNET}
+LAST_DAILY_SUITE = 5f9fe92cf10dba603d82bb9e
+LAST_NIGHTLY_RUN = 2025/05/27
+JOB_CONFIG_DEFAULT = OPENJDK_17
+JOB_CONFIG_SCHEDULED = {"5":"OPENJDK_21"}
+AGENT_GROUPS_DEFAULT = onprem,onprem-dotnet
+DAILY_JOB_PRIORITY_DEFAULT = 0
+NIGHTLY_JOB_PRIORITY_DEFAULT = 3

files/submitter.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+source submitter-env.sh
+
+# take branches from file
+branch_list=`cat ${BRANCH_FILE_PATH}`
+IFS=',' read -a branch_array <<< "${branch_list}"
+echo "starting loop over branches ..."
+
+# loop over all branches
+for branch in "${branch_array[@]}"
+do
+        echo "Start submitting XAP jobs. Date is [`date`]"
+        # current hour
+        HOURS=$(date +%H)
+
+        # check if nightly or daily mode - every branch
+#                if [ $HOURS -ge 20 -a $HOURS -le 23 ]; then
+        if [ $HOURS -ge 20 -a $HOURS -le 23 ]; then
+                echo "running in nightly mode, will trigger new jobs even if no changes where made, date is `date`"
+                export NEWMAN_MODE="NIGHTLY"
+                export NEWMAN_BUILD_TAGS="XAP,DOTNET"
+        else
+                echo "running in daily mode, will trigger new jobs only if changes in build were made, date is `date`"
+                export NEWMAN_MODE="DAILY"
+                export NEWMAN_BUILD_TAGS="XAP"
+        fi
+
+        export NEWMAN_BUILD_BRANCH=${branch}
+
+        echo "NEWMAN_BUILD_BRANCH=${NEWMAN_BUILD_BRANCH}"
+        echo "NEWMAN_BUILD_TAGS=${NEWMAN_BUILD_TAGS}"
+        echo "NEWMAN_MODE=${NEWMAN_MODE}"
+
+        #checking future job
+        java -jar newman-submitter-2.0.jar
+        HAS_FUTURE_JOBS=$?
+        echo "FINISHED SUBMIT LOOP!! [HAS_FUTURE_JOBS=${HAS_FUTURE_JOBS}]. Date is [`date`]"
+done

newman-agent/src/main/java/com/gigaspaces/newman/JobExecutor.java

@@ -1,8 +1,8 @@
 package com.gigaspaces.newman;
 
-import com.gigaspaces.newman.beans.Job;
-import com.gigaspaces.newman.beans.Suite;
-import com.gigaspaces.newman.beans.Test;
+import com.gigaspaces.newman.entities.Job;
+import com.gigaspaces.newman.entities.Suite;
+import com.gigaspaces.newman.entities.Test;
 import com.gigaspaces.newman.utils.FileUtils;
 import com.gigaspaces.newman.utils.ProcessResult;
 import com.gigaspaces.newman.utils.ProcessUtils;
@@ -12,11 +12,13 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
 
 import static com.gigaspaces.newman.utils.FileUtils.*;
+import static com.gigaspaces.newman.utils.StringUtils.getNonEmptySystemProperty;
 
 public class JobExecutor {
     private static final Logger logger = LoggerFactory.getLogger(JobExecutor.class);
@@ -25,6 +27,7 @@ public class JobExecutor {
     private final Job job;
     private final Path jobFolder;
     private final Path newmanLogFolder;
+    private final String proxy = getNonEmptySystemProperty("resources.proxy.url", null);
 
     public JobExecutor(Job job, String basePath) {
         this.job = job;
@@ -50,8 +53,9 @@ public boolean setup() {
             logger.info("Downloading {} resources into {}...", job.getBuild().getResources().size(), resourcesFolder);
             validateUris(job.getBuild().getResources());
             for (URI resource : job.getBuild().getResources()) {
-                logger.info("Downloading {}...", resource);
-                download(resource.toURL(), resourcesFolder);
+                URI proxiedResource = wrapWithProxy(resource);
+                logger.info("Downloading {}...", proxiedResource);
+                download(proxiedResource.toURL(), resourcesFolder);
             }
 
             logger.info("Extracting Newman Artifacts...");
@@ -88,6 +92,24 @@ public boolean setup() {
         }
     }
 
+    private URI wrapWithProxy(URI origPath) {
+        if (proxy == null) {
+            logger.info("No proxy has been found");
+            return origPath;
+        }
+
+        logger.info("Using proxy - {}", proxy);
+        String origPathStr = proxy.split(":")[0];
+        String proxyPathStr = proxy.split(":")[1];
+
+        try {
+            return new URI(origPath.toString().replace(origPathStr, proxyPathStr));
+        } catch (URISyntaxException e) {
+            logger.error("Failed to wrap proxy URI {} with {}: {}", origPathStr, proxyPathStr, e);
+            return origPath;
+        }
+    }
+
     private long overrideSetupTimeoutIfRequested() {
         Map<String, String> customVariablesFromSuite = getCustomVariablesFromSuite();
         String customTimeout = customVariablesFromSuite.get(Suite.CUSTOM_SETUP_TIMEOUT);
@@ -178,9 +200,16 @@ private Collection<String> wrapUniqueArgsWithQuotes(List<String> arguments) {
     }
 
     private void teardownTest(Test test) {
+        logger.info("Tearing down test {}", test.getName());
+
         Path testFolder = append(jobFolder, "test-" + test.getId());
         Path outputFile = append(append(testFolder, "output"), "end-" + test.getTestType() + ".log");
         Path teardownScript = append(jobFolder, "end-" + test.getTestType() + SCRIPT_SUFFIX);
+
+        logger.info("testFolder path: {}", testFolder);
+        logger.info("outputFile path: {}", outputFile);
+        logger.info("teardownScript path: {}", teardownScript);
+
         if (exists(teardownScript)) {
             logger.info("Executing teardown for test {}", test);
             try {