Replace uber-jar with Jib layered Docker build

.circleci/config.yml

@@ -308,11 +308,11 @@ jobs:
             echo "default-key $(gpg --list-keys --with-colons | grep pub | cut -d':' -f5)" >> ~/.gnupg/gpg.conf
             echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
       - run:
-          name: Build shaded JAR
+          name: Build and deploy Maven artifacts
           command: mvn -U clean -T1C -B deploy -DonlyJars ${CIRCLE_TAG:+-Prelease} -Deasyjacoco.skip=true
 
   build-images:
-    # use a full VM so we can run Docker / Buildx
+    # use a full VM so we can run Maven and Jib
     executor: default-machine
 
     steps:
@@ -334,53 +334,51 @@ jobs:
       - generate-settings
       - install-jdk
 
-      - run:
-          name: Build shaded JARs
-          command: mvn -U -T1C -B install -DonlyJars
-
-      - determine-version
-      - compute-base-tag
-      - setup-buildx:
-          builder_name: "sqrl-builder"
-
-      - run:
-          name: Pull base image
-          command: |
-            docker pull datasqrl/base-sqrl:$BASE_TAG
-
       - run:
           name: Login to registries
           command: |
             echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_USER" --password-stdin
             echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
 
       - run:
-          name: Build + push Docker images
+          name: Build and push Docker images using Jib
           command: |
             set -x
 
-            build_and_push() {
-              local repo="$1" ; shift
-              local dir="$1"  ; shift
-
-              docker buildx build \
-                --platform linux/amd64,linux/arm64 \
-                --push \
-                --build-arg BASE_TAG=$BASE_TAG \
-                -t $REGISTRY/$repo:$TAG_SUFFIX \
-                -f $dir/Dockerfile \
-                $dir
-
-              # tag :latest on release builds
-              if [ "$LATEST_TAG" = "true" ]; then
-                docker buildx imagetools create \
-                  -t $REGISTRY/$repo:latest \
-                  $REGISTRY/$repo:$TAG_SUFFIX
-              fi
-            }
-
-            build_and_push cmd                 sqrl-cli
-            build_and_push sqrl-server         sqrl-server/sqrl-server-vertx
+            # Build and push CLI image
+            mvn -U -T1C -B package jib:build \
+              -pl sqrl-cli \
+              -am \
+              -Ddocker.image.tag=$TAG_SUFFIX \
+              -Djib.to.image=$REGISTRY/cmd:$TAG_SUFFIX \
+              -Deasyjacoco.skip=true
+
+            # Build and push server image  
+            mvn -U -T1C -B package jib:build \
+              -pl sqrl-server/sqrl-server-vertx \
+              -am \
+              -Ddocker.image.tag=$TAG_SUFFIX \
+              -Djib.to.image=$REGISTRY/sqrl-server:$TAG_SUFFIX \
+              -Deasyjacoco.skip=true
+
+            # Tag :latest on release builds
+            if [ "$LATEST_TAG" = "true" ]; then
+              # Re-tag and push CLI as latest
+              mvn jib:build \
+                -pl sqrl-cli \
+                -Ddocker.image.tag=latest \
+                -Djib.to.image=$REGISTRY/cmd:latest \
+                -Djib.from.image=$REGISTRY/cmd:$TAG_SUFFIX \
+                -Deasyjacoco.skip=true
+
+              # Re-tag and push server as latest  
+              mvn jib:build \
+                -pl sqrl-server/sqrl-server-vertx \
+                -Ddocker.image.tag=latest \
+                -Djib.to.image=$REGISTRY/sqrl-server:latest \
+                -Djib.from.image=$REGISTRY/sqrl-server:$TAG_SUFFIX \
+                -Deasyjacoco.skip=true
+            fi
 
   docker-build-base-image:
     machine:

pom.xml

@@ -95,6 +95,7 @@
     <maven.compiler.source>17</maven.compiler.source>
     <maven.compiler.target>17</maven.compiler.target>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <argLine></argLine>
 
     <!-- Flink runner props -->
     <flinkrunner.version>0.8.0</flinkrunner.version>
@@ -103,6 +104,7 @@
     <!-- Params -->
     <docker.image.tag>local</docker.image.tag>
     <snapshots.update>false</snapshots.update>
+    <skipDockerBuild>false</skipDockerBuild>
 
     <!-- Dependency versions -->
     <mockito.version>4.8.0</mockito.version>
@@ -170,6 +172,7 @@
     <zookeeper.version>3.9.4</zookeeper.version>
     <snakeyaml.version>2.5</snakeyaml.version>
     <aws-sdk-bom.version>2.33.4</aws-sdk-bom.version>
+    <aws-java-sdk.version>1.12.784</aws-java-sdk.version>
     <openai-gpt3-java.version>0.18.2</openai-gpt3-java.version>
     <flink-client.version>1.1.4</flink-client.version>
     <mcp-sdk.version>0.11.3</mcp-sdk.version>
@@ -183,6 +186,7 @@
     <git-code-format-maven-plugin.version>5.3</git-code-format-maven-plugin.version>
     <sortpom.plugin.version>4.0.0</sortpom.plugin.version>
     <dockerfile-maven-plugin.version>1.4.13</dockerfile-maven-plugin.version>
+    <jib-maven-plugin.version>3.4.4</jib-maven-plugin.version>
     <easy-jacoco-maven-plugin.version>0.1.4</easy-jacoco-maven-plugin.version>
   </properties>
 
@@ -200,11 +204,6 @@
         <artifactId>sqrl-server-core</artifactId>
         <version>${project.version}</version>
       </dependency>
-      <dependency>
-        <groupId>com.datasqrl</groupId>
-        <artifactId>sqrl-server-vertx-base</artifactId>
-        <version>${project.version}</version>
-      </dependency>
       <dependency>
         <groupId>com.datasqrl</groupId>
         <artifactId>sqrl-server-vertx</artifactId>
@@ -467,9 +466,9 @@
         <version>${flink.version}</version>
       </dependency>
       <dependency>
-        <groupId>org.apache.flink</groupId>
+        <groupId>com.datasqrl.flink</groupId>
         <artifactId>flink-s3-fs-hadoop</artifactId>
-        <version>${flink.version}</version>
+        <version>1.19.3-13</version>
       </dependency>
       <dependency>
         <groupId>org.apache.flink</groupId>
@@ -858,6 +857,54 @@
             <pullNewerImage>false</pullNewerImage>
           </configuration>
         </plugin>
+        <plugin>
+          <groupId>com.google.cloud.tools</groupId>
+          <artifactId>jib-maven-plugin</artifactId>
+          <version>${jib-maven-plugin.version}</version>
+          <configuration>
+            <platforms>
+              <platform>
+                <architecture>amd64</architecture>
+                <os>linux</os>
+              </platform>
+              <platform>
+                <architecture>arm64</architecture>
+                <os>linux</os>
+              </platform>
+            </platforms>
+            <container>
+              <appRoot>/opt/sqrl</appRoot>
+            </container>
+            <extraDirectories>
+              <paths>
+                <path>
+                  <from>${project.basedir}/src/main/jib</from>
+                  <includes>
+                    <include>**/*</include>
+                  </includes>
+                </path>
+              </paths>
+              <permissions>
+                <permission>
+                  <file>/opt/sqrl/entrypoint.sh</file>
+                  <mode>755</mode>
+                </permission>
+              </permissions>
+            </extraDirectories>
+          </configuration>
+          <executions>
+            <execution>
+              <id>build-docker-image</id>
+              <goals>
+                <goal>dockerBuild</goal>
+              </goals>
+              <phase>package</phase>
+              <configuration>
+                <skip>${skipDockerBuild}</skip>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
       </plugins>
     </pluginManagement>
     <plugins>
@@ -916,7 +963,28 @@
                     <exclude>org.apache.logging.log4j:log4j-slf4j2-impl</exclude>
                     <exclude>org.slf4j:slf4j-reload4j</exclude>
                   </excludes>
-                  <message>SLF4J 2.x is not allowed. Use log4j-slf4j-impl instead of log4j-slf4j2-impl</message>
+                  <message>SLF4J 2.x is not allowed. Use log4j-slf4j-impl instead of log4j-slf4j2-impl.</message>
+                </bannedDependencies>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>enforce-flink-s3-version</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <bannedDependencies>
+                  <searchTransitive>true</searchTransitive>
+                  <includes>
+                    <include>*:*</include>
+                  </includes>
+                  <excludes>
+                    <!-- Ban original flink-s3-fs-hadoop to avoid shading conflicts -->
+                    <exclude>org.apache.flink:flink-s3-fs-hadoop</exclude>
+                  </excludes>
+                  <message>Use com.datasqrl.flink:flink-s3-fs-hadoop instead of org.apache.flink:flink-s3-fs-hadoop to avoid shading conflicts.</message>
                 </bannedDependencies>
               </rules>
             </configuration>
@@ -1391,7 +1459,7 @@
       </activation>
       <properties>
         <skipITs>true</skipITs>
-        <dockerfile.skip>true</dockerfile.skip>
+        <skipDockerBuild>true</skipDockerBuild>
         <maven.test.excludeProjects>sqrl-testing-container</maven.test.excludeProjects>
       </properties>
       <build>
@@ -1422,7 +1490,7 @@
         </property>
       </activation>
       <properties>
-        <dockerfile.skip>true</dockerfile.skip>
+        <skipDockerBuild>true</skipDockerBuild>
         <maven.test.excludeProjects>sqrl-testing-container</maven.test.excludeProjects>
       </properties>
       <build>
@@ -1478,7 +1546,7 @@
       </properties>
     </profile>
     <profile>
-      <!-- builds the shaded JARs, but not the images -->
+      <!-- builds the JARs and deploys to Maven repository, but skips Docker image builds -->
       <id>only-jars</id>
       <activation>
         <property>
@@ -1488,7 +1556,29 @@
       <properties>
         <skipTests>true</skipTests>
         <skipITs>true</skipITs>
-        <dockerfile.skip>true</dockerfile.skip>
+        <skipDockerBuild>true</skipDockerBuild>
+      </properties>
+    </profile>
+    <profile>
+      <id>skip-docker-build-onlyITs</id>
+      <activation>
+        <property>
+          <name>onlyITs</name>
+        </property>
+      </activation>
+      <properties>
+        <skipDockerBuild>true</skipDockerBuild>
+      </properties>
+    </profile>
+    <profile>
+      <id>skip-docker-build-onlyUTs</id>
+      <activation>
+        <property>
+          <name>onlyUTs</name>
+        </property>
+      </activation>
+      <properties>
+        <skipDockerBuild>true</skipDockerBuild>
       </properties>
     </profile>
   </profiles>

sqrl-cli/Dockerfile

@@ -19,7 +19,13 @@ FROM datasqrl/base-sqrl:${BASE_TAG}
 
 COPY postgres.conf /etc/postgresql/${POSTGRES_VERSION}/main/conf.d/postgres.conf
 COPY redpanda.yaml /etc/redpanda/redpanda.yaml
+
+# Copy dependencies first (this layer will be cached if dependencies don't change)
+COPY target/libs /opt/sqrl/libs
+
+# Copy the application jar (this layer changes more frequently)
 COPY target/sqrl-cli.jar /opt/sqrl/sqrl-cli.jar
+
 COPY entrypoint.sh /opt/sqrl/entrypoint.sh
 
 ENV PATH="/opt/jbang/bin:${PATH}"

sqrl-cli/entrypoint.sh

@@ -33,4 +33,5 @@ export BUILD_UID=$(stat -c '%u' /build)
 export BUILD_GID=$(stat -c '%g' /build)
 
 echo "Executing SQRL command: \"$1\" ..."
-exec java $SQRL_JVM_TOOL_OPTS $SQRL_JVM_ARGS -jar /opt/sqrl/sqrl-cli.jar "${@}"
+# Jib sets up the classpath automatically, so we can run the main class directly
+exec java $SQRL_JVM_TOOL_OPTS $SQRL_JVM_ARGS -cp "/opt/sqrl/libs/*:/opt/sqrl/classes" com.datasqrl.cli.DatasqrlCli "${@}"