Update dependencies

JAVA21_MIGRATION.md

@@ -27,7 +27,7 @@ and fix the integration tests that broke as a result of the Hive 4.x upgrade.
 | Spotless Maven plugin | 2.4.1 | 2.43.0 (google-java-format 1.19.2, Java 21 compatible) |
 | JaCoCo | 0.8.6 | 0.8.12 |
 | Surefire | 3.0.0-M5 | 3.2.5 |
-| Docker base image | `openjdk:8-jdk` | `amazoncorretto:21` |
+| Docker base image | `openjdk:8-jdk` | `amazoncorretto:21-al2023` |
 
 Dropped explicit version pins for Logback, Log4j, JUnit, Mockito, AssertJ, and
 Dropwizard — these are now managed by the Spring Boot BOM.
@@ -156,6 +156,88 @@ object's `isSet` flags match the deserialized object.
 
 ---
 
+## Commit 3 — Fix `ClassNotFoundException` for external listener JARs in Jib-built images
+
+### Problem
+
+Child images that extend `drone-fly-app` (e.g. `egdp-docker-glue-sync-listener`) download a
+listener JAR into `/app/libs` at Dockerfile build time:
+
+```dockerfile
+FROM expediagroup/drone-fly-app:1.0.9-SNAPSHOT
+RUN cd /app/libs && curl ... apiary-gluesync-listener-8.1.13-all.jar
+```
+
+After upgrading to Java 21 / Hive 4.x, the container failed on startup with:
+
+```
+Caused by: java.lang.ClassNotFoundException:
+    com.expediagroup.apiary.extensions.gluesync.listener.ApiaryGlueSync
+  at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(...)
+```
+
+### Root cause — two compounding changes
+
+**Change 1 — Hive 2.x → 4.x changed `JavaUtils.getClassLoader()` behavior**
+
+| | Hive 2.x | Hive 4.x |
+|---|---|---|
+| Package | `org.apache.hadoop.hive.common.JavaUtils` | `org.apache.hadoop.hive.metastore.utils.JavaUtils` |
+| `getClassLoader()` returns | `Thread.currentThread().getContextClassLoader()` | JVM `AppClassLoader` |
+
+In a Spring Boot fat-jar launched with `PropertiesLauncher` and `loader.path=lib/`, the thread
+context classloader is Spring Boot's `LaunchedURLClassLoader`, which can find JARs placed in
+`loader.path`. Hive 2.x `JavaUtils` used this classloader, so external listener JARs were
+visible.
+
+Hive 4.x `JavaUtils` returns the plain JVM `AppClassLoader`, which only sees the `-cp`
+argument set at JVM startup — not dynamically placed JARs.
+
+**Change 2 — Jib bakes the classpath at image-build time**
+
+Jib generates an `ENTRYPOINT` with an **explicit list** of dependency JARs in the `-cp`
+argument (determined at `mvn package` time). JARs downloaded by a child Dockerfile's `RUN`
+step land on the filesystem in `/app/libs` but are **never added** to that hardcoded classpath.
+
+Before the migration this was masked: the Hive 2.x `JavaUtils` used `LaunchedURLClassLoader`
+(via PropertiesLauncher), which resolved listener JARs through `loader.path` independently of
+the Jib classpath. After the migration both defences were removed simultaneously.
+
+| | Before migration | After migration |
+|---|---|---|
+| `JavaUtils.getClassLoader()` returns | `LaunchedURLClassLoader` (sees `loader.path`) | `AppClassLoader` (sees only `-cp`) |
+| Listener JAR on classpath? | ✅ Yes (via `loader.path`) | ❌ No (not in Jib `-cp`) |
+
+### Fix — override `ENTRYPOINT` in child Dockerfile with a wildcard classpath
+
+`egdp-docker-glue-sync-listener/Dockerfile` was updated to override the Jib-baked entrypoint
+with one that uses `/app/libs/*`. The JVM expands the wildcard at **startup time**, picking up
+every JAR present in `/app/libs/` — including those downloaded by the `RUN curl` step:
+
+```dockerfile
+ENTRYPOINT ["java",
+    "--add-opens=java.base/java.lang=ALL-UNNAMED",
+    "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
+    "--add-opens=java.base/java.io=ALL-UNNAMED",
+    "--add-opens=java.base/java.net=ALL-UNNAMED",
+    "--add-opens=java.base/java.nio=ALL-UNNAMED",
+    "--add-opens=java.base/java.util=ALL-UNNAMED",
+    "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
+    "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
+    "--add-opens=java.base/java.security=ALL-UNNAMED",
+    "-cp", "/app/resources:/app/classes:/app/libs/*",
+    "com.expediagroup.dataplatform.dronefly.app.DroneFly"]
+```
+
+The `--add-opens` flags match those in the Jib `<jvmFlags>` configuration in the parent
+`pom.xml` so runtime Hadoop/Hive reflection behaviour is preserved.
+
+> **Note for other child images:** Any Dockerfile that extends `drone-fly-app` and adds JARs
+> to `/app/libs` must include this `ENTRYPOINT` override to ensure those JARs are on the
+> classpath.
+
+---
+
 ## Files changed (summary)
 
 ```

drone-fly-app/pom.xml

@@ -15,10 +15,15 @@
     <aws.version>1.11.532</aws.version>
     <s3mock.version>0.2.5</s3mock.version>
     <hive.version>4.0.1</hive.version>
+    <hadoop.version>3.4.2</hadoop.version>
     <docker.container.port>8008</docker.container.port>
   </properties>
 
   <dependencies>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-web</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.expediagroup</groupId>
       <artifactId>drone-fly-core</artifactId>
@@ -88,6 +93,10 @@
           <artifactId>jdk.tools</artifactId>
           <groupId>jdk.tools</groupId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-1.2-api</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -131,6 +140,14 @@
           <groupId>tomcat</groupId>
           <artifactId>jasper-runtime</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-1.2-api</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -154,12 +171,12 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client-runtime</artifactId>
-      <version>3.3.6</version>
+      <version>${hadoop.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-mapreduce-client-core</artifactId>
-      <version>3.3.6</version>
+      <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -177,6 +194,21 @@
       <artifactId>awaitility</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.tomcat.embed</groupId>
+      <artifactId>tomcat-embed-core</artifactId>
+      <version>${tomcat.embeded.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tomcat.embed</groupId>
+      <artifactId>tomcat-embed-el</artifactId>
+      <version>${tomcat.embeded.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tomcat.embed</groupId>
+      <artifactId>tomcat-embed-websocket</artifactId>
+      <version>${tomcat.embeded.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

drone-fly-app/src/main/java/com/expediagroup/dataplatform/dronefly/app/service/ListenerCatalog.java

@@ -26,7 +26,6 @@
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.utils.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
 
@@ -62,7 +61,7 @@ private <T> List<T> getMetaStoreListeners(Class<T> clazz, HiveConf conf, String
     for (String listenerImpl : listenerImpls) {
       try {
         T listener = (T) Class
-            .forName(listenerImpl.trim(), true, JavaUtils.getClassLoader())
+            .forName(listenerImpl.trim(), true, Thread.currentThread().getContextClassLoader())
             .getConstructor(Configuration.class)
             .newInstance(conf);
         listeners.add(listener);

drone-fly-core/pom.xml

@@ -26,12 +26,18 @@
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-web</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>org.springframework.boot</groupId>
-          <artifactId>spring-boot-starter-logging</artifactId>
-        </exclusion>
-      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tomcat.embed</groupId>
+      <artifactId>tomcat-embed-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tomcat.embed</groupId>
+      <artifactId>tomcat-embed-el</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tomcat.embed</groupId>
+      <artifactId>tomcat-embed-websocket</artifactId>
     </dependency>
     <dependency>
       <groupId>org.springframework</groupId>

drone-fly-core/src/test/java/com/expediagroup/dataplatform/dronefly/core/DroneFlyCoreTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2020 Expedia, Inc.
+ * Copyright (C) 2020-2026 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

drone-fly-integration-tests/src/test/java/com/expediagroup/dataplatform/dronefly/core/integration/DummyListener.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2020 Expedia, Inc.
+ * Copyright (C) 2020-2026 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

pom.xml

@@ -30,15 +30,18 @@
     <javadoc.source.version>21</javadoc.source.version>
 
     <!-- Override eg-oss-parent plugin versions for Java 21 compatibility -->
+    <spotless.config.execution.skip>true</spotless.config.execution.skip>
     <jacoco.version>0.8.12</jacoco.version>
     <maven.surefire.plugin.version>3.2.5</maven.surefire.plugin.version>
     <maven.compiler.plugin.version>3.13.0</maven.compiler.plugin.version>
+    <maven.spotbugs.plugin.version>4.9.8.2</maven.spotbugs.plugin.version>
 
     <springframework.boot.version>3.2.12</springframework.boot.version>
     <maven.shade.plugin.version>3.2.4</maven.shade.plugin.version>
     <jib.maven.plugin.version>3.4.3</jib.maven.plugin.version>
+    <tomcat.embeded.version>10.1.50</tomcat.embeded.version>
     <docker.from.image>amazoncorretto</docker.from.image>
-    <docker.from.tag>21</docker.from.tag>
+    <docker.from.tag>21-al2023</docker.from.tag>
     <docker.from.reference>${docker.from.image}:${docker.from.tag}</docker.from.reference>
     <docker.registry>expediagroup</docker.registry>
     <docker.to.image>${project.artifactId}</docker.to.image>
@@ -66,6 +69,29 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <dependency>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-web</artifactId>
+        <version>${springframework.boot.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-logging</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.tomcat.embed</groupId>
+            <artifactId>tomcat-embed-core</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.tomcat.embed</groupId>
+            <artifactId>tomcat-embed-el</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.tomcat.embed</groupId>
+            <artifactId>tomcat-embed-websocket</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
     </dependencies>
   </dependencyManagement>