diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 0000000000000..432ae52c53050 --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +-Xss1500k \ No newline at end of file diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000000000..d58dfb70bab56 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/buildtools/pom.xml b/buildtools/pom.xml index b16ee33ddd396..720542b862259 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -53,6 +53,7 @@ 3.4.0 8.37 3.1.2 + 4.1.121.Final 4.2.3 32.0.0-jre 1.10.12 diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index e74cdec53d505..18ef30b523675 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -40,6 +40,19 @@ ${project.version} + + ${project.groupId} + pulsar-metadata + ${project.version} + + + + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + + ${project.groupId} pulsar-proxy diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index f0099bce47086..4594c13f455b1 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -322,7 +322,7 @@ The Apache Software License, Version 2.0 - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar - * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar + * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.9.0.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar * Gson - com.google.code.gson-gson-2.8.9.jar @@ -352,34 +352,36 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-compress-1.26.0.jar - org.apache.commons-commons-lang3-3.11.jar * Netty - - io.netty-netty-buffer-4.1.100.Final.jar - - io.netty-netty-codec-4.1.100.Final.jar - - io.netty-netty-codec-dns-4.1.100.Final.jar - - io.netty-netty-codec-http-4.1.100.Final.jar - - io.netty-netty-codec-http2-4.1.100.Final.jar - - io.netty-netty-codec-socks-4.1.100.Final.jar - - io.netty-netty-codec-haproxy-4.1.100.Final.jar - - io.netty-netty-common-4.1.100.Final.jar - - io.netty-netty-handler-4.1.100.Final.jar - - io.netty-netty-handler-proxy-4.1.100.Final.jar - - io.netty-netty-resolver-4.1.100.Final.jar - - io.netty-netty-resolver-dns-4.1.100.Final.jar - - io.netty-netty-transport-4.1.100.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.100.Final.jar - - io.netty-netty-transport-native-epoll-4.1.100.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.100.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.100.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar - - io.netty-netty-tcnative-classes-2.0.61.Final.jar - - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.25.Final.jar - - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar - - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar + - io.netty-netty-buffer-4.1.121.Final.jar + - io.netty-netty-codec-4.1.121.Final.jar + - io.netty-netty-codec-dns-4.1.121.Final.jar + - io.netty-netty-codec-http-4.1.121.Final.jar + - io.netty-netty-codec-http2-4.1.121.Final.jar + - io.netty-netty-codec-socks-4.1.121.Final.jar + - io.netty-netty-codec-haproxy-4.1.121.Final.jar + - io.netty-netty-common-4.1.121.Final.jar + - io.netty-netty-handler-4.1.121.Final.jar + - io.netty-netty-handler-proxy-4.1.121.Final.jar + - io.netty-netty-resolver-4.1.121.Final.jar + - io.netty-netty-resolver-dns-4.1.121.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.121.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.121.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.121.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.121.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.121.Final.jar + - io.netty-netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.1.121.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.70.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.70.Final.jar + - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar + - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar + - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar * Prometheus client - io.prometheus-simpleclient-0.5.0.jar - io.prometheus-simpleclient_common-0.5.0.jar @@ -473,31 +475,32 @@ The Apache Software License, Version 2.0 - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.6.0.jar - org.jetbrains-annotations-13.0.jar * gRPC - - io.grpc-grpc-all-1.45.1.jar - - io.grpc-grpc-auth-1.45.1.jar - - io.grpc-grpc-context-1.45.1.jar - - io.grpc-grpc-core-1.45.1.jar - - io.grpc-grpc-netty-1.45.1.jar - - io.grpc-grpc-protobuf-1.45.1.jar - - io.grpc-grpc-protobuf-lite-1.45.1.jar - - io.grpc-grpc-stub-1.45.1.jar - - io.grpc-grpc-alts-1.45.1.jar - - io.grpc-grpc-api-1.45.1.jar - - io.grpc-grpc-grpclb-1.45.1.jar - - io.grpc-grpc-netty-shaded-1.45.1.jar - - io.grpc-grpc-services-1.45.1.jar - - io.grpc-grpc-xds-1.45.1.jar - - io.grpc-grpc-rls-1.45.1.jar - - com.google.auto.service-auto-service-annotations-1.0.jar + - io.grpc-grpc-all-1.55.3.jar + - io.grpc-grpc-auth-1.55.3.jar + - io.grpc-grpc-context-1.55.3.jar + - io.grpc-grpc-core-1.55.3.jar + - io.grpc-grpc-protobuf-1.55.3.jar + - io.grpc-grpc-protobuf-lite-1.55.3.jar + - io.grpc-grpc-stub-1.55.3.jar + - io.grpc-grpc-alts-1.55.3.jar + - io.grpc-grpc-api-1.55.3.jar + - io.grpc-grpc-grpclb-1.55.3.jar + - io.grpc-grpc-netty-shaded-1.55.3.jar + - io.grpc-grpc-services-1.55.3.jar + - io.grpc-grpc-xds-1.55.3.jar + - io.grpc-grpc-rls-1.55.3.jar + - io.grpc-grpc-servlet-1.55.3.jar + - io.grpc-grpc-servlet-jakarta-1.55.3.jar + - io.grpc-grpc-util-1.60.0.jar * Perfmark - - io.perfmark-perfmark-api-0.19.0.jar + - io.perfmark-perfmark-api-0.26.0.jar * OpenCensus - io.opencensus-opencensus-api-0.28.0.jar - io.opencensus-opencensus-contrib-http-util-0.28.0.jar - io.opencensus-opencensus-proto-0.2.0.jar * Jodah - net.jodah-typetools-0.5.0.jar - - net.jodah-failsafe-2.4.4.jar + - dev.failsafe-failsafe-3.3.2.jar * Byte Buddy - net.bytebuddy-byte-buddy-1.14.12.jar * zt-zip @@ -544,10 +547,8 @@ The Apache Software License, Version 2.0 - com.google.http-client-google-http-client-gson-1.41.0.jar - com.google.http-client-google-http-client-1.41.0.jar - com.google.auto.value-auto-value-annotations-1.9.jar - - com.google.re2j-re2j-1.5.jar - * Jetcd - - io.etcd-jetcd-common-0.5.11.jar - - io.etcd-jetcd-core-0.5.11.jar + - com.google.re2j-re2j-1.6.jar + * Jetcd - shaded * IPAddress - com.github.seancfoley-ipaddress-5.3.3.jar * RoaringBitmap diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml new file mode 100644 index 0000000000000..54d1d82e29c2f --- /dev/null +++ b/jetcd-core-shaded/pom.xml @@ -0,0 +1,187 @@ + + + + 4.0.0 + + org.apache.pulsar + pulsar + 2.10.7-SNAPSHOT + + + jetcd-core-shaded + Apache Pulsar :: jetcd-core shaded + + + + io.etcd + jetcd-core + + + io.grpc + grpc-netty + + + io.netty + * + + + + + io.grpc + grpc-netty-shaded + + + + dev.failsafe + failsafe + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + io.grpc + grpc-grpclb + + + io.grpc + grpc-util + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + true + false + + + io.etcd:* + io.vertx:* + + + + + + io.vertx + org.apache.pulsar.jetcd.shaded.io.vertx + + + + io.grpc.netty + io.grpc.netty.shaded.io.grpc.netty + + + + io.netty + io.grpc.netty.shaded.io.netty + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml + + + + + + + + META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml + ${project.basedir}/dependency-reduced-pom.xml + + + + true + shaded + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-shaded-jar + package + + attach-artifact + + + + + ${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar + jar + shaded + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + ${maven-antrun-plugin.version} + + + unpack-shaded-jar + package + + run + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 312a615d00bca..e45f1982f4652 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,8 @@ flexible messaging model and an intuitive client API. 1.1.10.5 4.1.12.1 5.7.1 - 4.1.100.Final + 4.1.121.Final + 0.0.26.Final 9.4.54.v20240208 2.5.2 2.34 @@ -149,9 +150,9 @@ flexible messaging model and an intuitive client API. 1.17 3.19.6 ${protobuf3.version} - 1.45.1 + 1.55.3 1.41.0 - 0.19.0 + 0.26.0 ${grpc.version} 2.8.9 1.2.1 @@ -226,12 +227,15 @@ flexible messaging model and an intuitive client API. 9.1.6 5.3.19 4.5.13 - 0.5.11 + 0.7.7 2.0 1.10.12 5.3.3 2.0.6 1.0.3 + 3.3.2 + + 3.3.2 3.6.0 @@ -272,6 +276,7 @@ flexible messaging model and an intuitive client API. 2.3.0 3.4.0 3.0.0 + 3.6.0 1.0.0 1.5.0 3.1.2 @@ -289,9 +294,7 @@ flexible messaging model and an intuitive client API. 9.1.0 0.9.44 6.4.0 - - - rename-netty-native-libs.sh + 3.33.0 @@ -351,6 +354,19 @@ flexible messaging model and an intuitive client API. ${mockito.version} + + dev.failsafe + failsafe + ${failsafe.version} + + + + io.grpc + grpc-util + + 1.60.0 + + org.mockito mockito-inline @@ -363,6 +379,12 @@ flexible messaging model and an intuitive client API. ${powermock.version} + + dev.failsafe + failsafe + ${failsafe.version} + + org.apache.zookeeper zookeeper @@ -536,6 +558,14 @@ flexible messaging model and an intuitive client API. com.squareup.okio okio + + jose4j + org.bitbucket.b_c + + + io.grpc + grpc-netty + @@ -572,6 +602,13 @@ flexible messaging model and an intuitive client API. + + io.grpc + grpc-util + + 1.60.0 + + org.apache.bookkeeper bookkeeper-common @@ -654,6 +691,29 @@ flexible messaging model and an intuitive client API. ${jcommander.version} + + io.netty.incubator + netty-incubator-transport-classes-io_uring + ${netty-iouring.version} + + + io.netty.incubator + netty-incubator-transport-native-io_uring + ${netty-iouring.version} + + + io.netty.incubator + netty-incubator-transport-native-io_uring + ${netty-iouring.version} + linux-x86_64 + + + io.netty.incubator + netty-incubator-transport-native-io_uring + ${netty-iouring.version} + linux-aarch_64 + + com.google.guava guava @@ -925,12 +985,51 @@ flexible messaging model and an intuitive client API. io.etcd jetcd-core ${jetcd.version} + + + io.grpc + grpc-netty + + - io.etcd jetcd-test ${jetcd.version} + + + io.grpc + grpc-netty + + + io.etcd + jetcd-core + + + io.etcd + jetcd-api + + + io.vertx + * + + + + + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + + + io.etcd + * + + + io.vertx + * + + @@ -1024,6 +1123,10 @@ flexible messaging model and an intuitive client API. com.squareup.okio okio + + io.grpc + grpc-netty + @@ -1855,6 +1958,11 @@ flexible messaging model and an intuitive client API. + + org.codehaus.mojo + build-helper-maven-plugin + ${build-helper-maven-plugin.version} + org.apache.maven.plugins maven-compiler-plugin @@ -2152,10 +2260,6 @@ flexible messaging model and an intuitive client API. Windows - - rename-netty-native-libs.cmd - - @@ -2177,7 +2281,6 @@ flexible messaging model and an intuitive client API. pulsar-common pulsar-broker-common pulsar-broker - pulsar-broker-shaded pulsar-client-api pulsar-client pulsar-client-shaded @@ -2218,6 +2321,7 @@ flexible messaging model and an intuitive client API. pulsar-client-messagecrypto-bc pulsar-metadata + jetcd-core-shaded jclouds-shaded @@ -2274,7 +2378,7 @@ flexible messaging model and an intuitive client API. distribution pulsar-metadata - + jetcd-core-shaded pulsar-package-management diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml deleted file mode 100644 index 873b7e392ce3f..0000000000000 --- a/pulsar-broker-shaded/pom.xml +++ /dev/null @@ -1,377 +0,0 @@ - - - - 4.0.0 - - - org.apache.pulsar - pulsar - 2.10.7-SNAPSHOT - .. - - - pulsar-broker-shaded - jar - Pulsar Shaded Broker - - - - ${project.groupId} - pulsar-broker - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - org.asynchttpclient - async-http-client - ${asynchttpclient.version} - jar - true - org/asynchttpclient/config/ahc-default.properties - ${project.build.directory}/classes - - - - - - - - - maven-antrun-plugin - - - shade-ahc-properties - prepare-package - - run - - - - - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - true - true - false - - - - org.apache.pulsar:* - org.apache.bookkeeper*:* - commons-*:* - org.apache.commons:* - org.asynchttpclient:* - org.reactivestreams:reactive-streams - com.typesafe.netty:netty-reactive-streams - org.javassist:javassist - com.google.*:* - com.fasterxml.jackson.*:* - io.netty:* - org.apache.pulsar:pulsar-common - org.apache.bookkeeper:circe-checksum - com.yahoo.datasketches:sketches-core - - javax.ws.rs:* - javax.websocket:* - org.glassfish.hk2*:* - org.eclipse.jetty*:* - net.java.dev.jna:* - com.carrotsearch:* - io.prometheus:* - io.perfmark:* - com.github.ben-manes.caffeine:* - org.glassfish.jersey.*:* - org.rocksdb:* - org.apache.bookkeeper:* - org.apache.zookeeper:* - jline:* - javax.servlet:* - - com.beust:* - io.swagger:* - joda-time:* - org.yaml:snakeyaml - org.hdrhistogram:* - com.github.zafarkhaja:java-semver - org.apache.avro:avro - - com.thoughtworks.paranamer:paranamer - org.apache.commons:commons-compress - org.tukaani:xz - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - org.asynchttpclient - org.apache.pulsar.shade.org.asynchttpclient - - - org.apache.commons - org.apache.pulsar.shade.org.apache.commons - - - com.google - org.apache.pulsar.shade.com.google - - - com.fasterxml.jackson - org.apache.pulsar.shade.com.fasterxml.jackson - - - io.netty - org.apache.pulsar.shade.io.netty - - - org.apache.pulsar.common - org.apache.pulsar.shade.org.apache.pulsar.common - - - org.apache.pulsar.policies - org.apache.pulsar.shade.org.apache.pulsar.policies - - - org.apache.pulsar.checksum - org.apache.pulsar.shade.org.apache.pulsar.checksum - - - com.yahoo.datasketches - org.apache.pulsar.shade.com.yahoo.datasketches - - - com.yahoo.sketches - org.apache.pulsar.shade.com.yahoo.sketches - - - - org.apache.zookeeper - org.apache.pulsar.shade.org.apache.zookeeper - - - org.apache.jute - org.apache.pulsar.shade.org.apache.jute - - - com.typesafe - org.apache.pulsar.shade.com.typesafe - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - javax.websocket - org.apache.pulsar.shade.javax.websocket - - - org.glassfish - org.apache.pulsar.shade.org.glassfish - - - org.apache.bookkeeper - org.apache.pulsar.shade.org.apache.bookkeeper - - - org.rocksdb - org.apache.pulsar.shade.org.rocksdb - - - com.sun.jna - org.apache.pulsar.shade.com.sun.jna - - - com.carrotsearch - org.apache.pulsar.shade.com.carrotsearch - - - javax.servlet - org.apache.pulsar.shade.javax.servlet - - - com.github - org.apache.pulsar.shade.com.github - - - jline - org.apache.pulsar.shade.jline - - - com.sun - org.apache.pulsar.shade.com.sun - - - jersey - org.apache.pulsar.shade.jersey - - - javax.ws - org.apache.pulsar.shade.javax.ws - - - javax.inject - org.apache.pulsar.shade.javax.inject - - - org.jvnet - org.apache.pulsar.shade.org.jvnet - - - com.beust - org.apache.pulsar.shade.com.beust - - - com.wordnik - org.apache.pulsar.shade.com.worknik - - - io.prometheus - org.apache.pulsar.shade.io.prometheus - - - io.swagger - org.apache.pulsar.shade.io.swagger - - - org.joda - org.apache.pulsar.shade.org.joda - - - org.yaml - org.apache.pulsar.shade.org.yaml - - - org.HdrHistogram - org.apache.pulsar.shade.org.HdrHistogram - - - com.ea - org.apache.pulsar.shade.com.ea - - - javassist - org.apache.pulsar.shade.javassist - - - org.reactivestreams - org.apache.pulsar.shade.org.reactivestreams - - - org.aopalliance - org.apache.pulsar.shade.org.aopalliance - - - com.wordnik - org.apache.pulsar.shade.com.worknik - - - org.apache.avro - org.apache.pulsar.shade.org.apache.avro - - - - org.codehaus.jackson - org.apache.pulsar.shade.org.codehaus.jackson - - - com.thoughtworks.paranamer - org.apache.pulsar.shade.com.thoughtworks.paranamer - - - org.tukaani - org.apache.pulsar.shade.org.tukaani - - - - - - - - - exec-maven-plugin - org.codehaus.mojo - - - rename-epoll-library - package - - exec - - - ${project.parent.basedir}/src/${rename.netty.native.libs} - - ${project.artifactId} - - - - - - - - diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 24f204415cf51..9b7886c825e81 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -435,6 +435,18 @@ ${project.version} + + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + test + + + io.grpc + grpc-netty-shaded + test + io.etcd jetcd-test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java index bdf1c3bdecbc9..4f20c1b360556 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/ResourceGroupsTest.java @@ -27,6 +27,7 @@ import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.ResourceGroup; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -167,8 +168,9 @@ public void testNamespaceResourceGroup() throws Exception { } // remove resourcegroup from namespace admin.namespaces().removeNamespaceResourceGroup(testNameSpace); - resourcegroups.deleteResourceGroup("test-resourcegroup-three"); - + Awaitility.await().untilAsserted(() -> { + resourcegroups.deleteResourceGroup("test-resourcegroup-three"); + }); } private void prepareData() throws PulsarAdminException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java index 1eb04ca1ccf0c..e80d0554299d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/pendingack/PendingAckPersistentTest.java @@ -365,7 +365,7 @@ private void testDeleteTopicThenDeletePendingAckManagedLedger() throws Exception assertFalse(topics.contains(topic)); } - @Test + @Test(groups = "quarantine") public void testDeleteUselessLogDataWhenSubCursorMoved() throws Exception { getPulsarServiceList().get(0).getConfig().setTransactionPendingAckLogIndexMinLag(5); getPulsarServiceList().get(0).getConfiguration().setManagedLedgerDefaultMarkDeleteRateLimit(5); diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 2121f4edd1f25..3a455226572a5 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -284,6 +284,12 @@ org.apache.bookkeeper org.apache.pulsar.shade.org.apache.bookkeeper + + + (META-INF/native/(lib)?)(netty.+\.(so|jnilib|dll))$ + $1org_apache_pulsar_shade_$3 + true + diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index fbd1b2d5cc9c4..c30ae0849ab20 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -364,6 +364,12 @@ org.tukaani org.apache.pulsar.shade.org.tukaani + + + (META-INF/native/(lib)?)(netty.+\.(so|jnilib|dll))$ + $1org_apache_pulsar_shade_$3 + true + @@ -373,31 +379,6 @@ - - - - exec-maven-plugin - org.codehaus.mojo - - - rename-epoll-library - package - - exec - - - ${project.parent.basedir}/src/${rename.netty.native.libs} - - ${project.artifactId} - - - - - diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 9f01e96d6ee60..013e4aa7450b0 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -297,6 +297,12 @@ org.apache.bookkeeper org.apache.pulsar.shade.org.apache.bookkeeper + + + (META-INF/native/(lib)?)(netty.+\.(so|jnilib|dll))$ + $1org_apache_pulsar_shade_$3 + true + @@ -320,31 +326,6 @@ - - - - exec-maven-plugin - org.codehaus.mojo - - - rename-epoll-library - package - - exec - - - ${project.parent.basedir}/src/${rename.netty.native.libs} - - ${project.artifactId} - - - - - diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 13987177b7ad4..5d90dc980deb1 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -83,6 +83,16 @@ io.netty netty-resolver-dns + + io.netty + netty-resolver-dns-native-macos + osx-aarch_64 + + + io.netty + netty-resolver-dns-native-macos + osx-x86_64 + org.apache.commons diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index b3c331613e5bf..b3b02516effba 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -85,14 +85,13 @@ io.netty - netty-transport-native-unix-common - linux-x86_64 + netty-transport-native-epoll + linux-aarch_64 io.netty - netty-transport-native-epoll - linux-aarch_64 + netty-transport-native-unix-common @@ -137,6 +136,23 @@ netty-tcnative-boringssl-static + + io.netty.incubator + netty-incubator-transport-classes-io_uring + + + + io.netty.incubator + netty-incubator-transport-native-io_uring + linux-x86_64 + + + + io.netty.incubator + netty-incubator-transport-native-io_uring + linux-aarch_64 + + io.netty netty-codec-haproxy diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index b300bed1a3f34..c458b20e99998 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -107,7 +107,7 @@ io.grpc - grpc-all + * com.google.protobuf @@ -116,6 +116,11 @@ + + io.grpc + grpc-netty-shaded + + io.grpc grpc-stub diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 16f6751be0d45..ba40d8487414b 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -83,10 +83,15 @@ - io.etcd - jetcd-core + ${project.groupId} + jetcd-core-shaded + ${project.version} + shaded + + + io.grpc + grpc-netty-shaded - io.etcd diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java index 6a15e1e8cc23e..3f5f26ba302b5 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java @@ -21,7 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import io.etcd.jetcd.launcher.EtcdCluster; -import io.etcd.jetcd.launcher.EtcdClusterFactory; +import io.etcd.jetcd.test.EtcdClusterExtension; import java.io.File; import java.net.URI; import java.util.UUID; @@ -84,10 +84,11 @@ public Object[][] implementations() { private synchronized String getEtcdClusterConnectString() { if (etcdCluster == null) { - etcdCluster = EtcdClusterFactory.buildCluster("test", 1, false); + etcdCluster = EtcdClusterExtension.builder().withClusterName("test").withNodes(1).withSsl(false).build() + .cluster(); etcdCluster.start(); } - return etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + return etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); } public static Supplier stringSupplier(Supplier supplier) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java index 8f70ab5195852..5169225a0ae0b 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java @@ -29,7 +29,7 @@ public class LeaderElectionImplTest extends BaseMetadataStoreTest { - @Test(dataProvider = "impl", timeOut = 10000) + @Test(dataProvider = "impl", timeOut = 20000) public void validateDeadLock(String provider, Supplier urlSupplier) throws Exception { if (provider.equals("Memory") || provider.equals("RocksDB")) { diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 5cea16335652e..b9ceef48124d2 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -231,52 +231,52 @@ The Apache Software License, Version 2.0 - commons-compress-1.26.0.jar - commons-lang3-3.11.jar * Netty - - netty-3.10.6.Final.jar - - netty-buffer-4.1.100.Final.jar - - netty-codec-4.1.100.Final.jar - - netty-codec-dns-4.1.100.Final.jar - - netty-codec-http-4.1.100.Final.jar - - netty-codec-haproxy-4.1.100.Final.jar - - netty-codec-socks-4.1.100.Final.jar - - netty-handler-proxy-4.1.100.Final.jar - - netty-common-4.1.100.Final.jar - - netty-handler-4.1.100.Final.jar + - netty-3.10.6.Final.jar + - netty-buffer-4.1.121.Final.jar + - netty-codec-4.1.121.Final.jar + - netty-codec-dns-4.1.121.Final.jar + - netty-codec-http-4.1.121.Final.jar + - netty-codec-haproxy-4.1.121.Final.jar + - netty-codec-socks-4.1.121.Final.jar + - netty-handler-proxy-4.1.121.Final.jar + - netty-common-4.1.121.Final.jar + - netty-handler-4.1.121.Final.jar - netty-reactive-streams-2.0.6.jar - - netty-resolver-4.1.100.Final.jar - - netty-resolver-dns-4.1.100.Final.jar - - netty-tcnative-boringssl-static-2.0.61.Final.jar - - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.61.Final.jar - - netty-transport-4.1.100.Final.jar - - netty-transport-classes-epoll-4.1.100.Final.jar - - netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar - - netty-transport-native-epoll-4.1.100.Final-linux-aarch_64.jar - - netty-transport-native-unix-common-4.1.100.Final.jar - - netty-transport-native-unix-common-4.1.100.Final-linux-x86_64.jar - - netty-codec-http2-4.1.100.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.25.Final.jar - - netty-incubator-transport-native-io_uring-0.0.25.Final-linux-aarch_64.jar - - netty-incubator-transport-native-io_uring-0.0.25.Final-linux-x86_64.jar + - netty-resolver-4.1.121.Final.jar + - netty-resolver-dns-4.1.121.Final.jar + - netty-resolver-dns-classes-macos-4.1.121.Final.jar + - netty-resolver-dns-native-macos-4.1.121.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.121.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.70.Final.jar + - netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.70.Final.jar + - netty-transport-4.1.121.Final.jar + - netty-transport-classes-epoll-4.1.121.Final.jar + - netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.121.Final.jar + - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar + - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar + - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar * GRPC - - grpc-api-1.45.1.jar - - grpc-context-1.45.1.jar - - grpc-core-1.45.1.jar - - grpc-grpclb-1.45.1.jar - - grpc-netty-1.45.1.jar - - grpc-protobuf-1.45.1.jar - - grpc-protobuf-lite-1.45.1.jar - - grpc-stub-1.45.1.jar - * JEtcd - - jetcd-common-0.5.11.jar - - jetcd-core-0.5.11.jar - + - grpc-api-1.55.3.jar + - grpc-context-1.55.3.jar + - grpc-core-1.55.3.jar + - grpc-grpclb-1.55.3.jar + - grpc-protobuf-1.55.3.jar + - grpc-protobuf-lite-1.55.3.jar + - grpc-netty-shaded-1.55.3.jar + - grpc-stub-1.55.3.jar + - grpc-util-1.60.0.jar + * JEtcd - shaded + * Vertx - shaded * Joda Time - joda-time-2.10.5.jar - - failsafe-2.4.4.jar + - failsafe-3.3.2.jar * Jetty - http2-client-9.4.54.v20240208.jar - http2-common-9.4.54.v20240208.jar @@ -488,9 +488,7 @@ The Apache Software License, Version 2.0 * Swagger - swagger-annotations-1.6.10.jar * Perfmark - - perfmark-api-0.19.0.jar - * Annotations - - auto-service-annotations-1.0.jar + - perfmark-api-0.26.0.jar * RxJava - rxjava-3.0.1.jar * High Performance Primitive Collections for Java @@ -499,7 +497,7 @@ Protocol Buffers License * Protocol Buffers - protobuf-java-3.19.6.jar - protobuf-java-util-3.19.6.jar - - proto-google-common-protos-2.0.1.jar + - proto-google-common-protos-2.9.0.jar BSD 3-clause "New" or "Revised" License * RE2J TD -- re2j-td-1.4.jar @@ -530,9 +528,6 @@ MIT License - jul-to-slf4j-1.7.32.jar * Checker Qual - checker-qual-3.33.0.jar - * Annotations - - animal-sniffer-annotations-1.19.jar - - annotations-4.1.1.4.jar CDDL - 1.0 * OSGi Resource Locator diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml index bc1fe5ed46037..c3b0a18c959f1 100644 --- a/pulsar-sql/presto-distribution/src/assembly/assembly.xml +++ b/pulsar-sql/presto-distribution/src/assembly/assembly.xml @@ -61,6 +61,9 @@ io.airlift:launcher:tar.gz:bin:${airlift.version} io.airlift:launcher:tar.gz:properties:${airlift.version} *:tar.gz + + org.codehaus.mojo:animal-sniffer-annotations + com.google.android:annotations diff --git a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml index 6650abfda3fc3..ac17aaed70bdf 100644 --- a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml +++ b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml @@ -33,6 +33,9 @@ runtime jakarta.ws.rs:jakarta.ws.rs-api + + org.codehaus.mojo:animal-sniffer-annotations + com.google.android:annotations diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh index afe380d5c8ffb..54539c704d6fe 100755 --- a/src/check-binary-license.sh +++ b/src/check-binary-license.sh @@ -96,7 +96,7 @@ done if [ "$NO_PRESTO" -ne 1 ]; then # check pulsar sql jars - JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | grep -v 'java-version-trim-agent' | sed 's!.*/!!' | sort) + JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v jetcd-core-shaded | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | grep -v 'java-version-trim-agent' | sed 's!.*/!!' | sort) LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/lib\/presto\/LICENSE/') LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH") LICENSEJARS=$(echo "$LICENSE" | sed -nE 's!.* (.*\.jar).*!\1!gp') diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index e242ac52cec08..735fb522a24c1 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -187,51 +187,6 @@ CVE-2021-42550 - - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15106 - - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15112 - - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15113 - - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15106 - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15112 - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15113 - - %Obj_Current.TO%" -call ren %Obj_Current.FROM% %Obj_Current.TO% - -SET /A Obj_Index=%Obj_Index% + 1 - -GOTO LoopStart -:: Loop end - -:END -cd /d %TMP_DIR% - -:: Overwrite the original ZIP archive -rd %JAR_PATH% /s /q -set ZIP_CMD=zip -q -r %JAR_PATH% . -:: echo %ZIP_CMD% -call %ZIP_CMD% -:: echo %TMP_DIR% -rd %TMP_DIR% /s /q - -exit /b 0 -:: echo.&pause&goto:eof \ No newline at end of file diff --git a/src/rename-netty-native-libs.sh b/src/rename-netty-native-libs.sh deleted file mode 100755 index 9674d9c66c030..0000000000000 --- a/src/rename-netty-native-libs.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -e - -ARTIFACT_ID=$1 -JAR_PATH="$PWD/target/$ARTIFACT_ID.jar" - -FILE_PREFIX='META-INF/native' - -FILES_TO_RENAME=( - 'libnetty_transport_native_epoll_x86_64.so liborg_apache_pulsar_shade_netty_transport_native_epoll_x86_64.so' - 'libnetty_transport_native_epoll_aarch_64.so liborg_apache_pulsar_shade_netty_transport_native_epoll_aarch_64.so' - 'libnetty_tcnative_linux_x86_64.so liborg_apache_pulsar_shade_netty_tcnative_linux_x86_64.so' - 'libnetty_tcnative_linux_aarch_64.so liborg_apache_pulsar_shade_netty_tcnative_linux_aarch_64.so' - 'libnetty_tcnative_osx_x86_64.jnilib liborg_apache_pulsar_shade_netty_tcnative_osx_x86_64.jnilib' - 'libnetty_tcnative_osx_aarch_64.jnilib liborg_apache_pulsar_shade_netty_tcnative_osx_aarch_64.jnilib' -) - -echo "----- Renaming epoll lib in $JAR_PATH ------" -TMP_DIR=`mktemp -d` -CUR_DIR=$(pwd) -cd ${TMP_DIR} -# exclude `META-INF/LICENSE` -unzip -q $JAR_PATH -x "META-INF/LICENSE" -# include `META-INF/LICENSE` as LICENSE.netty. -# This approach is to get around the issue that MacOS is not able to recognize the difference between `META-INF/LICENSE` and `META-INF/license/`. -unzip -p $JAR_PATH META-INF/LICENSE > META-INF/LICENSE.netty -cd ${CUR_DIR} - -pushd $TMP_DIR - -for line in "${FILES_TO_RENAME[@]}"; do - read -r -a A <<< "$line" - FROM=${A[0]} - TO=${A[1]} - - if [ -f $FILE_PREFIX/$FROM ]; then - echo "Renaming $FROM -> $TO" - mv $FILE_PREFIX/$FROM $FILE_PREFIX/$TO - fi -done - -# Overwrite the original ZIP archive -rm $JAR_PATH -zip -q -r $JAR_PATH . -popd - -rm -rf $TMP_DIR diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index c9da4011938ca..4e0a5d0e4aaf2 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -101,6 +101,12 @@ test + + dev.failsafe + failsafe + test + + org.testcontainers mysql diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java new file mode 100644 index 0000000000000..04db0c1cb9e4c --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.bookkeeper; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static org.testng.Assert.assertEquals; + +/** + * Test bookkeeper setup with http server enabled. + */ +@Slf4j +public class BookkeeperInstallWithHttpServerEnabledTest extends PulsarClusterTestBase { + + @BeforeClass(alwaysRun = true) + @Override + public final void setupCluster() throws Exception { + incrementSetupNumber(); + + final String clusterName = Stream.of(this.getClass().getSimpleName(), randomName(5)) + .filter(s -> !s.isEmpty()) + .collect(joining("-")); + bookkeeperEnvs.put("httpServerEnabled", "true"); + bookieAdditionalPorts.add(8000); + PulsarClusterSpec spec = PulsarClusterSpec.builder() + .numBookies(2) + .numBrokers(1) + .bookkeeperEnvs(bookkeeperEnvs) + .bookieAdditionalPorts(bookieAdditionalPorts) + .clusterName(clusterName) + .build(); + + log.info("Setting up cluster {} with {} bookies, {} brokers", + spec.clusterName(), spec.numBookies(), spec.numBrokers()); + + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + + log.info("Cluster {} is setup", spec.clusterName()); + } + + @AfterClass(alwaysRun = true) + @Override + public final void tearDownCluster() throws Exception { + super.tearDownCluster(); + } + + @Test + public void testBookieHttpServerIsRunning() throws Exception { + ContainerExecResult result = pulsarCluster.getAnyBookie().execCmd( + PulsarCluster.CURL, + "-X", + "GET", + "http://localhost:8000/heartbeat"); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout(), "OK\n"); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java index e00e8fbe677df..882162b98e684 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/PulsarIOTestRunner.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.tests.integration.io; +import dev.failsafe.RetryPolicy; import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; @@ -33,7 +34,6 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.RetryPolicy; @Slf4j public abstract class PulsarIOTestRunner { @@ -42,11 +42,11 @@ public abstract class PulsarIOTestRunner { final Duration ONE_MINUTE = Duration.ofMinutes(1); final Duration TEN_SECONDS = Duration.ofSeconds(10); - @SuppressWarnings({ "unchecked", "rawtypes" }) - protected final RetryPolicy statusRetryPolicy = new RetryPolicy() + protected final RetryPolicy statusRetryPolicy = RetryPolicy.builder() .withMaxDuration(ONE_MINUTE) .withDelay(TEN_SECONDS) - .onRetry(e -> log.error("Retry ... ")); + .onRetry(e -> log.error("Retry ... ")) + .build(); protected PulsarCluster pulsarCluster; protected String functionRuntimeType; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java index b6500e36a8424..28a65e55ffdc5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/PulsarIOSinkRunner.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import dev.failsafe.Failsafe; import java.util.LinkedHashMap; import java.util.Map; @@ -46,7 +47,6 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; @Slf4j public class PulsarIOSinkRunner extends PulsarIOTestRunner { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java index d1e5049bdcf8e..105c31cc88ca7 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/PulsarIOSourceRunner.java @@ -22,6 +22,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import dev.failsafe.Failsafe; import java.util.Map; import org.apache.commons.lang3.StringUtils; @@ -45,7 +46,6 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; @Slf4j public class PulsarIOSourceRunner extends PulsarIOTestRunner { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java index da1e7597ea636..2b953f8956c2d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sources/debezium/PulsarIODebeziumSourceRunner.java @@ -19,6 +19,9 @@ package org.apache.pulsar.tests.integration.io.sources.debezium; import com.google.common.base.Preconditions; +import dev.failsafe.Failsafe; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.SubscriptionInitialPosition; @@ -28,10 +31,6 @@ import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.containers.GenericContainer; -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.Failsafe; - @Slf4j public class PulsarIODebeziumSourceRunner extends PulsarIOSourceRunner { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index 0889bee187d28..5bc7df0f1cb45 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -149,20 +149,26 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s // create bookies bookieContainers.putAll( - runNumContainers("bookie", spec.numBookies(), (name) -> new BKContainer(clusterName, name) - .withNetwork(network) - .withNetworkAliases(appendClusterName(name)) - .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) - .withEnv("useHostNameAsBookieID", "true") - // Disable fsyncs for tests since they're slow within the containers - .withEnv("journalSyncData", "false") - .withEnv("journalMaxGroupWaitMSec", "0") - .withEnv("clusterName", clusterName) - .withEnv("PULSAR_PREFIX_diskUsageWarnThreshold", "0.95") - .withEnv("diskUsageThreshold", "0.99") - .withEnv("PULSAR_PREFIX_diskUsageLwmThreshold", "0.97") - .withEnv("nettyMaxFrameSizeBytes", "" + spec.maxMessageSize) - ) + runNumContainers("bookie", spec.numBookies(), (name) -> { + BKContainer bookieContainer = new BKContainer(clusterName, name) + .withNetwork(network) + .withNetworkAliases(appendClusterName(name)) + .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) + .withEnv("useHostNameAsBookieID", "true") + // Disable fsyncs for tests since they're slow within the containers + .withEnv("journalSyncData", "false") + .withEnv("journalMaxGroupWaitMSec", "0") + .withEnv("clusterName", clusterName) + .withEnv("diskUsageThreshold", "0.99") + .withEnv("nettyMaxFrameSizeBytes", String.valueOf(spec.maxMessageSize)); + if (spec.bookkeeperEnvs != null) { + bookieContainer.withEnv(spec.bookkeeperEnvs); + } + if (spec.bookieAdditionalPorts != null) { + spec.bookieAdditionalPorts.forEach(bookieContainer::addExposedPort); + } + return bookieContainer; + }) ); // create brokers @@ -699,4 +705,8 @@ public void dumpFunctionLogs(String name) { private String appendClusterName(String name) { return sharedCsContainer ? clusterName + "-" + name : name; } + + public BKContainer getAnyBookie() { + return getAnyContainer(bookieContainers, "bookie"); + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java index eed604205bdce..bea58194a2eb5 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java @@ -19,6 +19,7 @@ package org.apache.pulsar.tests.integration.topologies; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -144,6 +145,11 @@ public class PulsarClusterSpec { */ Map brokerEnvs; + /** + * Specify envs for bookkeeper. + */ + Map bookkeeperEnvs; + /** * Specify mount files. */ @@ -156,4 +162,9 @@ public class PulsarClusterSpec { @Default int maxMessageSize = Commands.DEFAULT_MAX_MESSAGE_SIZE; + + /** + * Additional ports to expose on bookie containers. + */ + List bookieAdditionalPorts; } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java index 34994f75216d4..a0459931330c6 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java @@ -19,6 +19,8 @@ package org.apache.pulsar.tests.integration.topologies; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; @@ -32,6 +34,8 @@ @Slf4j public abstract class PulsarClusterTestBase extends PulsarTestBase { protected final Map brokerEnvs = new HashMap<>(); + protected final Map bookkeeperEnvs = new HashMap<>(); + protected final List bookieAdditionalPorts = new LinkedList<>(); @Override protected final void setup() throws Exception {