Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *
http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *
Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.model; import java.time.LocalDateTime; -import javax.persistence.Column; -import javax.persistence.Convert; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - import org.hibernate.annotations.UpdateTimestamp; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java index b80e895c..7d7da4d4 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *
http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *
Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.model; @@ -19,18 +17,17 @@ import java.time.LocalDateTime; -import javax.persistence.Column; -import javax.persistence.Convert; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - import org.hibernate.annotations.UpdateTimestamp; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -83,6 +80,7 @@ public class HousekeepingPath implements HousekeepingEntity { @Column(name = "client_id") private String clientId; + @Column(name = "lifecycle_type", nullable = false) private String lifecycleType; diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java index 0da210b7..4e228b9a 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java @@ -1,22 +1,20 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *
http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.core.model;
-import javax.persistence.AttributeConverter;
-import javax.persistence.Converter;
+import jakarta.persistence.AttributeConverter;
+import jakarta.persistence.Converter;
@Converter(autoApply = true)
public class PeriodDurationConverter implements AttributeConverter Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.core.model.history;
import java.time.LocalDateTime;
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.Table;
-
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import jakarta.persistence.Table;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
@@ -69,8 +66,7 @@ public BeekeeperHistory(
String tableName,
String lifecycleType,
String housekeepingStatus,
- String eventDetails
- ) {
+ String eventDetails) {
this.id = id;
this.eventTimestamp = eventTimestamp;
this.databaseName = databaseName;
@@ -84,5 +80,4 @@ public BeekeeperHistory(
public MetricTag getMetricTag() {
return new MetricTag("table", String.join(".", databaseName, tableName));
}
-
}
diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java
index 5c3dffb3..99636269 100644
--- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java
+++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java
@@ -1,16 +1,14 @@
/**
- * Copyright (C) 2019-2025 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.core.repository;
@@ -19,16 +17,16 @@
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.CrudRepository;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.data.repository.query.Param;
import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
public interface BeekeeperHistoryRepository extends PagingAndSortingRepository Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.core.repository;
@@ -24,13 +22,15 @@
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.CrudRepository;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.data.repository.query.Param;
import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
public interface HousekeepingMetadataRepository
- extends PagingAndSortingRepository Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.core.repository;
@@ -22,6 +20,7 @@
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.CrudRepository;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
@@ -30,7 +29,8 @@
@Repository
public interface HousekeepingPathRepository
- extends PagingAndSortingRepository Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.core;
import java.util.TimeZone;
-import javax.annotation.PostConstruct;
-
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import jakarta.annotation.PostConstruct;
+
@SpringBootApplication
@EnableConfigurationProperties
@ComponentScan("com.expediagroup.beekeeper.core")
@@ -36,5 +34,4 @@ public class TestApplication {
void started() {
TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
}
-
}
diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java
index ef6df464..6a405baa 100644
--- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java
+++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2019-2023 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
import static java.time.temporal.ChronoUnit.DAYS;
import static java.time.temporal.ChronoUnit.HOURS;
+import static java.time.temporal.ChronoUnit.MICROS;
import static java.time.temporal.ChronoUnit.MONTHS;
import static org.assertj.core.api.Assertions.assertThat;
@@ -71,7 +72,7 @@ public class HousekeepingMetadataRepositoryTest {
private static final String DATABASE_NAME = "database";
private static final String TABLE_NAME = "table";
private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A";
- private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC"));
+ private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")).truncatedTo(MICROS);
private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D");
private static final LocalDateTime CLEANUP_TIMESTAMP = CREATION_TIMESTAMP.plus(CLEANUP_DELAY);
@@ -525,8 +526,10 @@ private void compare(HousekeepingMetadata expected, HousekeepingMetadata actual)
assertThat(actual.getTableName()).isEqualTo(expected.getTableName());
assertThat(actual.getPartitionName()).isEqualTo(expected.getPartitionName());
assertThat(actual.getHousekeepingStatus()).isEqualTo(expected.getHousekeepingStatus());
- assertThat(actual.getCreationTimestamp()).isEqualTo(expected.getCreationTimestamp());
- assertThat(actual.getModifiedTimestamp()).isEqualTo(expected.getModifiedTimestamp());
+ assertThat(actual.getCreationTimestamp().truncatedTo(MICROS))
+ .isEqualTo(expected.getCreationTimestamp().truncatedTo(MICROS));
+ assertThat(actual.getModifiedTimestamp().truncatedTo(MICROS))
+ .isEqualTo(expected.getModifiedTimestamp().truncatedTo(MICROS));
assertThat(actual.getCleanupDelay()).isEqualTo(expected.getCleanupDelay());
assertThat(actual.getCleanupAttempts()).isEqualTo(expected.getCleanupAttempts());
assertThat(actual.getLifecycleType()).isEqualTo(expected.getLifecycleType());
diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java
index 6aba9e9d..a79f2621 100644
--- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java
+++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2019-2023 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
import static java.time.temporal.ChronoUnit.DAYS;
import static java.time.temporal.ChronoUnit.HOURS;
+import static java.time.temporal.ChronoUnit.MICROS;
import static java.time.temporal.ChronoUnit.MONTHS;
import static org.assertj.core.api.Assertions.assertThat;
@@ -64,7 +65,7 @@
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
public class HousekeepingPathRepositoryTest {
- private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC"));
+ private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")).truncatedTo(MICROS);
private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("PT1H");
private static final LocalDateTime CLEANUP_TIMESTAMP = CREATION_TIMESTAMP.plus(CLEANUP_DELAY);
diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml
index 20065a7c..96443448 100644
--- a/beekeeper-integration-tests/pom.xml
+++ b/beekeeper-integration-tests/pom.xml
@@ -34,6 +34,18 @@
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *
http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration;
@@ -28,6 +26,7 @@
import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE;
import java.sql.SQLException;
+import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -38,8 +37,6 @@
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.thrift.TException;
-import org.awaitility.Duration;
-import org.junit.Rule;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -47,6 +44,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import ch.qos.logback.classic.spi.ILoggingEvent;
@@ -116,20 +114,16 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte
private static final String S3_CLIENT_CLASS_NAME = "S3Client";
private static final String HIVE_CLIENT_CLASS_NAME = "HiveClient";
- @Rule
- public static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3);
- static {
- S3_CONTAINER.start();
- }
+ @Container
+ private static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3);
private static AmazonS3 amazonS3;
- private static final String S3_ENDPOINT = ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3);
private final ExecutorService executorService = Executors.newFixedThreadPool(1);
private final TestAppender appender = new TestAppender();
- private static Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration;
@@ -24,12 +22,12 @@
import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE;
import java.sql.SQLException;
+import java.time.Duration;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import org.awaitility.Duration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -115,8 +113,7 @@ public void setup() {
.getObjectSummaries()
.forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey()));
executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {}));
- await().atMost(Duration.ONE_MINUTE)
- .until(BeekeeperPathCleanup::isRunning);
+ await().atMost(Duration.ofMinutes(1)).until(BeekeeperPathCleanup::isRunning);
// clear all logs before asserting them
appender.clear();
diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java
index 398033b4..98e3ddb3 100644
--- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java
+++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java
@@ -1,16 +1,14 @@
/**
- * Copyright (C) 2019-2025 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration;
@@ -36,6 +34,7 @@
import java.io.IOException;
import java.net.URISyntaxException;
import java.sql.SQLException;
+import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -46,7 +45,6 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
-import org.awaitility.Duration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -86,6 +84,8 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek
protected static final int TIMEOUT = 30;
protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url";
+ protected static final String SQS_ENDPOINT_PROPERTY = "properties.sqs.endpoint";
+ protected static final String SQS_REGION_PROPERTY = "properties.sqs.region";
protected static final String QUEUE = "apiary-receiver-queue";
protected static final String SCHEDULED_EXPIRED_METRIC = "metadata-scheduled";
@@ -116,16 +116,13 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek
protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3);
protected static AmazonSQS amazonSQS;
+ protected static String ACTUAL_QUEUE_URL;
protected HiveTestUtils hiveTestUtils;
protected HiveMetaStoreClient metastoreClient;
- static {
- S3_CONTAINER.start();
- }
-
protected static AmazonS3 amazonS3;
- private static Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration;
@@ -35,6 +33,7 @@
import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE;
import java.sql.SQLException;
+import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -50,8 +49,6 @@
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.thrift.TException;
-import org.awaitility.Duration;
-import org.junit.Rule;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -59,6 +56,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import io.micrometer.core.instrument.Meter;
@@ -126,19 +124,15 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio
+ UNPARTITIONED_TABLE_NAME
+ "/id1/file1";
- @Rule
- public static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3);
- static {
- S3_CONTAINER.start();
- }
+ @Container
+ protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3);
protected static AmazonS3 amazonS3;
- protected static final String S3_ENDPOINT = ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3);
protected final ExecutorService executorService = Executors.newFixedThreadPool(1);
- private static Map Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration;
@@ -27,6 +25,7 @@
import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE;
import java.sql.SQLException;
+import java.time.Duration;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@@ -36,7 +35,6 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
-import org.awaitility.Duration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -117,7 +115,7 @@ public void setup() {
.getObjectSummaries()
.forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey()));
executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {}));
- await().atMost(Duration.ONE_MINUTE)
+ await().atMost(Duration.ofMinutes(1))
.until(BeekeeperPathCleanup::isRunning);
}
diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java
index 67a2c39d..eda37c91 100644
--- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java
+++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java
@@ -1,16 +1,14 @@
/**
- * Copyright (C) 2019-2025 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration;
@@ -34,6 +32,7 @@
import java.io.IOException;
import java.net.URISyntaxException;
import java.sql.SQLException;
+import java.time.Duration;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -41,7 +40,6 @@
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
-import org.awaitility.Duration;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -75,7 +73,7 @@
@Testcontainers
public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase {
- protected static final int TIMEOUT = 5;
+ protected static final int TIMEOUT = 30;
protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url";
protected static final String METASTORE_URI_PROPERTY = "properties.metastore-uri";
@@ -87,23 +85,30 @@ public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends Bee
@Container
protected static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS);
protected static AmazonSQS amazonSQS;
+ protected static String ACTUAL_QUEUE_URL;
@RegisterExtension
protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension(
DATABASE_NAME_VALUE, emptyMap());
+ protected static final String SQS_ENDPOINT_PROPERTY = "properties.sqs.endpoint";
+ protected static final String SQS_REGION_PROPERTY = "properties.sqs.region";
+
@BeforeAll
public static void init() {
- String queueUrl = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE);
- System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl);
-
amazonSQS = ContainerTestUtils.sqsClient(SQS_CONTAINER, AWS_REGION);
+ ACTUAL_QUEUE_URL = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE);
amazonSQS.createQueue(QUEUE);
+ System.setProperty(APIARY_QUEUE_URL_PROPERTY, ACTUAL_QUEUE_URL);
+ System.setProperty(SQS_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(SQS_CONTAINER, SQS));
+ System.setProperty(SQS_REGION_PROPERTY, AWS_REGION);
}
@AfterAll
public static void teardown() {
System.clearProperty(APIARY_QUEUE_URL_PROPERTY);
+ System.clearProperty(SQS_ENDPOINT_PROPERTY);
+ System.clearProperty(SQS_REGION_PROPERTY);
amazonSQS.shutdown();
}
@@ -112,9 +117,9 @@ public static void teardown() {
public void setup() {
System.setProperty(METASTORE_URI_PROPERTY, thriftHiveMetaStore.getThriftConnectionUri());
- amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE)));
+ amazonSQS.purgeQueue(new PurgeQueueRequest(ACTUAL_QUEUE_URL));
executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {}));
- await().atMost(Duration.ONE_MINUTE).until(BeekeeperSchedulerApiary::isRunning);
+ await().atMost(Duration.ofMinutes(1)).until(BeekeeperSchedulerApiary::isRunning);
}
@AfterEach
@@ -242,7 +247,7 @@ public void prometheus() {
}
protected SendMessageRequest sendMessageRequest(String payload) {
- return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload);
+ return new SendMessageRequest(ACTUAL_QUEUE_URL, payload);
}
protected void assertUnreferencedPath(HousekeepingPath actual, String expectedPath) {
diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java
index d7cbc63c..9c59c733 100644
--- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java
+++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java
@@ -1,16 +1,14 @@
/**
- * Copyright (C) 2019-2025 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration.api;
@@ -26,6 +24,7 @@
import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE;
import java.io.IOException;
+import java.net.ServerSocket;
import java.net.http.HttpResponse;
import java.sql.SQLException;
import java.time.Duration;
@@ -43,7 +42,6 @@
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.data.domain.Page;
import org.springframework.http.HttpStatus;
-import org.springframework.util.SocketUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -91,10 +89,14 @@ public ObjectMapper geObjectMapper() {
protected final HousekeepingMetadata testMetadataC = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED);
protected final HousekeepingMetadata testMetadataD = createHousekeepingMetadata(someTable, pathC, partitionB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.SCHEDULED);
protected final HousekeepingMetadata testMetadataE = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString(),HousekeepingStatus.SCHEDULED);
+
@BeforeEach
- public void beforeEach() {
- int port = SocketUtils.findAvailableTcpPort();
- String[] args = new String[] { "--server.port=" + port };
+ public void beforeEach() throws IOException {
+ int port;
+ try (ServerSocket socket = new ServerSocket(0)) {
+ port = socket.getLocalPort();
+ }
+ String[] args = new String[] {"--server.port=" + port};
final String url = format("http://localhost:%d", port);
log.info("Starting to run Beekeeper API on: {} and args: {}", url, args);
context = SpringApplication.run(BeekeeperApiApplication.class, args);
diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java
index 32b8241d..38f9bb7b 100644
--- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java
+++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java
@@ -1,16 +1,14 @@
/**
- * Copyright (C) 2019-2023 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.integration.utils;
@@ -20,6 +18,7 @@
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.utility.DockerImageName;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.services.s3.AmazonS3;
@@ -36,15 +35,16 @@ public static MySQLContainer mySqlContainer() {
}
public static LocalStackContainer awsContainer(LocalStackContainer.Service service) {
- return new LocalStackContainer().withServices(service);
+ return new LocalStackContainer(DockerImageName.parse("localstack/localstack:3.0.0"))
+ .withServices(service);
}
public static String awsServiceEndpoint(LocalStackContainer awsContainer, LocalStackContainer.Service service) {
- return awsContainer.getEndpointConfiguration(service).getServiceEndpoint();
+ return awsContainer.getEndpointOverride(service).toString();
}
public static String queueUrl(LocalStackContainer awsContainer, String queue) {
- return awsServiceEndpoint(awsContainer, SQS) + "/queue/" + queue;
+ return awsServiceEndpoint(awsContainer, SQS) + "/000000000000/" + queue;
}
public static AmazonSQS sqsClient(LocalStackContainer awsContainer, String region) {
@@ -61,6 +61,7 @@ public static AmazonS3 s3Client(LocalStackContainer awsContainer, String region)
return AmazonS3ClientBuilder
.standard()
.withEndpointConfiguration(endpointConfiguration)
+ .withPathStyleAccessEnabled(true)
.disableChunkedEncoding()
.build();
}
diff --git a/beekeeper-metadata-cleanup/pom.xml b/beekeeper-metadata-cleanup/pom.xml
index 279c8a5f..ac1c7086 100644
--- a/beekeeper-metadata-cleanup/pom.xml
+++ b/beekeeper-metadata-cleanup/pom.xml
@@ -37,9 +37,13 @@
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *
http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *
Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.metadata.cleanup; import java.util.TimeZone; -import javax.annotation.PostConstruct; - import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import jakarta.annotation.PostConstruct; + @SpringBootApplication @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java index 0757d3ec..1fc99238 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java @@ -21,9 +21,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; +import java.time.Duration; import java.time.Instant; -import org.awaitility.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -58,7 +58,7 @@ class CleanupServiceSchedulerTest { @Test void typical() { - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any())); } @@ -66,7 +66,7 @@ void typical() { void cleanupServiceException() { doThrow(BeekeeperException.class).when(cleanupService) .cleanUp(any(Instant.class)); - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any())); } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java index b022d3b2..6d3b4e94 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *
http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *
Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.metadata.cleanup.service;
@@ -61,8 +59,8 @@
import com.expediagroup.beekeeper.cleanup.hive.HiveClientFactory;
import com.expediagroup.beekeeper.cleanup.metadata.MetadataCleaner;
import com.expediagroup.beekeeper.cleanup.path.PathCleaner;
+import com.expediagroup.beekeeper.core.model.HousekeepingEntity;
import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
-import com.expediagroup.beekeeper.core.model.HousekeepingPath;
import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
import com.expediagroup.beekeeper.core.model.PeriodDuration;
import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository;
@@ -73,7 +71,7 @@
@ExtendWith(SpringExtension.class)
@ExtendWith(MockitoExtension.class)
-@ContextConfiguration(classes = { TestApplication.class }, loader = AnnotationConfigContextLoader.class)
+@ContextConfiguration(classes = {TestApplication.class}, loader = AnnotationConfigContextLoader.class)
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
public class PagingMetadataCleanupServiceTest {
@@ -83,7 +81,7 @@ public class PagingMetadataCleanupServiceTest {
private final LocalDateTime localNow = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC);
private PagingMetadataCleanupService pagingCleanupService;
private @Captor ArgumentCaptor Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.path.cleanup;
import java.util.TimeZone;
-import javax.annotation.PostConstruct;
-
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import jakarta.annotation.PostConstruct;
+
@SpringBootApplication
@EnableConfigurationProperties
@ComponentScan("com.expediagroup.beekeeper.core")
diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java
index 85390fa0..7807da61 100644
--- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java
+++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java
@@ -21,9 +21,9 @@
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
+import java.time.Duration;
import java.time.Instant;
-import org.awaitility.Duration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -60,7 +60,7 @@ class CleanupServiceSchedulerTest {
@Test
void typical() {
- await().atMost(Duration.TEN_SECONDS)
+ await().atMost(Duration.ofSeconds(10))
.untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any()));
}
@@ -68,7 +68,7 @@ void typical() {
void cleanupServiceException() {
doThrow(BeekeeperException.class).when(cleanupService)
.cleanUp(any(Instant.class));
- await().atMost(Duration.TEN_SECONDS)
+ await().atMost(Duration.ofSeconds(10))
.untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any()));
}
}
diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java
index fafeecbc..6945c34e 100644
--- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java
+++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java
@@ -21,9 +21,9 @@
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.verify;
+import java.time.Duration;
import java.time.Instant;
-import org.awaitility.Duration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -55,7 +55,7 @@ class RepositoryCleanupSchedulerTest {
@Test
void typical() {
- await().atMost(Duration.TEN_SECONDS)
+ await().atMost(Duration.ofSeconds(10))
.untilAsserted(() -> verify(repositoryCleanupService, atLeast(2)).cleanUp(any()));
}
@@ -63,7 +63,7 @@ void typical() {
void cleanupServiceException() {
doThrow(BeekeeperException.class).when(repositoryCleanupService)
.cleanUp(any(Instant.class));
- await().atMost(Duration.TEN_SECONDS)
+ await().atMost(Duration.ofSeconds(10))
.untilAsserted(() -> verify(repositoryCleanupService, atLeast(2)).cleanUp(any()));
}
}
diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml
index ccd89134..96a69bfa 100644
--- a/beekeeper-scheduler-apiary/pom.xml
+++ b/beekeeper-scheduler-apiary/pom.xml
@@ -48,10 +48,22 @@
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.scheduler.apiary.app;
@@ -20,8 +18,6 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
-import javax.annotation.PreDestroy;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -29,6 +25,8 @@
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;
+import jakarta.annotation.PreDestroy;
+
import com.expediagroup.beekeeper.core.error.BeekeeperException;
import com.expediagroup.beekeeper.scheduler.apiary.service.SchedulerApiary;
diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java
index 53893e35..01824679 100644
--- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java
+++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java
@@ -1,16 +1,14 @@
/**
- * Copyright (C) 2019-2025 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.scheduler.apiary.context;
@@ -29,6 +27,10 @@
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.retry.annotation.EnableRetry;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
+
import com.expedia.apiary.extensions.receiver.common.event.AddPartitionEvent;
import com.expedia.apiary.extensions.receiver.common.event.AlterPartitionEvent;
import com.expedia.apiary.extensions.receiver.common.event.AlterTableEvent;
@@ -78,8 +80,19 @@ public EnumMap Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.scheduler.apiary.context;
@@ -86,7 +84,7 @@ public void validateSchedulerServiceMap() {
@Test
public void validateMessageReader() {
- MessageReader reader = commonBeans.messageReader("some_path");
+ MessageReader reader = commonBeans.messageReader("some_path", "", "us-east-1");
assertThat(reader).isInstanceOf(SqsMessageReader.class);
}
@@ -98,7 +96,8 @@ public void validateRetryingMessageReader() {
@Test
public void validateUnreferencedHousekeepingPathGenerator() {
- HousekeepingEntityGenerator generator = commonBeans.unreferencedHousekeepingPathGenerator("P30D");
+ HousekeepingEntityGenerator generator =
+ commonBeans.unreferencedHousekeepingPathGenerator("P30D");
assertThat(generator).isInstanceOf(UnreferencedHousekeepingPathGenerator.class);
}
diff --git a/beekeeper-scheduler/pom.xml b/beekeeper-scheduler/pom.xml
index 091a46af..82b5d9f8 100644
--- a/beekeeper-scheduler/pom.xml
+++ b/beekeeper-scheduler/pom.xml
@@ -64,6 +64,14 @@
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.vacuum;
@@ -37,7 +35,7 @@
import com.hotels.hcommon.hive.metastore.client.supplier.HiveMetaStoreClientSupplier;
@Configuration
-@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" })
+@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"})
@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository",
"com.expediagroup.beekeeper.vacuum.repository" })
public class CommonBeans {
@@ -60,6 +58,13 @@ public HiveConf hiveConf(@Value("${metastore-uri}") String metastoreUri) {
conf.set("fs.s3a.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName());
conf.set("fs.s3n.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName());
+ // Disable FileSystem cache to avoid UserGroupInformation.getCurrentUser() which is
+ // incompatible with Java 17+ when the security manager is not installed
+ conf.setBoolean("fs.file.impl.disable.cache", true);
+ conf.setBoolean("fs.s3.impl.disable.cache", true);
+ conf.setBoolean("fs.s3a.impl.disable.cache", true);
+ conf.setBoolean("fs.s3n.impl.disable.cache", true);
+
return conf;
}
@@ -77,7 +82,7 @@ Supplier Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.vacuum;
@@ -36,8 +34,10 @@ void metastorePathsCorrect() {
@Test
void metastorePathsDepthIncorrect() {
assertThatExceptionOfType(IllegalStateException.class)
- .isThrownBy(() -> ConsistencyCheck.checkMetastorePaths(
- Collections.singleton(new Path("/db/snapshot/partition")), 4));
+ .isThrownBy(
+ () ->
+ ConsistencyCheck.checkMetastorePaths(
+ Collections.singleton(new Path("/db/snapshot/partition")), 4));
}
@Test
@@ -55,14 +55,18 @@ void metastorePathDepthIncorrect() {
@Test
void unvisitedPath() throws IOException {
Path nonExistent = new Path("/db/table/snapshot/" + RandomStringUtils.randomAlphanumeric(8) + "/partition");
- FileSystem fs = nonExistent.getFileSystem(new Configuration(false));
+ Configuration conf = new Configuration(false);
+ conf.setBoolean("fs.file.impl.disable.cache", true);
+ FileSystem fs = nonExistent.getFileSystem(conf);
ConsistencyCheck.checkUnvisitedPath(fs, nonExistent);
}
@Test
void unvisitedPathExists() throws IOException {
Path exists = new Path("/tmp");
- FileSystem fs = exists.getFileSystem(new Configuration(false));
+ Configuration conf = new Configuration(false);
+ conf.setBoolean("fs.file.impl.disable.cache", true);
+ FileSystem fs = exists.getFileSystem(conf);
assertThatExceptionOfType(IllegalStateException.class)
.isThrownBy(() -> ConsistencyCheck.checkUnvisitedPath(fs, exists));
}
diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java
index e658c020..3a5c9415 100644
--- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java
+++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java
@@ -1,32 +1,30 @@
/**
- * Copyright (C) 2019-2021 Expedia, Inc.
+ * Copyright (C) 2019-2026 Expedia, Inc.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expediagroup.beekeeper.vacuum;
import java.util.TimeZone;
-import javax.annotation.PostConstruct;
-
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import jakarta.annotation.PostConstruct;
+
@SpringBootApplication
@EnableConfigurationProperties
-@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" })
+@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"})
@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository",
"com.expediagroup.beekeeper.vacuum.repository" })
public class TestApplication {
diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java
index 881aba7e..fce3eb0f 100644
--- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java
+++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java
@@ -21,6 +21,8 @@
import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED;
import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED;
+import static java.time.temporal.ChronoUnit.MICROS;
+
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;
@@ -109,7 +111,7 @@ public void notNullableField() {
}
private HousekeepingPath createEntityHousekeepingPath() {
- LocalDateTime creationTimestamp = LocalDateTime.now(ZoneId.of("UTC"));
+ LocalDateTime creationTimestamp = LocalDateTime.now(ZoneId.of("UTC")).truncatedTo(MICROS);
return HousekeepingPath
.builder()
.path("path")
diff --git a/pom.xml b/pom.xml
index 4fce8c0a..1e5971a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -40,25 +40,27 @@