From b6392ed7decf3475df2db0b5ea9303d87d02595d Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Wed, 18 Mar 2026 17:18:09 -0500 Subject: [PATCH 01/12] chore: upgrade to Java 21 and Spring Boot 3.2.12 - Bump Java source/target/release to 21 across all modules - Upgrade Spring Boot from 2.7.9 to 3.2.12 - Migrate javax.* to jakarta.* (persistence, servlet, annotation) - Add CrudRepository to repository interfaces (Spring Data 3.x breaking change) - Update springdoc-openapi from 1.x to 2.x (GroupedOpenApi, ParameterObject) - Fix Testcontainers LocalStack API: getEndpointOverride() replaces getEndpointConfiguration() - Fix Awaitility 4.x: replace org.awaitility.Duration with java.time.Duration - Fix Hadoop+Java21: disable FileSystem cache to avoid UserGroupInformation incompatibility - Add JVM --add-opens flags for Hadoop/reflection compatibility in tests - Exclude JPA auto-configuration from @WebMvcTest context in API tests Co-Authored-By: Claude Sonnet 4.6 --- beekeeper-api/pom.xml | 27 ++- .../api/conf/SwaggerConfiguration.java | 21 +-- .../api/controller/BeekeeperController.java | 122 +++++++++---- .../api/error/BeekeeperExceptionHandler.java | 39 ++-- .../HousekeepingMetadataResponse.java | 29 ++- .../beekeeper/api/TestApplication.java | 47 +++-- beekeeper-cleanup/pom.xml | 42 +++++ .../beekeeper/cleanup/TestApplication.java | 25 +-- .../beekeeper/cleanup/aws/S3ClientTest.java | 54 +++--- .../cleanup/aws/S3DryRunPathCleanerTest.java | 56 +++--- .../cleanup/aws/S3PathCleanerTest.java | 88 +++++---- .../aws/S3SentinelFilesCleanerTest.java | 42 +++-- beekeeper-core/pom.xml | 2 - .../core/model/HousekeepingMetadata.java | 35 ++-- .../core/model/HousekeepingPath.java | 51 +++--- .../core/model/PeriodDurationConverter.java | 20 +-- .../core/model/history/BeekeeperHistory.java | 33 ++-- .../BeekeeperHistoryRepository.java | 26 +-- .../HousekeepingMetadataRepository.java | 152 +++++++++------- .../HousekeepingPathRepository.java | 38 ++-- .../beekeeper/core/TestApplication.java | 25 ++- beekeeper-integration-tests/pom.xml | 40 +++++ ...rDryRunMetadataCleanupIntegrationTest.java | 4 +- ...eeperDryRunPathCleanupIntegrationTest.java | 4 +- ...etadataSchedulerApiaryIntegrationTest.java | 4 +- ...ekeeperMetadataCleanupIntegrationTest.java | 4 +- .../BeekeeperPathCleanupIntegrationTest.java | 4 +- ...cedPathSchedulerApiaryIntegrationTest.java | 4 +- beekeeper-metadata-cleanup/pom.xml | 38 +++- ...beekeeper-metadata-cleanup-application.yml | 2 +- .../metadata/cleanup/TestApplication.java | 24 ++- .../service/CleanupServiceSchedulerTest.java | 6 +- .../RepositoryCleanupSchedulerTest.java | 6 +- beekeeper-path-cleanup/pom.xml | 21 ++- .../beekeeper-path-cleanup-application.yml | 2 +- .../path/cleanup/TestApplication.java | 24 ++- .../service/CleanupServiceSchedulerTest.java | 6 +- .../RepositoryCleanupSchedulerTest.java | 6 +- beekeeper-scheduler-apiary/pom.xml | 36 +++- .../apiary/app/SchedulerApiaryRunner.java | 20 +-- ...beekeeper-scheduler-apiary-application.yml | 2 +- .../apiary/app/SchedulerApiaryRunnerTest.java | 8 +- beekeeper-scheduler/pom.xml | 28 +++ beekeeper-vacuum-tool/pom.xml | 124 ++++++++++++- .../beekeeper/vacuum/CommonBeans.java | 54 +++--- .../vacuum/ConsistencyCheckTest.java | 40 +++-- .../beekeeper/vacuum/TestApplication.java | 29 +-- pom.xml | 167 +++++++++++++----- 48 files changed, 1083 insertions(+), 598 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index b6c8cb2b..082467ec 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -11,8 +11,8 @@ ${project.groupId}:${project.artifactId} - 2.18.1 - 1.6.14 + 3.1.0 + 2.3.0 @@ -39,13 +39,12 @@ lombok - mysql - mysql-connector-java - 8.0.17 + com.mysql + mysql-connector-j org.springdoc - springdoc-openapi-ui + springdoc-openapi-starter-webmvc-ui ${springdoc.version} @@ -77,14 +76,16 @@ - pl.project13.maven - git-commit-id-plugin + io.github.git-commit-id + git-commit-id-maven-plugin + 7.0.0 com.google.cloud.tools jib-maven-plugin + eclipse-temurin:21-jre amd64 @@ -98,6 +99,16 @@ USE_CURRENT_TIMESTAMP + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java index 156f4f32..212a05d9 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java @@ -1,22 +1,19 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.conf; - -import org.springdoc.core.GroupedOpenApi; +import org.springdoc.core.models.GroupedOpenApi; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,7 +21,7 @@ import io.swagger.v3.oas.annotations.servers.Server; @Configuration -@OpenAPIDefinition(servers = { @Server(url = "/", description = "Default Server URL") }) +@OpenAPIDefinition(servers = {@Server(url = "/", description = "Default Server URL")}) public class SwaggerConfiguration { @Bean diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 68cc24b5..d7b6a03c 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -1,21 +1,19 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.controller; -import org.springdoc.api.annotations.ParameterObject; +import org.springdoc.core.annotations.ParameterObject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -51,7 +49,9 @@ public BeekeeperController(HousekeepingEntityService housekeepingEntityService) this.housekeepingEntityService = housekeepingEntityService; } - @RequestMapping(value = "/database/{databaseName}/table/{tableName}/metadata", method = RequestMethod.GET) + @RequestMapping( + value = "/database/{databaseName}/table/{tableName}/metadata", + method = RequestMethod.GET) @Parameter(name = "tableName", in = ParameterIn.PATH) @Parameter(name = "databaseName", in = ParameterIn.PATH) @Parameter(name = "path", in = ParameterIn.QUERY) @@ -65,23 +65,49 @@ public BeekeeperController(HousekeepingEntityService housekeepingEntityService) public ResponseEntity> getAllMetadata( @PathVariable String databaseName, @PathVariable String tableName, - @Parameter(hidden = true) @And(value = { - @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), - @Spec(path = "databaseName", pathVars = "databaseName", spec = EqualIgnoreCase.class), - @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), - @Spec(path = "partitionName", params = "partition_name", spec = EqualIgnoreCase.class), - @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), - @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), - @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), - @Spec(path = "cleanupTimestamp", params = "deleted_after", spec = GreaterThan.class), - @Spec(path = "creationTimestamp", params = "registered_before", spec = LessThan.class), - @Spec(path = "creationTimestamp", params = "registered_after", spec = GreaterThan.class) }) Specification spec, + @Parameter(hidden = true) + @And( + value = { + @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), + @Spec( + path = "databaseName", + pathVars = "databaseName", + spec = EqualIgnoreCase.class), + @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), + @Spec( + path = "partitionName", + params = "partition_name", + spec = EqualIgnoreCase.class), + @Spec( + path = "housekeepingStatus", + params = "housekeeping_status", + spec = EqualIgnoreCase.class), + @Spec( + path = "lifecycleType", + params = "lifecycle_type", + spec = EqualIgnoreCase.class), + @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), + @Spec( + path = "cleanupTimestamp", + params = "deleted_after", + spec = GreaterThan.class), + @Spec( + path = "creationTimestamp", + params = "registered_before", + spec = LessThan.class), + @Spec( + path = "creationTimestamp", + params = "registered_after", + spec = GreaterThan.class) + }) + Specification spec, @ParameterObject Pageable pageable) { return ResponseEntity.ok(housekeepingEntityService.getAllMetadata(spec, pageable)); } - - @RequestMapping(value = "/database/{databaseName}/table/{tableName}/unreferenced-paths", method = RequestMethod.GET) + @RequestMapping( + value = "/database/{databaseName}/table/{tableName}/unreferenced-paths", + method = RequestMethod.GET) @Parameter(name = "tableName", in = ParameterIn.PATH) @Parameter(name = "databaseName", in = ParameterIn.PATH) @Parameter(name = "path", in = ParameterIn.QUERY) @@ -95,19 +121,43 @@ public ResponseEntity> getAllMetadata( public ResponseEntity> getAllPaths( @PathVariable String databaseName, @PathVariable String tableName, - @Parameter(hidden = true) @And(value ={ - @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), - @Spec(path = "databaseName", pathVars = "databaseName", spec = EqualIgnoreCase.class), - @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), - @Spec(path = "partitionName", params = "partition_name", spec = EqualIgnoreCase.class), - @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), - @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), - @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), - @Spec(path = "cleanupTimestamp", params = "deleted_after", spec = GreaterThan.class), - @Spec(path = "creationTimestamp", params = "registered_before", spec = LessThan.class), - @Spec(path = "creationTimestamp", params = "registered_after", spec = GreaterThan.class) }) Specification spec, + @Parameter(hidden = true) + @And( + value = { + @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), + @Spec( + path = "databaseName", + pathVars = "databaseName", + spec = EqualIgnoreCase.class), + @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), + @Spec( + path = "partitionName", + params = "partition_name", + spec = EqualIgnoreCase.class), + @Spec( + path = "housekeepingStatus", + params = "housekeeping_status", + spec = EqualIgnoreCase.class), + @Spec( + path = "lifecycleType", + params = "lifecycle_type", + spec = EqualIgnoreCase.class), + @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), + @Spec( + path = "cleanupTimestamp", + params = "deleted_after", + spec = GreaterThan.class), + @Spec( + path = "creationTimestamp", + params = "registered_before", + spec = LessThan.class), + @Spec( + path = "creationTimestamp", + params = "registered_after", + spec = GreaterThan.class) + }) + Specification spec, @ParameterObject Pageable pageable) { return ResponseEntity.ok(housekeepingEntityService.getAllPaths(spec, pageable)); } - } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java index ceea01b5..0c5b2e26 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java @@ -1,54 +1,51 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.error; import java.time.LocalDateTime; -import javax.servlet.http.HttpServletRequest; - import org.springframework.data.mapping.PropertyReferenceException; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; +import jakarta.servlet.http.HttpServletRequest; + @RestControllerAdvice public class BeekeeperExceptionHandler { /** * Handles invalid sort parameters. * - * @param exception the exception is thrown when an invalid property is referenced + * @param exception the exception is thrown when an invalid property is referenced * @param request the HTTP request * @return a ResponseEntity containing the error response */ - @ExceptionHandler(PropertyReferenceException.class) public ResponseEntity handlePropertyReferenceException( PropertyReferenceException exception, HttpServletRequest request) { - ErrorResponse errorResponse = ErrorResponse.builder() - .timestamp(LocalDateTime.now().toString()) - .status(HttpStatus.BAD_REQUEST.value()) - .error(HttpStatus.BAD_REQUEST.getReasonPhrase()) - .message(exception.getMessage()) - .path(request.getRequestURI()) - .build(); + ErrorResponse errorResponse = + ErrorResponse.builder() + .timestamp(LocalDateTime.now().toString()) + .status(HttpStatus.BAD_REQUEST.value()) + .error(HttpStatus.BAD_REQUEST.getReasonPhrase()) + .message(exception.getMessage()) + .path(request.getRequestURI()) + .build(); return new ResponseEntity<>(errorResponse, HttpStatus.BAD_REQUEST); } - } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java index f2510c0f..855bea4a 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java @@ -1,27 +1,24 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.response; import java.time.LocalDateTime; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; - import org.hibernate.annotations.UpdateTimestamp; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Value; @@ -43,12 +40,9 @@ public class HousekeepingMetadataResponse { HousekeepingStatus housekeepingStatus; - @EqualsAndHashCode.Exclude - LocalDateTime creationTimestamp; + @EqualsAndHashCode.Exclude LocalDateTime creationTimestamp; - @EqualsAndHashCode.Exclude - @UpdateTimestamp - LocalDateTime modifiedTimestamp; + @EqualsAndHashCode.Exclude @UpdateTimestamp LocalDateTime modifiedTimestamp; LocalDateTime cleanupTimestamp; @@ -57,5 +51,4 @@ public class HousekeepingMetadataResponse { int cleanupAttempts; String lifecycleType; - } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java index e74e28fe..31f3eb76 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java @@ -1,23 +1,50 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api; +import java.util.List; + import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; +import org.springframework.web.method.support.HandlerMethodArgumentResolver; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import net.kaczmarzyk.spring.data.jpa.web.SpecificationArgumentResolver; + +import com.expediagroup.beekeeper.api.conf.JpaConfiguration; @SpringBootApplication @EnableConfigurationProperties -public class TestApplication {} +@ComponentScan( + excludeFilters = { + @ComponentScan.Filter( + type = FilterType.ASSIGNABLE_TYPE, + classes = BeekeeperApiApplication.class), + @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, classes = JpaConfiguration.class) + }) +public class TestApplication { + + @Bean + public WebMvcConfigurer specificationArgumentResolverConfigurer() { + return new WebMvcConfigurer() { + @Override + public void addArgumentResolvers(List argumentResolvers) { + argumentResolvers.add(new SpecificationArgumentResolver()); + } + }; + } +} diff --git a/beekeeper-cleanup/pom.xml b/beekeeper-cleanup/pom.xml index 58581098..d87edc63 100644 --- a/beekeeper-cleanup/pom.xml +++ b/beekeeper-cleanup/pom.xml @@ -65,6 +65,14 @@ javax.servlet servlet-api + + javax.servlet + javax.servlet-api + + + log4j + log4j + @@ -76,6 +84,26 @@ net.java.dev.jna jna + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + @@ -124,6 +152,20 @@ hadoop-aws 2.9.2 test + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + + diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java index 2d3c3237..dd0bdc0f 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java @@ -1,29 +1,30 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup; import java.util.TimeZone; -import javax.annotation.PostConstruct; - import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; -@SpringBootApplication +import jakarta.annotation.PostConstruct; + +@SpringBootApplication( + exclude = {DataSourceAutoConfiguration.class, HibernateJpaAutoConfiguration.class}) @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") public class TestApplication { diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java index 7c692bd4..0561baff 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2022 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -58,20 +56,24 @@ class S3ClientTest { private AmazonS3 amazonS3; @Rule - public static LocalStackContainer awsContainer = new LocalStackContainer( - DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); + public static LocalStackContainer awsContainer = + new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) + .withServices(S3); + static { awsContainer.start(); } - public static String S3_ENDPOINT = awsContainer.getEndpointConfiguration(S3).getServiceEndpoint(); + + public static String S3_ENDPOINT = awsContainer.getEndpointOverride(S3).toString(); @BeforeEach void setUp() { - amazonS3 = AmazonS3ClientBuilder - .standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")).build(); + amazonS3 = + AmazonS3ClientBuilder.standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) + .build(); amazonS3.createBucket(bucket); emptyBucket(bucket); assertThat(amazonS3.listObjectsV2(bucket).getObjectSummaries()).isEmpty(); @@ -83,16 +85,16 @@ private void emptyBucket(String bucket) { ListObjectsV2Result listObjectsV2Result; String continuationToken = null; do { - ListObjectsV2Request request = new ListObjectsV2Request() - .withBucketName(bucket) - .withContinuationToken(continuationToken); + ListObjectsV2Request request = + new ListObjectsV2Request() + .withBucketName(bucket) + .withContinuationToken(continuationToken); listObjectsV2Result = amazonS3.listObjectsV2(request); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket); - List keys = listObjectsV2Result - .getObjectSummaries() - .stream() - .map(S3ObjectSummary::getKey) - .collect(Collectors.toList()); + List keys = + listObjectsV2Result.getObjectSummaries().stream() + .map(S3ObjectSummary::getKey) + .collect(Collectors.toList()); if (keys.size() > 0) { amazonS3.deleteObjects(deleteObjectsRequest.withKeys(keys.toArray(new String[] {}))); } @@ -210,7 +212,7 @@ void deleteObjectsInDirectory() { } @ParameterizedTest - @ValueSource(ints = { 500, 1000, 1500 }) + @ValueSource(ints = {500, 1000, 1500}) void splitDeleteObjectsInDirectory(final int totalObjects) { ArrayList keys = new ArrayList<>(); for (int i = 1; i <= totalObjects; i++) { diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java index 5fab57af..326c555b 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2024 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -64,32 +62,35 @@ class S3DryRunPathCleanerTest { private S3PathCleaner s3DryRunPathCleaner; @Container - public static LocalStackContainer awsContainer = new LocalStackContainer( - DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); + public static LocalStackContainer awsContainer = + new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) + .withServices(S3); @BeforeEach void setUp() { - String S3_ENDPOINT = awsContainer.getEndpointConfiguration(S3).getServiceEndpoint(); - amazonS3 = AmazonS3ClientBuilder - .standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) - .build(); + String S3_ENDPOINT = awsContainer.getEndpointOverride(S3).toString(); + amazonS3 = + AmazonS3ClientBuilder.standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) + .build(); amazonS3.createBucket(bucket); amazonS3 .listObjectsV2(bucket) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(bucket, object.getKey())); S3Client s3Client = new S3Client(amazonS3, dryRunEnabled); - s3DryRunPathCleaner = new S3PathCleaner(s3Client, new S3SentinelFilesCleaner(s3Client), bytesDeletedReporter); - housekeepingPath = HousekeepingPath - .builder() - .path(absolutePath) - .tableName(tableName) - .databaseName(databaseName) - .creationTimestamp(LocalDateTime.now()) - .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) - .build(); + s3DryRunPathCleaner = + new S3PathCleaner(s3Client, new S3SentinelFilesCleaner(s3Client), bytesDeletedReporter); + housekeepingPath = + HousekeepingPath.builder() + .path(absolutePath) + .tableName(tableName) + .databaseName(databaseName) + .creationTimestamp(LocalDateTime.now()) + .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) + .build(); } @Test @@ -207,6 +208,7 @@ void deleteTable() { @Test void pathDoesNotExist() { - assertThatCode(() -> s3DryRunPathCleaner.cleanupPath(housekeepingPath)).doesNotThrowAnyException(); + assertThatCode(() -> s3DryRunPathCleaner.cleanupPath(housekeepingPath)) + .doesNotThrowAnyException(); } } diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java index 7b00bcb6..b3675e99 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -82,20 +80,24 @@ class S3PathCleanerTest { private S3PathCleaner s3PathCleaner; @Rule - public static LocalStackContainer awsContainer = new LocalStackContainer( - DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); + public static LocalStackContainer awsContainer = + new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) + .withServices(S3); + static { awsContainer.start(); } - public static String S3_ENDPOINT = awsContainer.getEndpointConfiguration(S3).getServiceEndpoint(); + + public static String S3_ENDPOINT = awsContainer.getEndpointOverride(S3).toString(); @BeforeEach void setUp() { - amazonS3 = AmazonS3ClientBuilder - .standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) - .build(); + amazonS3 = + AmazonS3ClientBuilder.standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) + .build(); amazonS3.createBucket(bucket); amazonS3 .listObjectsV2(bucket) @@ -107,14 +109,14 @@ void setUp() { s3PathCleaner = new S3PathCleaner(s3Client, s3SentinelFilesCleaner, bytesDeletedReporter); String tableName = "table"; String databaseName = "database"; - housekeepingPath = HousekeepingPath - .builder() - .path(absolutePath) - .tableName(tableName) - .databaseName(databaseName) - .creationTimestamp(LocalDateTime.now()) - .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) - .build(); + housekeepingPath = + HousekeepingPath.builder() + .path(absolutePath) + .tableName(tableName) + .databaseName(databaseName) + .creationTimestamp(LocalDateTime.now()) + .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) + .build(); } @Test @@ -126,7 +128,8 @@ void typicalForDirectory() { assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isFalse(); - verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter) + .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -152,7 +155,8 @@ void directoryWithTrailingSlash() { assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isFalse(); - verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter) + .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -165,7 +169,8 @@ void typicalForFile() { s3PathCleaner.cleanupPath(housekeepingPath); assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isTrue(); - verify(bytesDeletedReporter).reportTaggable(content.getBytes().length, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter) + .reportTaggable(content.getBytes().length, housekeepingPath, FileSystemType.S3); } @Test @@ -179,7 +184,8 @@ void typicalWithSentinelFile() { assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, partition1Sentinel)).isFalse(); - verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter) + .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -242,7 +248,8 @@ void deleteTable() { assertThat(amazonS3.doesObjectExist(bucket, partition1Sentinel)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, parentSentinelFile)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, tableSentinelFile)).isFalse(); - verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter) + .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -253,7 +260,9 @@ void pathDoesNotExist() { @Test void sentinelFilesCleanerThrowsException() { S3SentinelFilesCleaner s3SentinelFilesCleaner = mock(S3SentinelFilesCleaner.class); - doThrow(IllegalArgumentException.class).when(s3SentinelFilesCleaner).deleteSentinelFiles(absolutePath); + doThrow(IllegalArgumentException.class) + .when(s3SentinelFilesCleaner) + .deleteSentinelFiles(absolutePath); amazonS3.putObject(bucket, key1, content); @@ -339,7 +348,9 @@ void noBytesDeletedMetricWhenFileDeletionFails() { void noBytesDeletedMetricWhenDirectoryDeletionFails() { S3Client mockS3Client = mock(S3Client.class); s3PathCleaner = new S3PathCleaner(mockS3Client, s3SentinelFilesCleaner, bytesDeletedReporter); - doThrow(AmazonServiceException.class).when(mockS3Client).listObjects(bucket, keyRootAsDirectory); + doThrow(AmazonServiceException.class) + .when(mockS3Client) + .listObjects(bucket, keyRootAsDirectory); assertThatExceptionOfType(AmazonServiceException.class) .isThrownBy(() -> s3PathCleaner.cleanupPath(housekeepingPath)); @@ -354,8 +365,11 @@ void reportBytesDeletedWhenDirectoryDeletionPartiallyFails() { s3PathCleaner = new S3PathCleaner(mockS3Client, s3SentinelFilesCleaner, bytesDeletedReporter); assertThatExceptionOfType(BeekeeperException.class) .isThrownBy(() -> s3PathCleaner.cleanupPath(housekeepingPath)) - .withMessage(format("Not all files could be deleted at path \"%s/%s\"; deleted 1/2 objects. " - + "Objects not deleted: 'table/id1/partition_1/file2'.", bucket, keyRootAsDirectory)); + .withMessage( + format( + "Not all files could be deleted at path \"%s/%s\"; deleted 1/2 objects. " + + "Objects not deleted: 'table/id1/partition_1/file2'.", + bucket, keyRootAsDirectory)); verify(bytesDeletedReporter).reportTaggable(100L, housekeepingPath, FileSystemType.S3); } @@ -378,8 +392,10 @@ private void mockOneOutOfTwoObjectsDeleted(AmazonS3 mockAmazonS3) { s3ObjectSummary2.setKey(key2); s3ObjectSummary2.setSize(50L); ListObjectsV2Result listObjectsV2Result = mock(ListObjectsV2Result.class); - when(listObjectsV2Result.getObjectSummaries()).thenReturn(List.of(s3ObjectSummary, s3ObjectSummary2)); - when(mockAmazonS3.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Result); + when(listObjectsV2Result.getObjectSummaries()) + .thenReturn(List.of(s3ObjectSummary, s3ObjectSummary2)); + when(mockAmazonS3.listObjectsV2(any(ListObjectsV2Request.class))) + .thenReturn(listObjectsV2Result); DeleteObjectsResult.DeletedObject deletedObject = new DeleteObjectsResult.DeletedObject(); deletedObject.setKey(key1); when(mockAmazonS3.deleteObjects(any(DeleteObjectsRequest.class))) diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java index afa2b64d..01922796 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2022 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -47,22 +45,27 @@ class S3SentinelFilesCleanerTest { private AmazonS3 amazonS3; @Rule - public static LocalStackContainer awsContainer = new LocalStackContainer( - DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); + public static LocalStackContainer awsContainer = + new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) + .withServices(S3); + static { awsContainer.start(); } - public static String S3_ENDPOINT = awsContainer.getEndpointConfiguration(S3).getServiceEndpoint(); + + public static String S3_ENDPOINT = awsContainer.getEndpointOverride(S3).toString(); @BeforeEach void setUp() { - amazonS3 = AmazonS3ClientBuilder - .standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")).build(); + amazonS3 = + AmazonS3ClientBuilder.standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) + .build(); amazonS3.createBucket(bucket); - amazonS3.listObjectsV2(bucket) + amazonS3 + .listObjectsV2(bucket) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(bucket, object.getKey())); S3Client s3Client = new S3Client(amazonS3, false); @@ -114,7 +117,8 @@ void nonEmptySentinelFile() { @Test void sentinelFileDoesntExist() { amazonS3.putObject(bucket, "table/partition_1", "content"); - assertThatCode(() -> s3SentinelFilesCleaner.deleteSentinelFiles(partition1AbsolutePath)).doesNotThrowAnyException(); + assertThatCode(() -> s3SentinelFilesCleaner.deleteSentinelFiles(partition1AbsolutePath)) + .doesNotThrowAnyException(); } @Test diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index fff8af81..0e8aca83 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -90,13 +90,11 @@ com.h2database h2 - 1.4.199 test org.junit.jupiter junit-jupiter-params - ${junit.jupiter.version} test diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java index a06ee7eb..5a7943fc 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingMetadata.java @@ -1,34 +1,31 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.model; import java.time.LocalDateTime; -import javax.persistence.Column; -import javax.persistence.Convert; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - import org.hibernate.annotations.UpdateTimestamp; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java index b80e895c..78cce53e 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.model; @@ -19,18 +17,17 @@ import java.time.LocalDateTime; -import javax.persistence.Column; -import javax.persistence.Convert; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - import org.hibernate.annotations.UpdateTimestamp; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -83,6 +80,7 @@ public class HousekeepingPath implements HousekeepingEntity { @Column(name = "client_id") private String clientId; + @Column(name = "lifecycle_type", nullable = false) private String lifecycleType; @@ -127,8 +125,18 @@ public MetricTag getMetricTag() { public String toString() { return format( "%s(path=%s, databaseName=%s, tableName=%s, housekeepingStatus=%s, creationTimestamp=%s, modifiedTimestamp=%s, cleanupTimestamp=%s, cleanupDelay=%s, cleanupAttempts=%s, clientId=%s, lifecycleType=%s)", - HousekeepingPath.class.getSimpleName(), path, databaseName, tableName, housekeepingStatus, creationTimestamp, - modifiedTimestamp, cleanupTimestamp, cleanupDelay, cleanupAttempts, clientId, lifecycleType); + HousekeepingPath.class.getSimpleName(), + path, + databaseName, + tableName, + housekeepingStatus, + creationTimestamp, + modifiedTimestamp, + cleanupTimestamp, + cleanupDelay, + cleanupAttempts, + clientId, + lifecycleType); } private LocalDateTime configureCleanupTimestamp() { @@ -140,5 +148,4 @@ private LocalDateTime configureCleanupTimestamp() { } return creationTimestamp.plus(cleanupDelay); } - } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java index 0da210b7..4e228b9a 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDurationConverter.java @@ -1,22 +1,20 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.model; -import javax.persistence.AttributeConverter; -import javax.persistence.Converter; +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; @Converter(autoApply = true) public class PeriodDurationConverter implements AttributeConverter { diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java index 1046126f..bafe31d9 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java @@ -1,29 +1,26 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.model.history; import java.time.LocalDateTime; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -69,8 +66,7 @@ public BeekeeperHistory( String tableName, String lifecycleType, String housekeepingStatus, - String eventDetails - ) { + String eventDetails) { this.id = id; this.eventTimestamp = eventTimestamp; this.databaseName = databaseName; @@ -84,5 +80,4 @@ public BeekeeperHistory( public MetricTag getMetricTag() { return new MetricTag("table", String.join(".", databaseName, tableName)); } - } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java index 5c3dffb3..8fa5042d 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.repository; @@ -19,16 +17,18 @@ import org.springframework.data.domain.Slice; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; -public interface BeekeeperHistoryRepository extends PagingAndSortingRepository, - JpaSpecificationExecutor { +public interface BeekeeperHistoryRepository + extends PagingAndSortingRepository, + CrudRepository, + JpaSpecificationExecutor { @Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle") Slice findRecordsByLifecycleType( - @Param("lifecycle") String lifecycle, - Pageable pageable); + @Param("lifecycle") String lifecycle, Pageable pageable); } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index a6b2ce06..9c3581ef 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.repository; @@ -24,37 +22,43 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; public interface HousekeepingMetadataRepository - extends PagingAndSortingRepository, JpaSpecificationExecutor { + extends PagingAndSortingRepository, + CrudRepository, + JpaSpecificationExecutor { - @Query(value = "from HousekeepingMetadata t where t.cleanupTimestamp <= :instant " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " - + "and t.modifiedTimestamp <= :instant and t.cleanupAttempts < 10 order by t.modifiedTimestamp") + @Query( + value = + "from HousekeepingMetadata t where t.cleanupTimestamp <= :instant " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " + + "and t.modifiedTimestamp <= :instant and t.cleanupAttempts < 10 order by t.modifiedTimestamp") Slice findRecordsForCleanupByModifiedTimestamp( - @Param("instant") LocalDateTime instant, - Pageable pageable); + @Param("instant") LocalDateTime instant, Pageable pageable); /** * Returns the record that matches the inputs given, if there is one. * - * @implNote To get the record for a partitioned table both the input value and the value of the partitionName of the - * current record must be NULL. + * @implNote To get the record for a partitioned table both the input value and the value of the + * partitionName of the current record must be NULL. * @param databaseName * @param tableName * @param partitionName * @return */ - @Query(value = "from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.partitionName = :partitionName or (:partitionName is NULL and t.partitionName is NULL)) " - // To handle special null case - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query( + value = + "from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.partitionName = :partitionName or (:partitionName is NULL and t.partitionName is NULL)) " + // To handle special null case + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") Optional findRecordForCleanupByDbTableAndPartitionName( @Param("databaseName") String databaseName, @Param("tableName") String tableName, @@ -67,83 +71,94 @@ Optional findRecordForCleanupByDbTableAndPartitionName( * @param tableName * @return */ - @Query(value = "select max(cleanupTimestamp) from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query( + value = + "select max(cleanupTimestamp) from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") LocalDateTime findMaximumCleanupTimestampForDbAndTable( - @Param("databaseName") String databaseName, - @Param("tableName") String tableName); + @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method returns the count of all records for a database and table name pair where the partitionName is not - * null. + * This method returns the count of all records for a database and table name pair where the + * partitionName is not null. * * @param databaseName * @param tableName * @return A count of the number of partitions on this table. */ - @Query(value = "select count(partitionName) from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query( + value = + "select count(partitionName) from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") Long countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull( - @Param("databaseName") String databaseName, - @Param("tableName") String tableName); + @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method is used for dry runs since the entries are not being updated. It counts the number of partitions on a - * table which have not yet expired, i.e. they will not be cleaned up in this instant. + * This method is used for dry runs since the entries are not being updated. It counts the number + * of partitions on a table which have not yet expired, i.e. they will not be cleaned up in this + * instant. * * @param instant * @param databaseName * @param tableName * @return A count of the number of existing partitions on this table */ - @Query(value = "select count(partitionName) from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " - + "and t.cleanupTimestamp >= :instant") + @Query( + value = + "select count(partitionName) from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " + + "and t.cleanupTimestamp >= :instant") Long countRecordsForDryRunWherePartitionIsNotNullOrExpired( @Param("instant") LocalDateTime instant, @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method deletes the rows for scheduled or failed partitions for the specified {@code databaseName} and - * {@code tableName}. + * This method deletes the rows for scheduled or failed partitions for the specified {@code + * databaseName} and {@code tableName}. * * @param databaseName * @param tableName */ @Modifying - @Query(value = "delete from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and t.partitionName is not NULL " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query( + value = + "delete from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and t.partitionName is not NULL " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") void deleteScheduledOrFailedPartitionRecordsForTable( - @Param("databaseName") String databaseName, - @Param("tableName") String tableName); + @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method returns all table records where the partition name is NULL and the status is `SCHEDULED` or `FAILED` + * This method returns all table records where the partition name is NULL and the status is + * `SCHEDULED` or `FAILED` */ - @Query(value = "from HousekeepingMetadata t " - + "where t.partitionName is NULL " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query( + value = + "from HousekeepingMetadata t " + + "where t.partitionName is NULL " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") List findActiveTables(); /** - * This method deletes the rows which have "DELETED" or "DISABLED" status and are older than the specified - * {@code instant}. + * This method deletes the rows which have "DELETED" or "DISABLED" status and are older than the + * specified {@code instant}. * * @param instant */ @Modifying - @Query(value = "delete from HousekeepingMetadata t where t.cleanupTimestamp < :instant " - + "and (t.housekeepingStatus = 'DELETED' or t.housekeepingStatus = 'DISABLED')") + @Query( + value = + "delete from HousekeepingMetadata t where t.cleanupTimestamp < :instant " + + "and (t.housekeepingStatus = 'DELETED' or t.housekeepingStatus = 'DISABLED')") void cleanUpOldDeletedRecords(@Param("instant") LocalDateTime instant); /** @@ -153,12 +168,13 @@ void deleteScheduledOrFailedPartitionRecordsForTable( * @param tableName * @return List of records that match the inputs given. */ - @Query(value = "from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and t.partitionName IS NOT NULL " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query( + value = + "from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and t.partitionName IS NOT NULL " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") List findRecordsForCleanupByDbAndTableName( - @Param("databaseName") String databaseName, - @Param("tableName") String tableName); + @Param("databaseName") String databaseName, @Param("tableName") String tableName); } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java index 2e28e858..4cb8639a 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core.repository; @@ -22,6 +20,7 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @@ -30,15 +29,22 @@ @Repository public interface HousekeepingPathRepository - extends PagingAndSortingRepository, JpaSpecificationExecutor { + extends PagingAndSortingRepository, + CrudRepository, + JpaSpecificationExecutor { - @Query(value = "from HousekeepingPath p where p.cleanupTimestamp <= :instant " - + "and (p.housekeepingStatus = 'SCHEDULED' or p.housekeepingStatus = 'FAILED') " - + "and p.modifiedTimestamp <= :instant and p.cleanupAttempts < 10") - Slice findRecordsForCleanup(@Param("instant") LocalDateTime instant, Pageable pageable); + @Query( + value = + "from HousekeepingPath p where p.cleanupTimestamp <= :instant " + + "and (p.housekeepingStatus = 'SCHEDULED' or p.housekeepingStatus = 'FAILED') " + + "and p.modifiedTimestamp <= :instant and p.cleanupAttempts < 10") + Slice findRecordsForCleanup( + @Param("instant") LocalDateTime instant, Pageable pageable); @Modifying - @Query(value = "delete from HousekeepingPath p where p.cleanupTimestamp < :instant " - + "and p.housekeepingStatus = 'DELETED'") + @Query( + value = + "delete from HousekeepingPath p where p.cleanupTimestamp < :instant " + + "and p.housekeepingStatus = 'DELETED'") void cleanUpOldDeletedRecords(@Param("instant") LocalDateTime instant); } diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java index a4302a61..09c4c013 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java @@ -1,40 +1,37 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.core; import java.util.TimeZone; -import javax.annotation.PostConstruct; - import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import jakarta.annotation.PostConstruct; + @SpringBootApplication @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) +@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) public class TestApplication { @PostConstruct void started() { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); } - } diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 20065a7c..3265a4f3 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -34,6 +34,18 @@ org.apache.hive hive-service-rpc + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + log4j + log4j + @@ -53,6 +65,18 @@ org.apache.hive hive-service-rpc + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + log4j + log4j + @@ -74,6 +98,10 @@ slf4j-log4j12 org.slf4j + + log4j + log4j + @@ -128,6 +156,18 @@ org.eclipse.jetty.aggregate jetty-all + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + log4j + log4j + diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java index dd1c80b0..e6bd6518 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.thrift.TException; -import org.awaitility.Duration; +import java.time.Duration; import org.junit.Rule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -179,7 +179,7 @@ void setup() { .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperMetadataCleanup.main(new String[] {})); - await().atMost(Duration.ONE_MINUTE).until(BeekeeperMetadataCleanup::isRunning); + await().atMost(Duration.ofMinutes(1)).until(BeekeeperMetadataCleanup::isRunning); // clear all logs before asserting them appender.clear(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java index 63d2e443..3064bcda 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java @@ -29,7 +29,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.awaitility.Duration; +import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -115,7 +115,7 @@ public void setup() { .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {})); - await().atMost(Duration.ONE_MINUTE) + await().atMost(Duration.ofMinutes(1)) .until(BeekeeperPathCleanup::isRunning); // clear all logs before asserting them diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index 398033b4..ca90dc39 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -46,7 +46,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.awaitility.Duration; +import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -167,7 +167,7 @@ public void setup() { amazonS3.listObjectsV2(BUCKET).getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); - await().atMost(Duration.ONE_MINUTE).until(BeekeeperSchedulerApiary::isRunning); + await().atMost(Duration.ofMinutes(1)).until(BeekeeperSchedulerApiary::isRunning); } @AfterEach diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index 4ec80b7c..79728afa 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -50,7 +50,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.thrift.TException; -import org.awaitility.Duration; +import java.time.Duration; import org.junit.Rule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -188,7 +188,7 @@ public void setup() { .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperMetadataCleanup.main(new String[] {})); - await().atMost(Duration.ONE_MINUTE).until(BeekeeperMetadataCleanup::isRunning); + await().atMost(Duration.ofMinutes(1)).until(BeekeeperMetadataCleanup::isRunning); } @AfterEach diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index 64309c55..f392d148 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -36,7 +36,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.awaitility.Duration; +import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -117,7 +117,7 @@ public void setup() { .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {})); - await().atMost(Duration.ONE_MINUTE) + await().atMost(Duration.ofMinutes(1)) .until(BeekeeperPathCleanup::isRunning); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 67a2c39d..efc53065 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -41,7 +41,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.awaitility.Duration; +import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -114,7 +114,7 @@ public void setup() { amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE))); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); - await().atMost(Duration.ONE_MINUTE).until(BeekeeperSchedulerApiary::isRunning); + await().atMost(Duration.ofMinutes(1)).until(BeekeeperSchedulerApiary::isRunning); } @AfterEach diff --git a/beekeeper-metadata-cleanup/pom.xml b/beekeeper-metadata-cleanup/pom.xml index 279c8a5f..ac1c7086 100644 --- a/beekeeper-metadata-cleanup/pom.xml +++ b/beekeeper-metadata-cleanup/pom.xml @@ -37,9 +37,13 @@ - mysql - mysql-connector-java - 8.0.17 + com.h2database + h2 + test + + + com.mysql + mysql-connector-j org.springframework.boot @@ -76,6 +80,18 @@ javax.servlet servlet-api + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + @@ -133,14 +149,16 @@ - pl.project13.maven - git-commit-id-plugin + io.github.git-commit-id + git-commit-id-maven-plugin + 7.0.0 com.google.cloud.tools jib-maven-plugin + eclipse-temurin:21-jre amd64 @@ -154,6 +172,16 @@ USE_CURRENT_TIMESTAMP + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + diff --git a/beekeeper-metadata-cleanup/src/main/resources/beekeeper-metadata-cleanup-application.yml b/beekeeper-metadata-cleanup/src/main/resources/beekeeper-metadata-cleanup-application.yml index f0c7ae1f..0fd60f78 100644 --- a/beekeeper-metadata-cleanup/src/main/resources/beekeeper-metadata-cleanup-application.yml +++ b/beekeeper-metadata-cleanup/src/main/resources/beekeeper-metadata-cleanup-application.yml @@ -9,7 +9,7 @@ spring: hibernate: ddl-auto: validate properties.hibernate: - dialect: org.hibernate.dialect.MySQL8Dialect + dialect: org.hibernate.dialect.MySQLDialect properties: cleanup-page-size: 500 dry-run-enabled: false diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java index 96d4a8e9..439687e1 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java @@ -1,35 +1,33 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.metadata.cleanup; import java.util.TimeZone; -import javax.annotation.PostConstruct; - import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import jakarta.annotation.PostConstruct; + @SpringBootApplication @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) +@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) public class TestApplication { @PostConstruct diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java index 0757d3ec..1fc99238 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/CleanupServiceSchedulerTest.java @@ -21,9 +21,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; +import java.time.Duration; import java.time.Instant; -import org.awaitility.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -58,7 +58,7 @@ class CleanupServiceSchedulerTest { @Test void typical() { - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any())); } @@ -66,7 +66,7 @@ void typical() { void cleanupServiceException() { doThrow(BeekeeperException.class).when(cleanupService) .cleanUp(any(Instant.class)); - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any())); } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/RepositoryCleanupSchedulerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/RepositoryCleanupSchedulerTest.java index d84ea203..28b735b3 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/RepositoryCleanupSchedulerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/RepositoryCleanupSchedulerTest.java @@ -21,9 +21,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; +import java.time.Duration; import java.time.Instant; -import org.awaitility.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -55,7 +55,7 @@ class RepositoryCleanupSchedulerTest { @Test void typical() { - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(repositoryCleanupService, atLeast(2)).cleanUp(any())); } @@ -63,7 +63,7 @@ void typical() { void cleanupServiceException() { doThrow(BeekeeperException.class).when(repositoryCleanupService) .cleanUp(any(Instant.class)); - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(repositoryCleanupService, atLeast(2)).cleanUp(any())); } } diff --git a/beekeeper-path-cleanup/pom.xml b/beekeeper-path-cleanup/pom.xml index 99015d6f..d8b41fe7 100644 --- a/beekeeper-path-cleanup/pom.xml +++ b/beekeeper-path-cleanup/pom.xml @@ -39,9 +39,8 @@ h2 - mysql - mysql-connector-java - 8.0.17 + com.mysql + mysql-connector-j org.springframework.boot @@ -113,14 +112,16 @@ - pl.project13.maven - git-commit-id-plugin + io.github.git-commit-id + git-commit-id-maven-plugin + 7.0.0 com.google.cloud.tools jib-maven-plugin + eclipse-temurin:21-jre amd64 @@ -134,6 +135,16 @@ USE_CURRENT_TIMESTAMP + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + diff --git a/beekeeper-path-cleanup/src/main/resources/beekeeper-path-cleanup-application.yml b/beekeeper-path-cleanup/src/main/resources/beekeeper-path-cleanup-application.yml index fab3d2d3..43cdb976 100644 --- a/beekeeper-path-cleanup/src/main/resources/beekeeper-path-cleanup-application.yml +++ b/beekeeper-path-cleanup/src/main/resources/beekeeper-path-cleanup-application.yml @@ -9,7 +9,7 @@ spring: hibernate: ddl-auto: validate properties.hibernate: - dialect: org.hibernate.dialect.MySQL8Dialect + dialect: org.hibernate.dialect.MySQLDialect properties: cleanup-page-size: 500 dry-run-enabled: false diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java index 255a07f2..6a3afe68 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java @@ -1,35 +1,33 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.path.cleanup; import java.util.TimeZone; -import javax.annotation.PostConstruct; - import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.ComponentScan; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import jakarta.annotation.PostConstruct; + @SpringBootApplication @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) +@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) public class TestApplication { @PostConstruct diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java index 85390fa0..7807da61 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/CleanupServiceSchedulerTest.java @@ -21,9 +21,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; +import java.time.Duration; import java.time.Instant; -import org.awaitility.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -60,7 +60,7 @@ class CleanupServiceSchedulerTest { @Test void typical() { - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any())); } @@ -68,7 +68,7 @@ void typical() { void cleanupServiceException() { doThrow(BeekeeperException.class).when(cleanupService) .cleanUp(any(Instant.class)); - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(cleanupService, atLeast(2)).cleanUp(any())); } } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java index fafeecbc..6945c34e 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/RepositoryCleanupSchedulerTest.java @@ -21,9 +21,9 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; +import java.time.Duration; import java.time.Instant; -import org.awaitility.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -55,7 +55,7 @@ class RepositoryCleanupSchedulerTest { @Test void typical() { - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(repositoryCleanupService, atLeast(2)).cleanUp(any())); } @@ -63,7 +63,7 @@ void typical() { void cleanupServiceException() { doThrow(BeekeeperException.class).when(repositoryCleanupService) .cleanUp(any(Instant.class)); - await().atMost(Duration.TEN_SECONDS) + await().atMost(Duration.ofSeconds(10)) .untilAsserted(() -> verify(repositoryCleanupService, atLeast(2)).cleanUp(any())); } } diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml index ccd89134..96a69bfa 100644 --- a/beekeeper-scheduler-apiary/pom.xml +++ b/beekeeper-scheduler-apiary/pom.xml @@ -48,10 +48,22 @@ javax.servlet servlet-api + + javax.servlet + javax.servlet-api + org.eclipse.jetty.orbit javax.servlet + + org.slf4j + slf4j-reload4j + + + log4j + log4j + org.apache.geronimo.specs geronimo-jaspic_1.0_spec @@ -67,14 +79,12 @@ - mysql - mysql-connector-java - 8.0.17 + com.mysql + mysql-connector-j org.flywaydb - flyway-core - 5.2.4 + flyway-mysql org.springframework.boot @@ -131,14 +141,16 @@ - pl.project13.maven - git-commit-id-plugin + io.github.git-commit-id + git-commit-id-maven-plugin + 7.0.0 com.google.cloud.tools jib-maven-plugin + eclipse-temurin:21-jre amd64 @@ -152,6 +164,16 @@ USE_CURRENT_TIMESTAMP + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunner.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunner.java index 31e923e4..7234180a 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunner.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunner.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.scheduler.apiary.app; @@ -20,8 +18,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; -import javax.annotation.PreDestroy; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,6 +25,8 @@ import org.springframework.boot.ApplicationRunner; import org.springframework.stereotype.Component; +import jakarta.annotation.PreDestroy; + import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.scheduler.apiary.service.SchedulerApiary; diff --git a/beekeeper-scheduler-apiary/src/main/resources/beekeeper-scheduler-apiary-application.yml b/beekeeper-scheduler-apiary/src/main/resources/beekeeper-scheduler-apiary-application.yml index 089c744f..9b89d0a6 100644 --- a/beekeeper-scheduler-apiary/src/main/resources/beekeeper-scheduler-apiary-application.yml +++ b/beekeeper-scheduler-apiary/src/main/resources/beekeeper-scheduler-apiary-application.yml @@ -9,7 +9,7 @@ spring: hibernate: ddl-auto: validate properties.hibernate: - dialect: org.hibernate.dialect.MySQL8Dialect + dialect: org.hibernate.dialect.MySQLDialect properties: beekeeper: default-cleanup-delay: P3D diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunnerTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunnerTest.java index c57ddd7c..2c749cd1 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunnerTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/app/SchedulerApiaryRunnerTest.java @@ -27,7 +27,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.awaitility.Duration; +import java.time.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -54,7 +54,7 @@ public void init() { @Test public void typicalRun() throws Exception { runRunner(); - await().atMost(Duration.FIVE_SECONDS) + await().atMost(Duration.ofSeconds(5)) .untilAsserted(() -> verify(schedulerApiary, atLeast(1)).scheduleBeekeeperEvent()); destroy(); verify(schedulerApiary).close(); @@ -67,7 +67,7 @@ public void typicalRunWithException() throws Exception { .when(schedulerApiary) .scheduleBeekeeperEvent(); runRunner(); - await().atMost(Duration.FIVE_SECONDS) + await().atMost(Duration.ofSeconds(5)) .untilAsserted(() -> verify(schedulerApiary, atLeast(2)).scheduleBeekeeperEvent()); destroy(); verify(schedulerApiary).close(); @@ -83,7 +83,7 @@ public void typicalRunSchedulerTimesoutOnDestroy() throws Exception { try { runRunner(); - await().atMost(Duration.FIVE_SECONDS) + await().atMost(Duration.ofSeconds(5)) .untilAsserted(() -> verify(schedulerApiary, atLeast(1)).scheduleBeekeeperEvent()); destroy(); fail("Runner should have thrown exception"); diff --git a/beekeeper-scheduler/pom.xml b/beekeeper-scheduler/pom.xml index 091a46af..82b5d9f8 100644 --- a/beekeeper-scheduler/pom.xml +++ b/beekeeper-scheduler/pom.xml @@ -64,6 +64,14 @@ javax.servlet servlet-api + + javax.servlet + javax.servlet-api + + + log4j + log4j + @@ -75,6 +83,26 @@ net.java.dev.jna jna + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + diff --git a/beekeeper-vacuum-tool/pom.xml b/beekeeper-vacuum-tool/pom.xml index 94c71557..f879d77d 100644 --- a/beekeeper-vacuum-tool/pom.xml +++ b/beekeeper-vacuum-tool/pom.xml @@ -48,11 +48,32 @@ com.hotels hcommon-hive-metastore 1.4.1 + + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + + - mysql - mysql-connector-java - 8.0.17 + com.mysql + mysql-connector-j @@ -101,11 +122,55 @@ org.apache.hadoop hadoop-common ${hadoop.version} + + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + + org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + + org.apache.hadoop @@ -116,6 +181,26 @@ com.amazonaws aws-java-sdk-bundle + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + log4j + log4j + @@ -138,32 +223,57 @@ jdk.tools jdk.tools + + javax.servlet + servlet-api + + + javax.servlet + javax.servlet-api + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + org.apache.logging.log4j + log4j-slf4j-impl + + + log4j + log4j + + + com.h2database + h2 + test + org.assertj assertj-core - ${assertj.version} test org.junit.jupiter junit-jupiter - ${junit.jupiter.version} test org.mockito mockito-core - ${mockito.version} test org.mockito mockito-junit-jupiter - ${mockito.version} test diff --git a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java index 98d41423..4af7f3a3 100644 --- a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java +++ b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.vacuum; @@ -37,9 +35,12 @@ import com.hotels.hcommon.hive.metastore.client.supplier.HiveMetaStoreClientSupplier; @Configuration -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository", - "com.expediagroup.beekeeper.vacuum.repository" }) +@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) +@EnableJpaRepositories( + basePackages = { + "com.expediagroup.beekeeper.core.repository", + "com.expediagroup.beekeeper.vacuum.repository" + }) public class CommonBeans { @Bean @@ -56,9 +57,19 @@ public HiveConf hiveConf(@Value("${metastore-uri}") String metastoreUri) { conf.set("fs.s3.impl", S3AFileSystem.class.getName()); conf.set("fs.s3a.impl", S3AFileSystem.class.getName()); conf.set("fs.s3n.impl", S3AFileSystem.class.getName()); - conf.set("fs.s3.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); - conf.set("fs.s3a.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); - conf.set("fs.s3n.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); + conf.set( + "fs.s3.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); + conf.set( + "fs.s3a.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); + conf.set( + "fs.s3n.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); + + // Disable FileSystem cache to avoid UserGroupInformation.getCurrentUser() which is + // incompatible with Java 17+ when the security manager is not installed + conf.setBoolean("fs.file.impl.disable.cache", true); + conf.setBoolean("fs.s3.impl.disable.cache", true); + conf.setBoolean("fs.s3a.impl.disable.cache", true); + conf.setBoolean("fs.s3n.impl.disable.cache", true); return conf; } @@ -70,20 +81,21 @@ public CloseableMetaStoreClientFactory metaStoreClientFactory() { @Bean Supplier metaStoreClientSupplier( - CloseableMetaStoreClientFactory metaStoreClientFactory, - HiveConf hiveConf) { + CloseableMetaStoreClientFactory metaStoreClientFactory, HiveConf hiveConf) { String name = "beekeeper-vacuum-tool"; return new HiveMetaStoreClientSupplier(metaStoreClientFactory, hiveConf, name); } @Bean - public BeekeeperHistoryService beekeeperHistoryService(BeekeeperEventsHistoryRepository repository){ + public BeekeeperHistoryService beekeeperHistoryService( + BeekeeperEventsHistoryRepository repository) { return new BeekeeperHistoryService(repository); } @Bean - public SchedulerService schedulerService(BeekeeperRepository beekeeperRepository, BeekeeperHistoryService beekeeperHistoryService) { - return new UnreferencedHousekeepingPathSchedulerService(beekeeperRepository, beekeeperHistoryService); + public SchedulerService schedulerService( + BeekeeperRepository beekeeperRepository, BeekeeperHistoryService beekeeperHistoryService) { + return new UnreferencedHousekeepingPathSchedulerService( + beekeeperRepository, beekeeperHistoryService); } - } diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java index 7dc1f995..70a05977 100644 --- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java +++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.vacuum; @@ -30,14 +28,17 @@ class ConsistencyCheckTest { @Test void metastorePathsCorrect() { - ConsistencyCheck.checkMetastorePaths(Collections.singleton(new Path("/db/table/snapshot/partition")), 4); + ConsistencyCheck.checkMetastorePaths( + Collections.singleton(new Path("/db/table/snapshot/partition")), 4); } @Test void metastorePathsDepthIncorrect() { assertThatExceptionOfType(IllegalStateException.class) - .isThrownBy(() -> ConsistencyCheck.checkMetastorePaths( - Collections.singleton(new Path("/db/snapshot/partition")), 4)); + .isThrownBy( + () -> + ConsistencyCheck.checkMetastorePaths( + Collections.singleton(new Path("/db/snapshot/partition")), 4)); } @Test @@ -48,21 +49,26 @@ void metastorePathCorrect() { @Test void metastorePathDepthIncorrect() { assertThatExceptionOfType(IllegalStateException.class) - .isThrownBy(() -> ConsistencyCheck.checkMetastorePath( - new Path("/db/snapshot/partition"), 4)); + .isThrownBy( + () -> ConsistencyCheck.checkMetastorePath(new Path("/db/snapshot/partition"), 4)); } @Test void unvisitedPath() throws IOException { - Path nonExistent = new Path("/db/table/snapshot/" + RandomStringUtils.randomAlphanumeric(8) + "/partition"); - FileSystem fs = nonExistent.getFileSystem(new Configuration(false)); + Path nonExistent = + new Path("/db/table/snapshot/" + RandomStringUtils.randomAlphanumeric(8) + "/partition"); + Configuration conf = new Configuration(false); + conf.setBoolean("fs.file.impl.disable.cache", true); + FileSystem fs = nonExistent.getFileSystem(conf); ConsistencyCheck.checkUnvisitedPath(fs, nonExistent); } @Test void unvisitedPathExists() throws IOException { Path exists = new Path("/tmp"); - FileSystem fs = exists.getFileSystem(new Configuration(false)); + Configuration conf = new Configuration(false); + conf.setBoolean("fs.file.impl.disable.cache", true); + FileSystem fs = exists.getFileSystem(conf); assertThatExceptionOfType(IllegalStateException.class) .isThrownBy(() -> ConsistencyCheck.checkUnvisitedPath(fs, exists)); } diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java index e658c020..712f6d8e 100644 --- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java +++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java @@ -1,34 +1,35 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.vacuum; import java.util.TimeZone; -import javax.annotation.PostConstruct; - import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import jakarta.annotation.PostConstruct; + @SpringBootApplication @EnableConfigurationProperties -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository", - "com.expediagroup.beekeeper.vacuum.repository" }) +@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) +@EnableJpaRepositories( + basePackages = { + "com.expediagroup.beekeeper.core.repository", + "com.expediagroup.beekeeper.vacuum.repository" + }) public class TestApplication { @PostConstruct diff --git a/pom.xml b/pom.xml index 4fce8c0a..d30f1e65 100644 --- a/pom.xml +++ b/pom.xml @@ -40,25 +40,27 @@ 1.12.311 3.0 - 3.12.2 - 3.1.6 + 3.25.3 + 4.2.0 expediagroup - 11 - 5.6.3 - 1.2.3 - 2.16.0 - 1.18.20 + 21 + 21 + 21 + 21 + 1.4.14 + 2.22.1 + 1.18.36 3.1.0 - 3.11.2 - 1.27 - 2.7.9 - 5.3.25 - 1.17.6 - 11-slim - 2.17.2 + 2.2 + 3.2.12 + 6.1.15 + 1.19.3 + 21-slim + 2.16.1 2.3.7 1.4.2 3.4.4 + 2.43.0 @@ -69,33 +71,6 @@ ${awaitility.version} test - - org.junit - junit-bom - ${junit.jupiter.version} - pom - import - - - org.springframework - spring-core - ${springframework.version} - - - org.springframework - spring-context - ${springframework.version} - - - org.springframework - spring-beans - ${springframework.version} - - - org.springframework - spring-aop - ${springframework.version} - org.springframework.boot spring-boot-dependencies @@ -158,25 +133,21 @@ org.junit.jupiter junit-jupiter - ${junit.jupiter.version} test org.mockito mockito-core - ${mockito.version} test org.mockito mockito-junit-jupiter - ${mockito.version} test org.assertj assertj-core - ${assertj.version} test @@ -210,7 +181,113 @@ maven-install-plugin 3.0.0-M1 + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 21 + 21 + 21 + UTF-8 + true + + + org.projectlombok + lombok + ${lombok.version} + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + @{argLine} -Duser.timezone=UTC -Djava.security.manager=allow --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/javax.security.auth=ALL-UNNAMED --add-opens java.base/java.security=ALL-UNNAMED + + + + org.jacoco + jacoco-maven-plugin + 0.8.12 + + + com.diffplug.spotless + spotless-maven-plugin + 2.43.0 + + + + 1.19.2 + + + + \#java,\#javax,\#org,\#,\#com,\#com.expedia,\#com.expediagroup,\#com.hotels,java,javax,org,,com,com.expedia,com.expediagroup,com.hotels + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 21 + 21 + 21 + UTF-8 + true + + + org.projectlombok + lombok + ${lombok.version} + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + @{argLine} -Duser.timezone=UTC --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED + + + + org.jacoco + jacoco-maven-plugin + 0.8.12 + + + com.diffplug.spotless + spotless-maven-plugin + 2.43.0 + + + process-sources + + apply + + + + + + + 1.19.2 + + + + \#java,\#javax,\#org,\#,\#com,\#com.expedia,\#com.expediagroup,\#com.hotels,java,javax,org,,com,com.expedia,com.expediagroup,com.hotels + + + + + From 659b58e42c573e51522bf2e526dedc2784a105d5 Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Thu, 19 Mar 2026 13:24:22 -0500 Subject: [PATCH 02/12] Update gha workflows to use java 21 --- .github/workflows/build.yml | 10 +++++----- .github/workflows/main.yml | 8 ++++---- .github/workflows/release.yml | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a9a1b115..fbb46131 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,19 +17,19 @@ on: jobs: deploy: name: Build SNAPSHOT to Sonatype - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.inputs.branch }} - name: Set up JDK - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: - distribution: 'adopt' - java-version: '11' + distribution: 'corretto' + java-version: '21' java-package: jdk server-id: central # Value of the distributionManagement/repository/id field of the pom.xml server-username: SONATYPE_USERNAME # env variable for username in deploy diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e729239d..bdb9330d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,7 @@ jobs: name: Package and run all tests runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Init Coveralls @@ -24,10 +24,10 @@ jobs: fi echo "COVERALLS_SKIP=${COVERALLS_SKIP}" >> $GITHUB_ENV - name: Set up JDK - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: - distribution: 'adopt' - java-version: '11' + distribution: 'corretto' + java-version: '21' java-package: jdk - name: Run Maven Package diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e6349444..afc302c0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,17 +10,17 @@ jobs: steps: - name: Checkout source code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 ref: main # Hardcoded to main branch token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - name: Set up JDK - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: - distribution: 'adopt' - java-version: '11' + distribution: 'corretto' + java-version: '21' java-package: jdk server-id: central # Value of the distributionManagement/repository/id field of the pom.xml server-username: SONATYPE_USERNAME # env variable for username in deploy From 664ab85d103f24d23569c237f635fd19e3c7393b Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Fri, 20 Mar 2026 16:04:10 -0500 Subject: [PATCH 03/12] fix: fix integration tests for Java 21 / Spring Boot 3.2.12 upgrade - Replace JUnit 4 @Rule with @Container on LocalStack S3 containers in BeekeeperMetadataCleanupIntegrationTest and BeekeeperDryRunMetadataCleanupIntegrationTest to fix ExceptionInInitializerError caused by static block running before Testcontainers start - Add properties.sqs.endpoint and properties.sqs.region Spring properties to CommonBeans.messageReader() so the SqsMessageReader uses a custom AmazonSQS client with explicit LocalStack endpoint and matching region (us-west-2), fixing QueueDoesNotExistException caused by LocalStack's region-sensitive SQS queue lookup - Set properties.sqs.region in BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest and BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest to match the region used when creating SQS queues in LocalStack - Apply Spotless formatting to all changed integration test files All 71 integration tests now pass (1 skipped, pre-existing). Co-Authored-By: Claude Sonnet 4.6 --- beekeeper-integration-tests/pom.xml | 14 ++ ...rDryRunMetadataCleanupIntegrationTest.java | 139 ++++++----- ...eeperDryRunPathCleanupIntegrationTest.java | 63 +++-- ...etadataSchedulerApiaryIntegrationTest.java | 161 +++++++------ .../BeekeeperIntegrationTestBase.java | 225 +++++++++++------- ...ekeeperMetadataCleanupIntegrationTest.java | 201 ++++++++++------ .../BeekeeperPathCleanupIntegrationTest.java | 81 ++++--- ...cedPathSchedulerApiaryIntegrationTest.java | 141 +++++++---- .../integration/CommonTestVariables.java | 19 +- .../api/BeekeeperApiIntegrationTest.java | 224 +++++++++++------ .../model/AddPartitionSqsMessage.java | 32 ++- .../model/AlterPartitionSqsMessage.java | 42 ++-- .../model/AlterTableSqsMessage.java | 32 +-- .../model/CreateTableSqsMessage.java | 29 +-- .../model/DropPartitionSqsMessage.java | 31 ++- .../model/DropTableSqsMessage.java | 23 +- .../integration/model/SqsMessage.java | 53 +++-- .../integration/model/SqsMessageTest.java | 78 +++--- .../utils/BeekeeperApiTestClient.java | 76 +++--- .../integration/utils/ContainerTestUtils.java | 43 ++-- .../integration/utils/HiveTestUtils.java | 61 +++-- .../integration/utils/MySqlTestUtils.java | 33 +-- .../integration/utils/RestResponsePage.java | 23 +- .../ResultSetToBeekeeperHistoryMapper.java | 16 +- .../ResultSetToHousekeepingEntityMapper.java | 43 ++-- .../integration/utils/TestAppender.java | 16 +- .../scheduler/apiary/context/CommonBeans.java | 131 +++++----- .../apiary/context/CommonBeansTest.java | 59 +++-- 28 files changed, 1225 insertions(+), 864 deletions(-) diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 3265a4f3..96443448 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -168,8 +168,22 @@ log4j log4j + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.apache.derby + derbytools + 10.16.1.1 + test + org.flywaydb flyway-mysql diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java index e6bd6518..3ba0415a 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -28,6 +26,7 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,8 +37,6 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.thrift.TException; -import java.time.Duration; -import org.junit.Rule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -47,6 +44,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import ch.qos.logback.classic.spi.ILoggingEvent; @@ -81,8 +79,10 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; private static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; private static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; - private static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disableGetObjectMD5Validation"; - private static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disablePutObjectMD5Validation"; + private static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = + "com.amazonaws.services.s3.disableGetObjectMD5Validation"; + private static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = + "com.amazonaws.services.s3.disablePutObjectMD5Validation"; private static final String BUCKET = "test-path-bucket"; private static final String TABLE_DATA = "1\tadam\tlondon\n2\tsusan\tglasgow\n"; @@ -96,47 +96,39 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private static final String PARTITIONED_TABLE_PATH = ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; private static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1"; - private static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; - private static final String PARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE - + "/" - + PARTITIONED_TABLE_NAME - + "/id1"; - - private static final String PARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE - + "/some_location/id1/" - + PARTITION_NAME - + "/file1"; - - private static final String UNPARTITIONED_TABLE_PATH = ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; - private static final String UNPARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE - + "/" - + UNPARTITIONED_TABLE_NAME - + "/id1"; + private static final String PARTITION_PATH = + PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; + private static final String PARTITIONED_TABLE_OBJECT_KEY = + DATABASE_NAME_VALUE + "/" + PARTITIONED_TABLE_NAME + "/id1"; + + private static final String PARTITIONED_OBJECT_KEY = + DATABASE_NAME_VALUE + "/some_location/id1/" + PARTITION_NAME + "/file1"; + + private static final String UNPARTITIONED_TABLE_PATH = + ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; + private static final String UNPARTITIONED_TABLE_OBJECT_KEY = + DATABASE_NAME_VALUE + "/" + UNPARTITIONED_TABLE_NAME + "/id1"; private static final String S3_CLIENT_CLASS_NAME = "S3Client"; private static final String HIVE_CLIENT_CLASS_NAME = "HiveClient"; - @Rule - public static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); - static { - S3_CONTAINER.start(); - } + @Container + private static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); private static AmazonS3 amazonS3; - private static final String S3_ENDPOINT = ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3); private final ExecutorService executorService = Executors.newFixedThreadPool(1); private final TestAppender appender = new TestAppender(); - private static Map metastoreProperties = ImmutableMap - .builder() - .put(ENDPOINT, S3_ENDPOINT) - .put(ACCESS_KEY, S3_ACCESS_KEY) - .put(SECRET_KEY, S3_SECRET_KEY) - .build(); + private Map metastoreProperties = + ImmutableMap.builder() + .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) + .put(ACCESS_KEY, S3_ACCESS_KEY) + .put(SECRET_KEY, S3_SECRET_KEY) + .build(); @RegisterExtension - public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( - DATABASE_NAME_VALUE, metastoreProperties); + public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = + new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, metastoreProperties); private HiveTestUtils hiveTestUtils; private HiveMetaStoreClient metastoreClient; @@ -146,7 +138,8 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, "test"); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, "true"); - System.setProperty(AWS_S3_ENDPOINT_PROPERTY, S3_ENDPOINT); + System.setProperty( + AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); System.setProperty(AWS_DISABLE_GET_VALIDATION_PROPERTY, "true"); System.setProperty(AWS_DISABLE_PUT_VALIDATION_PROPERTY, "true"); @@ -165,7 +158,6 @@ public static void teardown() { System.clearProperty(METASTORE_URI_PROPERTY); amazonS3.shutdown(); - S3_CONTAINER.stop(); } @BeforeEach @@ -193,11 +185,14 @@ void stop() throws InterruptedException { @Test public void dryRunDropUnpartitionedTable() throws TException, SQLException { - hiveTestUtils.createTableWithDeletionProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, true); + hiveTestUtils.createTableWithDeletionProperties( + UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, true); amazonS3.putObject(BUCKET, UNPARTITIONED_TABLE_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLine(UNPARTITIONED_TABLE_OBJECT_KEY)); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> logsContainLine(UNPARTITIONED_TABLE_OBJECT_KEY)); assertHiveClientLogs(1); assertS3ClientLogs(1); @@ -207,7 +202,9 @@ public void dryRunDropUnpartitionedTable() throws TException, SQLException { @Test public void dryRunDropPartitionedTable() throws Exception { - Table table = hiveTestUtils.createTableWithDeletionProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); + Table table = + hiveTestUtils.createTableWithDeletionProperties( + PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -225,14 +222,16 @@ public void dryRunDropPartitionedTable() throws Exception { @Test public void dryRunDontDropPartitionedTable() throws Exception { - Table table = hiveTestUtils.createTableWithDeletionProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); + Table table = + hiveTestUtils.createTableWithDeletionProperties( + PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); - hiveTestUtils - .addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); + hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); String partition2Name = "event_date=2020-01-01/event_hour=1/event_type=B"; String partition2Path = PARTITION_ROOT_PATH + "/" + partition2Name + "/file1"; - String partition2ObjectKey = DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; + String partition2ObjectKey = + DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -240,7 +239,8 @@ public void dryRunDontDropPartitionedTable() throws Exception { insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); insertExpiredMetadata(PARTITION_PATH, PARTITION_NAME); - insertExpiredMetadata(TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); + insertExpiredMetadata( + TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLine(PARTITIONED_OBJECT_KEY)); @@ -254,7 +254,9 @@ public void dryRunDontDropPartitionedTable() throws Exception { @Test public void dryRunMetrics() throws Exception { - Table table = hiveTestUtils.createTableWithDeletionProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); + Table table = + hiveTestUtils.createTableWithDeletionProperties( + PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); @@ -263,7 +265,9 @@ public void dryRunMetrics() throws Exception { insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); insertExpiredMetadata(PARTITION_PATH, PARTITION_NAME); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLine(PARTITION_PATH.replace("s3a://", ""))); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> logsContainLine(PARTITION_PATH.replace("s3a://", ""))); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isTrue(); @@ -272,14 +276,24 @@ public void dryRunMetrics() throws Exception { } private void assertMetrics() { - Set meterRegistry = ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); + Set meterRegistry = + ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach(registry -> { - List meters = registry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name") - .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, - "hive-partition-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, "s3-paths-deleted", "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); - }); + meterRegistry.forEach( + registry -> { + List meters = registry.getMeters(); + assertThat(meters) + .extracting("id", Meter.Id.class) + .extracting("name") + .contains( + "metadata-cleanup-job", + "hive-table-deleted", + "hive-partition-deleted", + "hive-table-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, + "hive-partition-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, + "s3-paths-deleted", + "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); + }); } private boolean logsContainLine(String messageFragment) { @@ -313,5 +327,4 @@ private void assertHiveClientLogs(int expected) { } assertThat(logsFromHiveClient).isEqualTo(expected); } - } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java index 3064bcda..e14165ea 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -24,12 +22,12 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -68,11 +66,13 @@ public class BeekeeperDryRunPathCleanupIntegrationTest extends BeekeeperIntegrat private static final String OBJECT_KEY_ROOT = DB_AND_TABLE_PREFIX + "/id1/partition1"; private static final String OBJECT_KEY1 = DB_AND_TABLE_PREFIX + "/id1/partition1/file1"; private static final String OBJECT_KEY2 = DB_AND_TABLE_PREFIX + "/id1/partition1/file2"; - private static final String OBJECT_KEY_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; + private static final String OBJECT_KEY_SENTINEL = + DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; private static final String ABSOLUTE_PATH = "s3://" + BUCKET + "/" + OBJECT_KEY_ROOT; private static final String OBJECT_KEY_OTHER = DB_AND_TABLE_PREFIX + "/id1/partition10/file1"; - private static final String OBJECT_KEY_OTHER_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; + private static final String OBJECT_KEY_OTHER_SENTINEL = + DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; private static final String SPRING_PROFILES_ACTIVE = "test"; private static final String SCHEDULER_DELAY_MS = "5000"; @@ -83,6 +83,7 @@ public class BeekeeperDryRunPathCleanupIntegrationTest extends BeekeeperIntegrat @Container private static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); + private static AmazonS3 amazonS3; private final ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -93,7 +94,8 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, SPRING_PROFILES_ACTIVE); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, DRY_RUN_ENABLED); - System.setProperty(AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); + System.setProperty( + AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); amazonS3 = ContainerTestUtils.s3Client(S3_CONTAINER, AWS_REGION); amazonS3.createBucket(new CreateBucketRequest(BUCKET, AWS_REGION)); @@ -111,12 +113,12 @@ public static void teardown() { @BeforeEach public void setup() { - amazonS3.listObjectsV2(BUCKET) + amazonS3 + .listObjectsV2(BUCKET) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {})); - await().atMost(Duration.ofMinutes(1)) - .until(BeekeeperPathCleanup::isRunning); + await().atMost(Duration.ofMinutes(1)).until(BeekeeperPathCleanup::isRunning); // clear all logs before asserting them appender.clear(); @@ -137,7 +139,9 @@ public void filesNotDeletedInDirectory() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isTrue(); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY2)).isTrue(); @@ -155,7 +159,9 @@ public void logsForDirectory() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertS3ClientLogs(3); } @@ -184,7 +190,9 @@ public void logsForParentDeletion() throws SQLException { amazonS3.putObject(BUCKET, tableSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(parentSentinel)); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> logsContainLineFromS3Client(parentSentinel)); assertS3ClientLogs(4); } @@ -201,7 +209,9 @@ public void logsForNonEmptyParentDeletion() throws SQLException { amazonS3.putObject(BUCKET, tableSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertS3ClientLogs(3); } @@ -212,7 +222,9 @@ public void metrics() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isTrue(); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_SENTINEL)).isTrue(); @@ -222,8 +234,13 @@ public void metrics() throws SQLException { private boolean assertMetrics() { MeterRegistry meterRegistry = BeekeeperPathCleanup.meterRegistry(); List meters = meterRegistry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name") - .contains("path-cleanup-job", "s3-paths-deleted", "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); + assertThat(meters) + .extracting("id", Meter.Id.class) + .extracting("name") + .contains( + "path-cleanup-job", + "s3-paths-deleted", + "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); return true; } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index ca90dc39..a45d80d5 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -36,6 +34,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Set; @@ -46,7 +45,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -82,10 +80,13 @@ import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension; @Testcontainers -public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { +public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest + extends BeekeeperIntegrationTestBase { protected static final int TIMEOUT = 30; protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; + protected static final String SQS_ENDPOINT_PROPERTY = "properties.sqs.endpoint"; + protected static final String SQS_REGION_PROPERTY = "properties.sqs.region"; protected static final String QUEUE = "apiary-receiver-queue"; protected static final String SCHEDULED_EXPIRED_METRIC = "metadata-scheduled"; @@ -93,7 +94,8 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek protected static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; protected static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; - protected static final String PARTITION_KEYS = "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; + protected static final String PARTITION_KEYS = + "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; protected static final String PARTITION_A_VALUES = "[ \"2020-01-01\", \"0\" ]"; protected static final String PARTITION_B_VALUES = "[ \"2020-01-01\", \"1\" ]"; protected static final String PARTITION_A_NAME = "event_date=2020-01-01/event_hour=0"; @@ -112,37 +114,37 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek @Container protected static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); + @Container protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); protected static AmazonSQS amazonSQS; + protected static String ACTUAL_QUEUE_URL; protected HiveTestUtils hiveTestUtils; protected HiveMetaStoreClient metastoreClient; - static { - S3_CONTAINER.start(); - } - protected static AmazonS3 amazonS3; - private static Map metastoreProperties = ImmutableMap - .builder() - .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) - .put(ACCESS_KEY, S3_ACCESS_KEY) - .put(SECRET_KEY, S3_SECRET_KEY) - .build(); + private Map metastoreProperties = + ImmutableMap.builder() + .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) + .put(ACCESS_KEY, S3_ACCESS_KEY) + .put(SECRET_KEY, S3_SECRET_KEY) + .build(); @RegisterExtension - protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( - DATABASE_NAME_VALUE, metastoreProperties); + protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = + new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, metastoreProperties); @BeforeAll public static void init() { - String queueUrl = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); - System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl); - amazonSQS = ContainerTestUtils.sqsClient(SQS_CONTAINER, AWS_REGION); + ACTUAL_QUEUE_URL = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); amazonSQS.createQueue(QUEUE); + System.setProperty(APIARY_QUEUE_URL_PROPERTY, ACTUAL_QUEUE_URL); + System.setProperty( + SQS_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(SQS_CONTAINER, SQS)); + System.setProperty(SQS_REGION_PROPERTY, AWS_REGION); amazonS3 = ContainerTestUtils.s3Client(S3_CONTAINER, AWS_REGION); amazonS3.createBucket(new CreateBucketRequest(BUCKET, AWS_REGION)); @@ -151,6 +153,8 @@ public static void init() { @AfterAll public static void teardown() { System.clearProperty(APIARY_QUEUE_URL_PROPERTY); + System.clearProperty(SQS_ENDPOINT_PROPERTY); + System.clearProperty(SQS_REGION_PROPERTY); System.clearProperty(METASTORE_URI_PROPERTY); amazonSQS.shutdown(); @@ -163,8 +167,10 @@ public void setup() { metastoreClient = thriftHiveMetaStore.client(); hiveTestUtils = new HiveTestUtils(metastoreClient); - amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE))); - amazonS3.listObjectsV2(BUCKET).getObjectSummaries() + amazonSQS.purgeQueue(new PurgeQueueRequest(ACTUAL_QUEUE_URL)); + amazonS3 + .listObjectsV2(BUCKET) + .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); await().atMost(Duration.ofMinutes(1)).until(BeekeeperSchedulerApiary::isRunning); @@ -177,7 +183,8 @@ public void stop() throws InterruptedException { } @Test - public void expiredMetadataCreateTableEvent() throws SQLException, IOException, URISyntaxException { + public void expiredMetadataCreateTableEvent() + throws SQLException, IOException, URISyntaxException { CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); @@ -188,7 +195,8 @@ public void expiredMetadataCreateTableEvent() throws SQLException, IOException, } @Test - public void expiredMetadataAlterTableEvent() throws SQLException, IOException, URISyntaxException { + public void expiredMetadataAlterTableEvent() + throws SQLException, IOException, URISyntaxException { insertExpiredMetadata(LOCATION_A + "-old", null); AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage(LOCATION_A, true); @@ -201,9 +209,10 @@ public void expiredMetadataAlterTableEvent() throws SQLException, IOException, U } @Test - public void expiredMetadataAddPartitionEvent() throws SQLException, IOException, URISyntaxException { - AddPartitionSqsMessage addPartitionSqsMessage = new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, - PARTITION_A_VALUES, true); + public void expiredMetadataAddPartitionEvent() + throws SQLException, IOException, URISyntaxException { + AddPartitionSqsMessage addPartitionSqsMessage = + new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage.getFormattedString())); // creating entry for table @@ -218,11 +227,12 @@ public void expiredMetadataAddPartitionEvent() throws SQLException, IOException, } @Test - public void expiredMetadataMultipleAddPartitionEvents() throws SQLException, IOException, URISyntaxException { - AddPartitionSqsMessage addPartitionSqsMessage = new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, - PARTITION_A_VALUES, true); - AddPartitionSqsMessage addPartitionSqsMessage2 = new AddPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, - PARTITION_B_VALUES, true); + public void expiredMetadataMultipleAddPartitionEvents() + throws SQLException, IOException, URISyntaxException { + AddPartitionSqsMessage addPartitionSqsMessage = + new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); + AddPartitionSqsMessage addPartitionSqsMessage2 = + new AddPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, PARTITION_B_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage.getFormattedString())); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage2.getFormattedString())); @@ -239,11 +249,12 @@ public void expiredMetadataMultipleAddPartitionEvents() throws SQLException, IOE } @Test - public void expiredMetadataAlterPartitionTableEvent() throws SQLException, IOException, URISyntaxException { + public void expiredMetadataAlterPartitionTableEvent() + throws SQLException, IOException, URISyntaxException { insertExpiredMetadata(LOCATION_A + "-old", PARTITION_A_NAME); - AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, - PARTITION_A_VALUES, true); + AlterPartitionSqsMessage alterPartitionSqsMessage = + new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUpdatedExpiredMetadataRowCount() == 1); @@ -253,14 +264,15 @@ public void expiredMetadataAlterPartitionTableEvent() throws SQLException, IOExc } @Test - public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLException, IOException, URISyntaxException { + public void expiredMetadataMultipleAlterPartitionTableEvents() + throws SQLException, IOException, URISyntaxException { insertExpiredMetadata(LOCATION_A + "-old", PARTITION_A_NAME); insertExpiredMetadata(LOCATION_B + "-old", PARTITION_B_NAME); - AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, - PARTITION_A_VALUES, true); - AlterPartitionSqsMessage alterPartitionSqsMessage2 = new AlterPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, - PARTITION_B_VALUES, true); + AlterPartitionSqsMessage alterPartitionSqsMessage = + new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); + AlterPartitionSqsMessage alterPartitionSqsMessage2 = + new AlterPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, PARTITION_B_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage2.getFormattedString())); @@ -272,7 +284,8 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLExcepti } @Test - public void expiredMetadataCreateIcebergTableEvent() throws SQLException, IOException, URISyntaxException { + public void expiredMetadataCreateIcebergTableEvent() + throws SQLException, IOException, URISyntaxException { CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true, true); amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); @@ -287,7 +300,9 @@ public void testEventAddedToHistoryTable() throws SQLException, IOException, URI CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); List beekeeperHistory = getBeekeeperHistory(EXPIRED); BeekeeperHistory history = beekeeperHistory.get(0); @@ -306,8 +321,7 @@ void scheduleExistingPartitionsWhenPropertiesExpireInTable() throws Exception { AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage(LOCATION_A, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 3); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 3); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); List expiredMetadata = getExpiredMetadata(); @@ -321,13 +335,14 @@ void scheduleMissingPartitionsWhenPropertiesExpireInTable() throws Exception { hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); insertExpiredMetadata(PARTITION_ROOT_PATH, null); - insertExpiredMetadata(PARTITION_ROOT_PATH + PARTITION_A_NAME + "/event_type=C", PARTITION_A_NAME + "/event_type=C"); + insertExpiredMetadata( + PARTITION_ROOT_PATH + PARTITION_A_NAME + "/event_type=C", + PARTITION_A_NAME + "/event_type=C"); AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage(PARTITION_ROOT_PATH, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 4); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 4); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); List expiredMetadata = getExpiredMetadata(); @@ -347,30 +362,34 @@ public void healthCheck() { public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } protected SendMessageRequest sendMessageRequest(String payload) { - return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload); + return new SendMessageRequest(ACTUAL_QUEUE_URL, payload); } - protected void assertExpiredMetadata(HousekeepingMetadata actual, String expectedPath, String partitionName) { + protected void assertExpiredMetadata( + HousekeepingMetadata actual, String expectedPath, String partitionName) { assertHousekeepingMetadata(actual, expectedPath, partitionName); assertMetrics(); } public void assertHousekeepingMetadata( - HousekeepingMetadata actual, - String expectedPath, - String expectedPartitionName) { + HousekeepingMetadata actual, String expectedPath, String expectedPartitionName) { assertThat(actual.getPath()).isEqualTo(expectedPath); assertThat(actual.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); assertThat(actual.getTableName()).isEqualTo(TABLE_NAME_VALUE); assertThat(actual.getPartitionName()).isEqualTo(expectedPartitionName); assertThat(actual.getHousekeepingStatus()).isEqualTo(SCHEDULED); - assertThat(actual.getCreationTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); - assertThat(actual.getModifiedTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); - assertThat(actual.getCleanupTimestamp()).isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); + assertThat(actual.getCreationTimestamp()) + .isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); + assertThat(actual.getModifiedTimestamp()) + .isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); + assertThat(actual.getCleanupTimestamp()) + .isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); assertThat(actual.getCleanupDelay()).isEqualTo(PeriodDuration.parse(SHORT_CLEANUP_DELAY_VALUE)); assertThat(actual.getCleanupAttempts()).isEqualTo(CLEANUP_ATTEMPTS_VALUE); assertThat(actual.getClientId()).isEqualTo(CLIENT_ID_VALUE); @@ -378,12 +397,16 @@ public void assertHousekeepingMetadata( } public void assertMetrics() { - Set meterRegistry = ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()) - .getRegistries(); + Set meterRegistry = + ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach(registry -> { - List meters = registry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name").contains(SCHEDULED_EXPIRED_METRIC); - }); + meterRegistry.forEach( + registry -> { + List meters = registry.getMeters(); + assertThat(meters) + .extracting("id", Meter.Id.class) + .extracting("name") + .contains(SCHEDULED_EXPIRED_METRIC); + }); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index de63bbad..105eb30f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -94,30 +92,64 @@ public abstract class BeekeeperIntegrationTestBase { // FIELDS TO INSERT INTO BEEKEEPER TABLES private Long id = 1L; - private static final String HOUSEKEEPING_PATH_FIELDS = String - .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, - CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, - CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); - private static final String HOUSEKEEPING_METADATA_FIELDS = String - .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, PARTITION_NAME_FIELD, - HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, - CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); - private static final String BEEKEEPER_HISTORY_FIELDS = String.join(",", ID_FIELD, EVENT_TIMESTAMP_FIELD, - DATABASE_NAME_FIELD, TABLE_NAME_FIELD, LIFECYCLE_TYPE_FIELD, HOUSEKEEPING_STATUS_FIELD, EVENT_DETAILS_FIELD); - private static final String LIFE_CYCLE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; - private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE " - + LIFECYCLE_TYPE_FIELD - + " = '%s'" - + " AND " - + MODIFIED_TIMESTAMP_FIELD - + " > " - + CREATION_TIMESTAMP_FIELD - + " ORDER BY " - + PATH_FIELD; + private static final String HOUSEKEEPING_PATH_FIELDS = + String.join( + ",", + ID_FIELD, + PATH_FIELD, + DATABASE_NAME_FIELD, + TABLE_NAME_FIELD, + HOUSEKEEPING_STATUS_FIELD, + CREATION_TIMESTAMP_FIELD, + MODIFIED_TIMESTAMP_FIELD, + CLEANUP_TIMESTAMP_FIELD, + CLEANUP_DELAY_FIELD, + CLEANUP_ATTEMPTS_FIELD, + CLIENT_ID_FIELD, + LIFECYCLE_TYPE_FIELD); + private static final String HOUSEKEEPING_METADATA_FIELDS = + String.join( + ",", + ID_FIELD, + PATH_FIELD, + DATABASE_NAME_FIELD, + TABLE_NAME_FIELD, + PARTITION_NAME_FIELD, + HOUSEKEEPING_STATUS_FIELD, + CREATION_TIMESTAMP_FIELD, + MODIFIED_TIMESTAMP_FIELD, + CLEANUP_TIMESTAMP_FIELD, + CLEANUP_DELAY_FIELD, + CLEANUP_ATTEMPTS_FIELD, + CLIENT_ID_FIELD, + LIFECYCLE_TYPE_FIELD); + private static final String BEEKEEPER_HISTORY_FIELDS = + String.join( + ",", + ID_FIELD, + EVENT_TIMESTAMP_FIELD, + DATABASE_NAME_FIELD, + TABLE_NAME_FIELD, + LIFECYCLE_TYPE_FIELD, + HOUSEKEEPING_STATUS_FIELD, + EVENT_DETAILS_FIELD); + private static final String LIFE_CYCLE_FILTER = + "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; + private static final String LIFE_CYCLE_AND_UPDATE_FILTER = + "WHERE " + + LIFECYCLE_TYPE_FIELD + + " = '%s'" + + " AND " + + MODIFIED_TIMESTAMP_FIELD + + " > " + + CREATION_TIMESTAMP_FIELD + + " ORDER BY " + + PATH_FIELD; // MySQL DB CONTAINER AND UTILS @Container private static final MySQLContainer MY_SQL_CONTAINER = ContainerTestUtils.mySqlContainer(); + protected static MySqlTestUtils mySQLTestUtils; protected final ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -164,75 +196,105 @@ protected void insertUnreferencedPath(String path) throws SQLException { } protected void insertUnreferencedPath(HousekeepingPath housekeepingPath) throws SQLException { - housekeepingPath.setCleanupTimestamp(housekeepingPath.getCleanupTimestamp().minus(Duration.ofDays(1))); - String values = Stream - .of(housekeepingPath.getId().toString(), housekeepingPath.getPath(), housekeepingPath.getDatabaseName(), - housekeepingPath.getTableName(), housekeepingPath.getHousekeepingStatus().toString(), - housekeepingPath.getCreationTimestamp().toString(), housekeepingPath.getModifiedTimestamp().toString(), - housekeepingPath.getCleanupTimestamp().toString(), housekeepingPath.getCleanupDelay().toString(), - String.valueOf(housekeepingPath.getCleanupAttempts()), housekeepingPath.getClientId(), - housekeepingPath.getLifecycleType()) - .map(s -> s == null ? null : "\"" + s + "\"") - .collect(Collectors.joining(", ")); - - mySQLTestUtils - .insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, HOUSEKEEPING_PATH_FIELDS, values); + housekeepingPath.setCleanupTimestamp( + housekeepingPath.getCleanupTimestamp().minus(Duration.ofDays(1))); + String values = + Stream.of( + housekeepingPath.getId().toString(), + housekeepingPath.getPath(), + housekeepingPath.getDatabaseName(), + housekeepingPath.getTableName(), + housekeepingPath.getHousekeepingStatus().toString(), + housekeepingPath.getCreationTimestamp().toString(), + housekeepingPath.getModifiedTimestamp().toString(), + housekeepingPath.getCleanupTimestamp().toString(), + housekeepingPath.getCleanupDelay().toString(), + String.valueOf(housekeepingPath.getCleanupAttempts()), + housekeepingPath.getClientId(), + housekeepingPath.getLifecycleType()) + .map(s -> s == null ? null : "\"" + s + "\"") + .collect(Collectors.joining(", ")); + + mySQLTestUtils.insertToTable( + BEEKEEPER_DB_NAME, + BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, + HOUSEKEEPING_PATH_FIELDS, + values); } protected void insertExpiredMetadata(String path, String partitionName) throws SQLException { insertExpiredMetadata(TABLE_NAME_VALUE, path, partitionName, SHORT_CLEANUP_DELAY_VALUE); } - protected void insertExpiredMetadata(String tableName, String path, String partitionName, String cleanupDelay) + protected void insertExpiredMetadata( + String tableName, String path, String partitionName, String cleanupDelay) throws SQLException { - HousekeepingMetadata metadata = createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); + HousekeepingMetadata metadata = + createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); insertExpiredMetadata(metadata); } protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLException { - String values = Stream - .of(metadata.getId().toString(), metadata.getPath(), metadata.getDatabaseName(), metadata.getTableName(), - metadata.getPartitionName(), metadata.getHousekeepingStatus().toString(), - metadata.getCreationTimestamp().toString(), metadata.getModifiedTimestamp().toString(), - metadata.getCleanupTimestamp().toString(), metadata.getCleanupDelay().toString(), - String.valueOf(metadata.getCleanupAttempts()), metadata.getClientId(), metadata.getLifecycleType()) - .map(s -> s == null ? null : "\"" + s + "\"") - .collect(Collectors.joining(", ")); - - mySQLTestUtils - .insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, HOUSEKEEPING_METADATA_FIELDS, - values); + String values = + Stream.of( + metadata.getId().toString(), + metadata.getPath(), + metadata.getDatabaseName(), + metadata.getTableName(), + metadata.getPartitionName(), + metadata.getHousekeepingStatus().toString(), + metadata.getCreationTimestamp().toString(), + metadata.getModifiedTimestamp().toString(), + metadata.getCleanupTimestamp().toString(), + metadata.getCleanupDelay().toString(), + String.valueOf(metadata.getCleanupAttempts()), + metadata.getClientId(), + metadata.getLifecycleType()) + .map(s -> s == null ? null : "\"" + s + "\"") + .collect(Collectors.joining(", ")); + + mySQLTestUtils.insertToTable( + BEEKEEPER_DB_NAME, + BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, + HOUSEKEEPING_METADATA_FIELDS, + values); } protected int getUnreferencedPathsRowCount() throws SQLException { - return mySQLTestUtils - .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, - format(LIFE_CYCLE_FILTER, UNREFERENCED)); + return mySQLTestUtils.getTableRowCount( + BEEKEEPER_DB_NAME, + BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, + format(LIFE_CYCLE_FILTER, UNREFERENCED)); } protected int getExpiredMetadataRowCount() throws SQLException { - return mySQLTestUtils - .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, - format(LIFE_CYCLE_FILTER, EXPIRED)); + return mySQLTestUtils.getTableRowCount( + BEEKEEPER_DB_NAME, + BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, + format(LIFE_CYCLE_FILTER, EXPIRED)); } protected int getUpdatedExpiredMetadataRowCount() throws SQLException { - return mySQLTestUtils - .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, - format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED)); + return mySQLTestUtils.getTableRowCount( + BEEKEEPER_DB_NAME, + BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, + format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED)); } - protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException { + protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) + throws SQLException { String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; - return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, - format(filter, lifecycleEventType)); + return mySQLTestUtils.getTableRowCount( + BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, format(filter, lifecycleEventType)); } protected List getUnreferencedPaths() throws SQLException { List paths = new ArrayList<>(); - ResultSet resultSet = mySQLTestUtils - .getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, + ResultSet resultSet = + mySQLTestUtils.getTableRows( + BEEKEEPER_DB_NAME, + BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, format(LIFE_CYCLE_FILTER, UNREFERENCED)); while (resultSet.next()) { @@ -244,8 +306,10 @@ protected List getUnreferencedPaths() throws SQLException { protected List getExpiredMetadata() throws SQLException { List metadata = new ArrayList<>(); - ResultSet resultSet = mySQLTestUtils - .getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, + ResultSet resultSet = + mySQLTestUtils.getTableRows( + BEEKEEPER_DB_NAME, + BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, format(LIFE_CYCLE_FILTER, EXPIRED)); while (resultSet.next()) { @@ -255,11 +319,13 @@ protected List getExpiredMetadata() throws SQLException { return metadata; } - protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException { + protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) + throws SQLException { String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; List history = new ArrayList<>(); - ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, - format(filter, lifecycleEventType)); + ResultSet resultSet = + mySQLTestUtils.getTableRows( + BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, format(filter, lifecycleEventType)); while (resultSet.next()) { history.add(mapToBeekeeperHistory(resultSet)); @@ -268,9 +334,9 @@ protected List getBeekeeperHistory(LifecycleEventType lifecycl return history; } - private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType lifecycleEventType) { - return HousekeepingPath - .builder() + private HousekeepingPath createHousekeepingPath( + String path, LifecycleEventType lifecycleEventType) { + return HousekeepingPath.builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -291,8 +357,7 @@ private HousekeepingMetadata createHousekeepingMetadata( String partitionName, LifecycleEventType lifecycleEventType, String cleanupDelay) { - return HousekeepingMetadata - .builder() + return HousekeepingMetadata.builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index 79728afa..2944a72f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -35,6 +33,7 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; import java.sql.SQLException; +import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,8 +49,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.thrift.TException; -import java.time.Duration; -import org.junit.Rule; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -59,6 +56,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import io.micrometer.core.instrument.Meter; @@ -91,8 +89,10 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio protected static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; protected static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; protected static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; - protected static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disableGetObjectMD5Validation"; - protected static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disablePutObjectMD5Validation"; + protected static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = + "com.amazonaws.services.s3.disableGetObjectMD5Validation"; + protected static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = + "com.amazonaws.services.s3.disablePutObjectMD5Validation"; protected static final String S3_ACCESS_KEY = "access"; protected static final String S3_SECRET_KEY = "secret"; @@ -107,45 +107,38 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio protected static final String ROOT_PATH = "s3a://" + BUCKET + "/" + DATABASE_NAME_VALUE + "/"; - protected static final String PARTITIONED_TABLE_PATH = ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; + protected static final String PARTITIONED_TABLE_PATH = + ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; protected static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1"; - protected static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; - protected static final String PARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE - + "/" - + PARTITIONED_TABLE_NAME - + "/id1"; - - protected static final String PARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE - + "/some_location/id1/" - + PARTITION_NAME - + "/file1"; - - protected static final String UNPARTITIONED_TABLE_PATH = ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; - protected static final String UNPARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE - + "/" - + UNPARTITIONED_TABLE_NAME - + "/id1/file1"; - - @Rule - public static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); - static { - S3_CONTAINER.start(); - } + protected static final String PARTITION_PATH = + PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; + protected static final String PARTITIONED_TABLE_OBJECT_KEY = + DATABASE_NAME_VALUE + "/" + PARTITIONED_TABLE_NAME + "/id1"; + + protected static final String PARTITIONED_OBJECT_KEY = + DATABASE_NAME_VALUE + "/some_location/id1/" + PARTITION_NAME + "/file1"; + + protected static final String UNPARTITIONED_TABLE_PATH = + ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; + protected static final String UNPARTITIONED_OBJECT_KEY = + DATABASE_NAME_VALUE + "/" + UNPARTITIONED_TABLE_NAME + "/id1/file1"; + + @Container + protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); protected static AmazonS3 amazonS3; - protected static final String S3_ENDPOINT = ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3); protected final ExecutorService executorService = Executors.newFixedThreadPool(1); - private static Map metastoreProperties = ImmutableMap - .builder() - .put(ENDPOINT, S3_ENDPOINT) - .put(ACCESS_KEY, S3_ACCESS_KEY) - .put(SECRET_KEY, S3_SECRET_KEY) - .build(); + private Map metastoreProperties = + ImmutableMap.builder() + .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) + .put(ACCESS_KEY, S3_ACCESS_KEY) + .put(SECRET_KEY, S3_SECRET_KEY) + .build(); @RegisterExtension - public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( - DATABASE_NAME_VALUE, metastoreProperties); + public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = + new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, metastoreProperties); protected HiveTestUtils hiveTestUtils; protected HiveMetaStoreClient metastoreClient; @@ -155,7 +148,8 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, "test"); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, "false"); - System.setProperty(AWS_S3_ENDPOINT_PROPERTY, S3_ENDPOINT); + System.setProperty( + AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); System.setProperty(AWS_DISABLE_GET_VALIDATION_PROPERTY, "true"); System.setProperty(AWS_DISABLE_PUT_VALIDATION_PROPERTY, "true"); @@ -166,7 +160,6 @@ public static void init() { @AfterAll public static void teardown() { amazonS3.shutdown(); - S3_CONTAINER.stop(); System.clearProperty(SPRING_PROFILES_ACTIVE_PROPERTY); System.clearProperty(SCHEDULER_DELAY_MS_PROPERTY); @@ -199,7 +192,12 @@ public void stop() throws InterruptedException { @Test public void cleanupUnpartitionedTable() throws TException, SQLException { - hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, createBeeKeeperDeletionProperties(), true); + hiveTestUtils.createTableWithProperties( + UNPARTITIONED_TABLE_PATH, + TABLE_NAME_VALUE, + false, + createBeeKeeperDeletionProperties(), + true); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); @@ -213,7 +211,13 @@ public void cleanupUnpartitionedTable() throws TException, SQLException { @Test public void cleanupPartitionedTable() throws Exception { - Table table = hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); + Table table = + hiveTestUtils.createTableWithProperties( + PARTITIONED_TABLE_PATH, + TABLE_NAME_VALUE, + true, + createBeeKeeperDeletionProperties(), + true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); @@ -254,12 +258,12 @@ public void cleanupPartitionButNotTable() throws Exception { Table table = hiveTestUtils.createTable(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); - hiveTestUtils - .addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); + hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); String partition2Name = "event_date=2020-01-01/event_hour=1/event_type=B"; String partition2Path = PARTITION_ROOT_PATH + "/" + partition2Name + "/file1"; - String partition2ObjectKey = DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; + String partition2ObjectKey = + DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -267,14 +271,16 @@ public void cleanupPartitionButNotTable() throws Exception { insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); insertExpiredMetadata(PARTITION_PATH, PARTITION_NAME); - insertExpiredMetadata(TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); + insertExpiredMetadata( + TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); await() .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getExpiredMetadata().get(0).getHousekeepingStatus() == DELETED); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); - List partitions = metastoreClient.listPartitions(DATABASE_NAME_VALUE, TABLE_NAME_VALUE, (short) 1); + List partitions = + metastoreClient.listPartitions(DATABASE_NAME_VALUE, TABLE_NAME_VALUE, (short) 1); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getValues(), List.of("2020-01-01", "1", "B")); assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isTrue(); @@ -284,7 +290,8 @@ public void cleanupPartitionButNotTable() throws Exception { @Test public void cleanupPartitionedTableWithNoPartitions() throws TException, SQLException { - hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); + hiveTestUtils.createTableWithProperties( + PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); @@ -342,25 +349,38 @@ public void disablePartitionedTable() throws Exception { @Test public void cleanupMultipleTablesOfMixedType() throws Exception { - hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, UNPARTITIONED_TABLE_NAME, false, createBeeKeeperDeletionProperties(), true); - - Table partitionedTable = hiveTestUtils - .createTableWithProperties(PARTITIONED_TABLE_PATH, PARTITIONED_TABLE_NAME, true, createBeeKeeperDeletionProperties(), true); - hiveTestUtils - .addPartitionsToTable(PARTITION_ROOT_PATH, partitionedTable, PARTITION_VALUES); + hiveTestUtils.createTableWithProperties( + UNPARTITIONED_TABLE_PATH, + UNPARTITIONED_TABLE_NAME, + false, + createBeeKeeperDeletionProperties(), + true); + + Table partitionedTable = + hiveTestUtils.createTableWithProperties( + PARTITIONED_TABLE_PATH, + PARTITIONED_TABLE_NAME, + true, + createBeeKeeperDeletionProperties(), + true); + hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, partitionedTable, PARTITION_VALUES); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, TABLE_DATA); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); - insertExpiredMetadata(UNPARTITIONED_TABLE_NAME, UNPARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); - insertExpiredMetadata(PARTITIONED_TABLE_NAME, PARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); - insertExpiredMetadata(PARTITIONED_TABLE_NAME, PARTITION_PATH, PARTITION_NAME, SHORT_CLEANUP_DELAY_VALUE); + insertExpiredMetadata( + UNPARTITIONED_TABLE_NAME, UNPARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); + insertExpiredMetadata( + PARTITIONED_TABLE_NAME, PARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); + insertExpiredMetadata( + PARTITIONED_TABLE_NAME, PARTITION_PATH, PARTITION_NAME, SHORT_CLEANUP_DELAY_VALUE); await() .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getExpiredMetadata().get(1).getHousekeepingStatus() == DELETED); - assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, UNPARTITIONED_TABLE_NAME)).isFalse(); + assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, UNPARTITIONED_TABLE_NAME)) + .isFalse(); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, PARTITIONED_TABLE_NAME)).isFalse(); assertThat(amazonS3.doesObjectExist(BUCKET, UNPARTITIONED_OBJECT_KEY)).isFalse(); assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isFalse(); @@ -376,7 +396,8 @@ public void onlyCleanupLocationWhenTableExists() throws SQLException { @Test public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLException { - hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties() ,true); + hiveTestUtils.createTableWithProperties( + PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -394,7 +415,12 @@ public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLExcep @Test public void testEventAddedToHistoryTable() throws TException, SQLException { - hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, createBeeKeeperDeletionProperties(), true); + hiveTestUtils.createTableWithProperties( + UNPARTITIONED_TABLE_PATH, + TABLE_NAME_VALUE, + false, + createBeeKeeperDeletionProperties(), + true); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); @@ -418,7 +444,8 @@ public void tableNotDeletedWhenDeletionPropertyIsFalse() throws TException, SQLE Map tableProperties = new HashMap<>(); tableProperties.put("beekeeper.expired.data.table.deletion.enabled", "false"); - hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, tableProperties, true); + hiveTestUtils.createTableWithProperties( + UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, tableProperties, true); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); @@ -446,7 +473,13 @@ public void tableNotDeletedWhenDeletionPropertyNotSet() throws TException, SQLEx @Test public void metrics() throws Exception { - Table table = hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); + Table table = + hiveTestUtils.createTableWithProperties( + PARTITIONED_TABLE_PATH, + TABLE_NAME_VALUE, + true, + createBeeKeeperDeletionProperties(), + true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); @@ -465,14 +498,24 @@ public void metrics() throws Exception { } protected void assertMetrics() { - Set meterRegistry = ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); + Set meterRegistry = + ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach(registry -> { - List meters = registry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name") - .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + METRIC_NAME, - "hive-partition-" + METRIC_NAME, "s3-paths-deleted", "s3-" + BytesDeletedReporter.METRIC_NAME); - }); + meterRegistry.forEach( + registry -> { + List meters = registry.getMeters(); + assertThat(meters) + .extracting("id", Meter.Id.class) + .extracting("name") + .contains( + "metadata-cleanup-job", + "hive-table-deleted", + "hive-partition-deleted", + "hive-table-" + METRIC_NAME, + "hive-partition-" + METRIC_NAME, + "s3-paths-deleted", + "s3-" + BytesDeletedReporter.METRIC_NAME); + }); } private Map createBeeKeeperDeletionProperties() { @@ -485,7 +528,8 @@ private Map createBeeKeeperDeletionProperties() { public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(HEALTHCHECK_URI); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } @@ -493,7 +537,8 @@ public void healthCheck() { public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index f392d148..f7cd36be 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2024 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -27,6 +25,7 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -36,7 +35,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -71,11 +69,13 @@ public class BeekeeperPathCleanupIntegrationTest extends BeekeeperIntegrationTes protected static final String OBJECT_KEY_ROOT = DB_AND_TABLE_PREFIX + "/id1/partition1"; protected static final String OBJECT_KEY1 = DB_AND_TABLE_PREFIX + "/id1/partition1/file1"; protected static final String OBJECT_KEY2 = DB_AND_TABLE_PREFIX + "/id1/partition1/file2"; - protected static final String OBJECT_KEY_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; + protected static final String OBJECT_KEY_SENTINEL = + DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; protected static final String ABSOLUTE_PATH = "s3://" + BUCKET + "/" + OBJECT_KEY_ROOT; protected static final String OBJECT_KEY_OTHER = DB_AND_TABLE_PREFIX + "/id1/partition10/file1"; - protected static final String OBJECT_KEY_OTHER_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; + protected static final String OBJECT_KEY_OTHER_SENTINEL = + DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; protected static final String SPRING_PROFILES_ACTIVE = "test"; protected static final String SCHEDULER_DELAY_MS = "5000"; @@ -86,6 +86,7 @@ public class BeekeeperPathCleanupIntegrationTest extends BeekeeperIntegrationTes @Container protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); + protected static AmazonS3 amazonS3; protected final ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -95,7 +96,8 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, SPRING_PROFILES_ACTIVE); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, DRY_RUN_ENABLED); - System.setProperty(AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); + System.setProperty( + AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); amazonS3 = ContainerTestUtils.s3Client(S3_CONTAINER, AWS_REGION); amazonS3.createBucket(new CreateBucketRequest(BUCKET, AWS_REGION)); @@ -113,12 +115,12 @@ public static void teardown() { @BeforeEach public void setup() { - amazonS3.listObjectsV2(BUCKET) + amazonS3 + .listObjectsV2(BUCKET) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {})); - await().atMost(Duration.ofMinutes(1)) - .until(BeekeeperPathCleanup::isRunning); + await().atMost(Duration.ofMinutes(1)).until(BeekeeperPathCleanup::isRunning); } @AfterEach @@ -135,7 +137,8 @@ public void cleanupPathsForFile() throws SQLException { String path = "s3://" + BUCKET + "/" + OBJECT_KEY1; insertUnreferencedPath(path); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -153,7 +156,8 @@ public void cleanupPathsForDirectory() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -175,7 +179,8 @@ public void cleanupPathsForDirectoryWithSpace() throws SQLException { amazonS3.putObject(BUCKET, objectKeySentinel, ""); insertUnreferencedPath(absolutePath); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, objectKey1)).isFalse(); @@ -191,7 +196,8 @@ public void cleanupPathsForDirectoryWithTrailingSlash() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH + "/"); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -213,7 +219,8 @@ public void cleanupSentinelForParent() throws SQLException { amazonS3.putObject(BUCKET, databaseSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -236,7 +243,8 @@ public void cleanupSentinelForNonEmptyParent() throws SQLException { amazonS3.putObject(BUCKET, tableSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -255,7 +263,9 @@ public void testEventAddedToHistoryTable() throws SQLException { String path = "s3://" + BUCKET + "/" + OBJECT_KEY1; insertUnreferencedPath(path); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); // deleting a file shouldn't delete a folder sentinel @@ -276,7 +286,8 @@ public void metrics() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -285,20 +296,25 @@ public void metrics() throws SQLException { } private void assertMetrics() { - Set meterRegistry = ((CompositeMeterRegistry) BeekeeperPathCleanup.meterRegistry()).getRegistries(); + Set meterRegistry = + ((CompositeMeterRegistry) BeekeeperPathCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach(registry -> { - List meters = registry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name") - .contains("path-cleanup-job", "s3-paths-deleted", "s3-" + METRIC_NAME); - }); + meterRegistry.forEach( + registry -> { + List meters = registry.getMeters(); + assertThat(meters) + .extracting("id", Meter.Id.class) + .extracting("name") + .contains("path-cleanup-job", "s3-paths-deleted", "s3-" + METRIC_NAME); + }); } @Test public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(HEALTHCHECK_URI); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } @@ -306,7 +322,8 @@ public void healthCheck() { public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await().atMost(TIMEOUT, TimeUnit.SECONDS) + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index efc53065..33155357 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -34,6 +32,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -41,7 +40,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import java.time.Duration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -73,9 +71,10 @@ import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension; @Testcontainers -public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { +public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest + extends BeekeeperIntegrationTestBase { - protected static final int TIMEOUT = 5; + protected static final int TIMEOUT = 30; protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; protected static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; @@ -86,24 +85,33 @@ public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends Bee @Container protected static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); + protected static AmazonSQS amazonSQS; + protected static String ACTUAL_QUEUE_URL; @RegisterExtension - protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( - DATABASE_NAME_VALUE, emptyMap()); + protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = + new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, emptyMap()); + + protected static final String SQS_ENDPOINT_PROPERTY = "properties.sqs.endpoint"; + protected static final String SQS_REGION_PROPERTY = "properties.sqs.region"; @BeforeAll public static void init() { - String queueUrl = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); - System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl); - amazonSQS = ContainerTestUtils.sqsClient(SQS_CONTAINER, AWS_REGION); + ACTUAL_QUEUE_URL = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); amazonSQS.createQueue(QUEUE); + System.setProperty(APIARY_QUEUE_URL_PROPERTY, ACTUAL_QUEUE_URL); + System.setProperty( + SQS_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(SQS_CONTAINER, SQS)); + System.setProperty(SQS_REGION_PROPERTY, AWS_REGION); } @AfterAll public static void teardown() { System.clearProperty(APIARY_QUEUE_URL_PROPERTY); + System.clearProperty(SQS_ENDPOINT_PROPERTY); + System.clearProperty(SQS_REGION_PROPERTY); amazonSQS.shutdown(); } @@ -112,7 +120,7 @@ public static void teardown() { public void setup() { System.setProperty(METASTORE_URI_PROPERTY, thriftHiveMetaStore.getThriftConnectionUri()); - amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE))); + amazonSQS.purgeQueue(new PurgeQueueRequest(ACTUAL_QUEUE_URL)); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); await().atMost(Duration.ofMinutes(1)).until(BeekeeperSchedulerApiary::isRunning); } @@ -125,8 +133,9 @@ public void stop() throws InterruptedException { @Test public void unreferencedAlterTableEvent() throws SQLException, IOException, URISyntaxException { - AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", - "s3://bucket/oldTableLocation", true, true); + AlterTableSqsMessage alterTableSqsMessage = + new AlterTableSqsMessage( + "s3://bucket/tableLocation", "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 1); @@ -135,9 +144,11 @@ public void unreferencedAlterTableEvent() throws SQLException, IOException, URIS } @Test - public void unreferencedMultipleAlterTableEvents() throws SQLException, IOException, URISyntaxException { - AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", - "s3://bucket/oldTableLocation", true, true); + public void unreferencedMultipleAlterTableEvents() + throws SQLException, IOException, URISyntaxException { + AlterTableSqsMessage alterTableSqsMessage = + new AlterTableSqsMessage( + "s3://bucket/tableLocation", "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); alterTableSqsMessage.setTableLocation("s3://bucket/tableLocation2"); alterTableSqsMessage.setOldTableLocation("s3://bucket/tableLocation"); @@ -150,10 +161,15 @@ public void unreferencedMultipleAlterTableEvents() throws SQLException, IOExcept } @Test - public void unreferencedAlterPartitionEvent() throws SQLException, IOException, URISyntaxException { - AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage( - "s3://bucket/table/expiredTableLocation", "s3://bucket/table/partitionLocation", - "s3://bucket/table/unreferencedPartitionLocation", true, true); + public void unreferencedAlterPartitionEvent() + throws SQLException, IOException, URISyntaxException { + AlterPartitionSqsMessage alterPartitionSqsMessage = + new AlterPartitionSqsMessage( + "s3://bucket/table/expiredTableLocation", + "s3://bucket/table/partitionLocation", + "s3://bucket/table/unreferencedPartitionLocation", + true, + true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); alterPartitionSqsMessage.setTableLocation("s3://bucket/table/expiredTableLocation2"); alterPartitionSqsMessage.setPartitionLocation("s3://bucket/table/partitionLocation2"); @@ -163,29 +179,41 @@ public void unreferencedAlterPartitionEvent() throws SQLException, IOException, List unreferencedPaths = getUnreferencedPaths(); assertUnreferencedPath(unreferencedPaths.get(0), "s3://bucket/table/partitionLocation"); - assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); + assertUnreferencedPath( + unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); } @Test - public void unreferencedMultipleAlterPartitionEvent() throws IOException, SQLException, URISyntaxException { - List - .of(new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation", - "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), - new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation2", - "s3://bucket/table/partitionLocation2", "s3://bucket/table/partitionLocation", true, true)) + public void unreferencedMultipleAlterPartitionEvent() + throws IOException, SQLException, URISyntaxException { + List.of( + new AlterPartitionSqsMessage( + "s3://bucket/table/expiredTableLocation", + "s3://bucket/table/partitionLocation", + "s3://bucket/table/unreferencedPartitionLocation", + true, + true), + new AlterPartitionSqsMessage( + "s3://bucket/table/expiredTableLocation2", + "s3://bucket/table/partitionLocation2", + "s3://bucket/table/partitionLocation", + true, + true)) .forEach(msg -> amazonSQS.sendMessage(sendMessageRequest(msg.getFormattedString()))); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); List unreferencedPaths = getUnreferencedPaths(); assertUnreferencedPath(unreferencedPaths.get(0), "s3://bucket/table/partitionLocation"); - assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); + assertUnreferencedPath( + unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); } @Test - public void unreferencedDropPartitionEvent() throws SQLException, IOException, URISyntaxException { - DropPartitionSqsMessage dropPartitionSqsMessage = new DropPartitionSqsMessage("s3://bucket/table/partitionLocation", - true, true); + public void unreferencedDropPartitionEvent() + throws SQLException, IOException, URISyntaxException { + DropPartitionSqsMessage dropPartitionSqsMessage = + new DropPartitionSqsMessage("s3://bucket/table/partitionLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(dropPartitionSqsMessage.getFormattedString())); dropPartitionSqsMessage.setPartitionLocation("s3://bucket/table/partitionLocation2"); amazonSQS.sendMessage(sendMessageRequest(dropPartitionSqsMessage.getFormattedString())); @@ -198,7 +226,8 @@ public void unreferencedDropPartitionEvent() throws SQLException, IOException, U @Test public void unreferencedDropTableEvent() throws SQLException, IOException, URISyntaxException { - DropTableSqsMessage dropTableSqsMessage = new DropTableSqsMessage("s3://bucket/tableLocation", true, true); + DropTableSqsMessage dropTableSqsMessage = + new DropTableSqsMessage("s3://bucket/tableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(dropTableSqsMessage.getFormattedString())); dropTableSqsMessage.setTableLocation("s3://bucket/tableLocation2"); amazonSQS.sendMessage(sendMessageRequest(dropTableSqsMessage.getFormattedString())); @@ -211,11 +240,14 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy @Test public void testEventAddedToHistoryTable() throws IOException, URISyntaxException, SQLException { - AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", - "s3://bucket/oldTableLocation", true, true); + AlterTableSqsMessage alterTableSqsMessage = + new AlterTableSqsMessage( + "s3://bucket/tableLocation", "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); BeekeeperHistory history = beekeeperHistory.get(0); @@ -238,11 +270,13 @@ public void healthCheck() { public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await().atMost(30, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); + await() + .atMost(30, TimeUnit.SECONDS) + .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } protected SendMessageRequest sendMessageRequest(String payload) { - return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload); + return new SendMessageRequest(ACTUAL_QUEUE_URL, payload); } protected void assertUnreferencedPath(HousekeepingPath actual, String expectedPath) { @@ -257,7 +291,8 @@ public void assertHousekeepingEntity(HousekeepingPath actual, String expectedPat assertThat(actual.getHousekeepingStatus()).isEqualTo(SCHEDULED); assertThat(actual.getCreationTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE); assertThat(actual.getModifiedTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE); - assertThat(actual.getCleanupTimestamp()).isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); + assertThat(actual.getCleanupTimestamp()) + .isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); assertThat(actual.getCleanupDelay()).isEqualTo(PeriodDuration.parse(SHORT_CLEANUP_DELAY_VALUE)); assertThat(actual.getCleanupAttempts()).isEqualTo(CLEANUP_ATTEMPTS_VALUE); assertThat(actual.getClientId()).isEqualTo(CLIENT_ID_VALUE); @@ -265,12 +300,16 @@ public void assertHousekeepingEntity(HousekeepingPath actual, String expectedPat } public void assertMetrics() { - Set meterRegistry = ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()) - .getRegistries(); + Set meterRegistry = + ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach(registry -> { - List meters = registry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name").contains(SCHEDULED_ORPHANED_METRIC); - }); + meterRegistry.forEach( + registry -> { + List meters = registry.getMeters(); + assertThat(meters) + .extracting("id", Meter.Id.class) + .extracting("name") + .contains(SCHEDULED_ORPHANED_METRIC); + }); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java index acce7868..ff2084d3 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -47,7 +45,8 @@ private CommonTestVariables() {} // HOUSEKEEPINGENTITY DEFAULT VALUES public static final String DATABASE_NAME_VALUE = "some_database"; public static final String TABLE_NAME_VALUE = "some_table"; - public static final LocalDateTime CREATION_TIMESTAMP_VALUE = LocalDateTime.now(UTC).minus(1L, ChronoUnit.MINUTES); + public static final LocalDateTime CREATION_TIMESTAMP_VALUE = + LocalDateTime.now(UTC).minus(1L, ChronoUnit.MINUTES); public static final String SHORT_CLEANUP_DELAY_VALUE = "PT1S"; public static final String LONG_CLEANUP_DELAY_VALUE = "P3D"; public static final int CLEANUP_ATTEMPTS_VALUE = 0; diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index d7cbc63c..eab87e1a 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.api; @@ -26,6 +24,7 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; import java.io.IOException; +import java.net.ServerSocket; import java.net.http.HttpResponse; import java.sql.SQLException; import java.time.Duration; @@ -43,7 +42,6 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.data.domain.Page; import org.springframework.http.HttpStatus; -import org.springframework.util.SocketUtils; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -66,8 +64,11 @@ public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { public ObjectMapper geObjectMapper() { - return new ObjectMapper().registerModule(new ParameterNamesModule()).registerModule(new JavaTimeModule()); + return new ObjectMapper() + .registerModule(new ParameterNamesModule()) + .registerModule(new JavaTimeModule()); } + private static final Logger log = LoggerFactory.getLogger(BeekeeperApiIntegrationTest.class); protected static ConfigurableApplicationContext context; protected BeekeeperApiTestClient testClient; @@ -75,26 +76,86 @@ public ObjectMapper geObjectMapper() { private Long id = 1L; protected static final String someTable = "some_table"; protected static final String someDatabase = "some_database"; - protected static final String pathA = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; - protected static final String pathB = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B"; - protected static final String pathC = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C"; + protected static final String pathA = + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; + protected static final String pathB = + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B"; + protected static final String pathC = + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C"; protected static final Duration duration = Duration.parse("P3D"); protected static final int pageSize = 1; protected static final String partitionA = "event_date=2020-01-01/event_hour=0/event_type=A"; protected static final String partitionB = "event_date=2020-01-01/event_hour=0/event_type=B"; protected static final String partitionC = "event_date=2020-01-01/event_hour=0/event_type=C"; - protected final HousekeepingPath testPathA = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString(), HousekeepingStatus.FAILED); - protected final HousekeepingPath testPathB = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); - protected final HousekeepingPath testPathC = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataA = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataB = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataC = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataD = createHousekeepingMetadata(someTable, pathC, partitionB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.SCHEDULED); - protected final HousekeepingMetadata testMetadataE = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString(),HousekeepingStatus.SCHEDULED); + protected final HousekeepingPath testPathA = + createHousekeepingPath( + someTable, + pathA, + LifecycleEventType.EXPIRED, + duration.toString(), + HousekeepingStatus.FAILED); + protected final HousekeepingPath testPathB = + createHousekeepingPath( + someTable, + pathB, + LifecycleEventType.UNREFERENCED, + duration.toString(), + HousekeepingStatus.FAILED); + protected final HousekeepingPath testPathC = + createHousekeepingPath( + someTable, + pathC, + LifecycleEventType.UNREFERENCED, + duration.toString(), + HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataA = + createHousekeepingMetadata( + someTable, + pathA, + partitionA, + LifecycleEventType.EXPIRED, + duration.toString(), + HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataB = + createHousekeepingMetadata( + someTable, + pathB, + partitionB, + LifecycleEventType.EXPIRED, + duration.toString(), + HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataC = + createHousekeepingMetadata( + someTable, + pathC, + partitionC, + LifecycleEventType.EXPIRED, + duration.toString(), + HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataD = + createHousekeepingMetadata( + someTable, + pathC, + partitionB, + LifecycleEventType.UNREFERENCED, + duration.toString(), + HousekeepingStatus.SCHEDULED); + protected final HousekeepingMetadata testMetadataE = + createHousekeepingMetadata( + someTable, + pathC, + partitionC, + LifecycleEventType.UNREFERENCED, + duration.toString(), + HousekeepingStatus.SCHEDULED); + @BeforeEach - public void beforeEach() { - int port = SocketUtils.findAvailableTcpPort(); - String[] args = new String[] { "--server.port=" + port }; + public void beforeEach() throws IOException { + int port; + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } + String[] args = new String[] {"--server.port=" + port}; final String url = format("http://localhost:%d", port); log.info("Starting to run Beekeeper API on: {} and args: {}", url, args); context = SpringApplication.run(BeekeeperApiApplication.class, args); @@ -121,32 +182,37 @@ public void testGetMetadataWhenTableNotFoundReturnsEmptyList() HttpResponse response = testClient.getMetadata("wrong_database", "wrong_table"); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = mapper - .readValue(body, new TypeReference>() {}); + Page responsePage = + mapper.readValue( + body, new TypeReference>() {}); assertThat(responsePage.getTotalElements()).isEqualTo(0); } @Test - public void testGetMetadataWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { + public void testGetMetadataWhenThereIsFiltering() + throws SQLException, InterruptedException, IOException { testMetadataA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testMetadataA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataD, testMetadataE)) { + for (HousekeepingMetadata testMetadata : + Arrays.asList(testMetadataA, testMetadataD, testMetadataE)) { insertExpiredMetadata(testMetadata); } - String filters = "?housekeeping_status=FAILED" - + "&lifecycle_type=EXPIRED" - + "&deleted_before=2000-05-05T10:41:20" - + "®istered_before=2000-04-04T10:41:20" - + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A" - + "&partition_name=event_date=2020-01-01/event_hour=0/event_type=A"; + String filters = + "?housekeeping_status=FAILED" + + "&lifecycle_type=EXPIRED" + + "&deleted_before=2000-05-05T10:41:20" + + "®istered_before=2000-04-04T10:41:20" + + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A" + + "&partition_name=event_date=2020-01-01/event_hour=0/event_type=A"; HttpResponse response = testClient.getMetadata(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = mapper - .readValue(body, new TypeReference>() {}); + Page responsePage = + mapper.readValue( + body, new TypeReference>() {}); List result = responsePage.getContent(); assertThatMetadataEqualsResponse(testMetadataA, result.get(0)); @@ -154,24 +220,27 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt } @Test - public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { + public void testGetPathsWhenThereIsFiltering() + throws SQLException, InterruptedException, IOException { testPathA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testPathA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { insertUnreferencedPath(testPath); } - String filters = "?housekeeping_status=FAILED" - + "&lifecycle_type=EXPIRED" - + "&deleted_before=2000-05-05T10:41:20" - + "®istered_before=2000-04-04T10:41:20" - + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; - - HttpResponse response = testClient.getUnreferencedPaths(someDatabase, someTable, filters); + String filters = + "?housekeeping_status=FAILED" + + "&lifecycle_type=EXPIRED" + + "&deleted_before=2000-05-05T10:41:20" + + "®istered_before=2000-04-04T10:41:20" + + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; + + HttpResponse response = + testClient.getUnreferencedPaths(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = mapper - .readValue(body, new TypeReference>() {}); + Page responsePage = + mapper.readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThat(responsePage.getTotalElements()).isEqualTo(1L); assertThatPathsEqualsResponse(testPathA, result.get(0)); @@ -185,12 +254,13 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc } String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; - HttpResponse response = testClient.getUnreferencedPaths(someDatabase, someTable, filters); + HttpResponse response = + testClient.getUnreferencedPaths(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = mapper - .readValue(body, new TypeReference>() {}); + Page responsePage = + mapper.readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); @@ -199,7 +269,8 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc @Test public void testMetadataPageable() throws SQLException, InterruptedException, IOException { - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { + for (HousekeepingMetadata testMetadata : + Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } @@ -208,8 +279,9 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = mapper - .readValue(body, new TypeReference>() {}); + Page responsePage = + mapper.readValue( + body, new TypeReference>() {}); List result = responsePage.getContent(); assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); @@ -220,7 +292,8 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO @Disabled @Test public void manualTest() throws SQLException, InterruptedException { - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { + for (HousekeepingMetadata testMetadata : + Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } @@ -234,27 +307,35 @@ public void manualTest() throws SQLException, InterruptedException { private void assertThatMetadataEqualsResponse( HousekeepingMetadata housekeepingMetadata, HousekeepingMetadataResponse housekeepingMetadataResponse) { - assertThat(housekeepingMetadata.getDatabaseName()).isEqualTo(housekeepingMetadataResponse.getDatabaseName()); - assertThat(housekeepingMetadata.getTableName()).isEqualTo(housekeepingMetadataResponse.getTableName()); + assertThat(housekeepingMetadata.getDatabaseName()) + .isEqualTo(housekeepingMetadataResponse.getDatabaseName()); + assertThat(housekeepingMetadata.getTableName()) + .isEqualTo(housekeepingMetadataResponse.getTableName()); assertThat(housekeepingMetadata.getPath()).isEqualTo(housekeepingMetadataResponse.getPath()); assertThat(housekeepingMetadata.getHousekeepingStatus()) .isEqualTo(housekeepingMetadataResponse.getHousekeepingStatus()); assertThat(housekeepingMetadata.getCleanupDelay().toString()) .isEqualTo(housekeepingMetadataResponse.getCleanupDelay()); - assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(housekeepingMetadataResponse.getCleanupAttempts()); - assertThat(housekeepingMetadata.getLifecycleType()).isEqualTo(housekeepingMetadataResponse.getLifecycleType()); + assertThat(housekeepingMetadata.getCleanupAttempts()) + .isEqualTo(housekeepingMetadataResponse.getCleanupAttempts()); + assertThat(housekeepingMetadata.getLifecycleType()) + .isEqualTo(housekeepingMetadataResponse.getLifecycleType()); } private void assertThatPathsEqualsResponse( - HousekeepingPath housekeepingPath, - HousekeepingPathResponse housekeepingPathResponse) { - assertThat(housekeepingPath.getDatabaseName()).isEqualTo(housekeepingPathResponse.getDatabaseName()); + HousekeepingPath housekeepingPath, HousekeepingPathResponse housekeepingPathResponse) { + assertThat(housekeepingPath.getDatabaseName()) + .isEqualTo(housekeepingPathResponse.getDatabaseName()); assertThat(housekeepingPath.getTableName()).isEqualTo(housekeepingPathResponse.getTableName()); assertThat(housekeepingPath.getPath()).isEqualTo(housekeepingPathResponse.getPath()); - assertThat(housekeepingPath.getHousekeepingStatus()).isEqualTo(housekeepingPathResponse.getHousekeepingStatus()); - assertThat(housekeepingPath.getCleanupDelay().toString()).isEqualTo(housekeepingPathResponse.getCleanupDelay()); - assertThat(housekeepingPath.getCleanupAttempts()).isEqualTo(housekeepingPathResponse.getCleanupAttempts()); - assertThat(housekeepingPath.getLifecycleType()).isEqualTo(housekeepingPathResponse.getLifecycleType()); + assertThat(housekeepingPath.getHousekeepingStatus()) + .isEqualTo(housekeepingPathResponse.getHousekeepingStatus()); + assertThat(housekeepingPath.getCleanupDelay().toString()) + .isEqualTo(housekeepingPathResponse.getCleanupDelay()); + assertThat(housekeepingPath.getCleanupAttempts()) + .isEqualTo(housekeepingPathResponse.getCleanupAttempts()); + assertThat(housekeepingPath.getLifecycleType()) + .isEqualTo(housekeepingPathResponse.getLifecycleType()); } private HousekeepingMetadata createHousekeepingMetadata( @@ -264,8 +345,7 @@ private HousekeepingMetadata createHousekeepingMetadata( LifecycleEventType lifecycleEventType, String cleanupDelay, HousekeepingStatus housekeepingStatus) { - return HousekeepingMetadata - .builder() + return HousekeepingMetadata.builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -287,8 +367,7 @@ private HousekeepingPath createHousekeepingPath( LifecycleEventType lifecycleEventType, String cleanupDelay, HousekeepingStatus housekeepingStatus) { - return HousekeepingPath - .builder() + return HousekeepingPath.builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -316,10 +395,11 @@ public void testInvalidSortParameter() throws SQLException, IOException, Interru ErrorResponse errorResponse = mapper.readValue(body, ErrorResponse.class); assertThat(errorResponse.getStatus()).isEqualTo(HttpStatus.BAD_REQUEST.value()); - assertThat(errorResponse.getMessage()).isEqualTo("No property 'nonExistentProperty' found for type 'HousekeepingMetadata'"); + assertThat(errorResponse.getMessage()) + .isEqualTo("No property 'nonExistentProperty' found for type 'HousekeepingMetadata'"); assertThat(errorResponse.getError()).isEqualTo("Bad Request"); - assertThat(errorResponse.getPath()).contains("/api/v1/database/some_database/table/some_table/metadata"); + assertThat(errorResponse.getPath()) + .contains("/api/v1/database/some_database/table/some_table/metadata"); assertThat(errorResponse.getTimestamp()).isNotNull(); } - } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java index acdf5876..2ff4deed 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -25,11 +23,8 @@ public class AddPartitionSqsMessage extends SqsMessage { public AddPartitionSqsMessage( - String partitionLocation, - String partitionKeys, - String partitionValues, - boolean isExpired - ) throws IOException, URISyntaxException { + String partitionLocation, String partitionKeys, String partitionValues, boolean isExpired) + throws IOException, URISyntaxException { super(ADD_PARTITION); setTableLocation(DUMMY_LOCATION); setPartitionLocation(partitionLocation); @@ -39,14 +34,17 @@ public AddPartitionSqsMessage( } private void setPartitionLocation(String partitionLocation) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); } private void setPartitionKeys(String partitionKeys) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); } private void setPartitionValues(String partitionValues) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java index 86c73929..218a6067 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -29,8 +27,8 @@ public AlterPartitionSqsMessage( String partitionLocation, String oldPartitionLocation, boolean isUnreferenced, - boolean isWhitelisted - ) throws IOException, URISyntaxException { + boolean isWhitelisted) + throws IOException, URISyntaxException { super(ALTER_PARTITION); setTableLocation(tableLocation); setPartitionLocation(partitionLocation); @@ -43,11 +41,8 @@ public AlterPartitionSqsMessage( } public AlterPartitionSqsMessage( - String partitionLocation, - String partitionKeys, - String partitionValues, - boolean isExpired - ) throws IOException, URISyntaxException { + String partitionLocation, String partitionKeys, String partitionValues, boolean isExpired) + throws IOException, URISyntaxException { super(ALTER_PARTITION); setTableLocation(DUMMY_LOCATION); setPartitionLocation(partitionLocation); @@ -59,22 +54,27 @@ public AlterPartitionSqsMessage( } public void setPartitionLocation(String partitionLocation) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); } public void setOldPartitionLocation(String oldPartitionLocation) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_OLD_PARTITION_LOCATION_KEY, new JsonPrimitive(oldPartitionLocation)); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_OLD_PARTITION_LOCATION_KEY, new JsonPrimitive(oldPartitionLocation)); } public void setPartitionKeys(String partitionKeys) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); } public void setPartitionValues(String partitionValues) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); } public void setOldPartitionValues(String oldPartitionValues) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_OLD_PARTITION_VALUES_KEY, PARSER.parse(oldPartitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_OLD_PARTITION_VALUES_KEY, PARSER.parse(oldPartitionValues).getAsJsonArray()); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java index 9fbff756..ac21492f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -27,11 +25,8 @@ public class AlterTableSqsMessage extends SqsMessage { public AlterTableSqsMessage( - String tableLocation, - String oldTableLocation, - boolean isUnreferenced, - boolean isWhitelisted - ) throws IOException, URISyntaxException { + String tableLocation, String oldTableLocation, boolean isUnreferenced, boolean isWhitelisted) + throws IOException, URISyntaxException { super(ALTER_TABLE); setTableLocation(tableLocation); setOldTableLocation(oldTableLocation); @@ -40,10 +35,8 @@ public AlterTableSqsMessage( setWhitelisted(isWhitelisted); } - public AlterTableSqsMessage( - String tableLocation, - boolean isExpired - ) throws IOException, URISyntaxException { + public AlterTableSqsMessage(String tableLocation, boolean isExpired) + throws IOException, URISyntaxException { super(ALTER_TABLE); setTableLocation(tableLocation); setOldTableLocation(DUMMY_LOCATION); @@ -52,7 +45,8 @@ public AlterTableSqsMessage( } public void setOldTableLocation(String oldTableLocation) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_OLD_LOCATION_KEY, new JsonPrimitive(oldTableLocation)); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_OLD_LOCATION_KEY, new JsonPrimitive(oldTableLocation)); } public void setOldTableName(String oldTableName) { diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java index 5c70e846..3c39331f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -22,20 +20,15 @@ public class CreateTableSqsMessage extends SqsMessage { - public CreateTableSqsMessage( - String tableLocation, - boolean isExpired - ) throws IOException, URISyntaxException { + public CreateTableSqsMessage(String tableLocation, boolean isExpired) + throws IOException, URISyntaxException { super(CREATE_TABLE); setTableLocation(tableLocation); setExpired(isExpired); } - public CreateTableSqsMessage( - String tableLocation, - boolean isIceberg, - boolean isExpired - ) throws IOException, URISyntaxException { + public CreateTableSqsMessage(String tableLocation, boolean isIceberg, boolean isExpired) + throws IOException, URISyntaxException { super(CREATE_TABLE); setTableLocation(tableLocation); setExpired(isExpired); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java index 536999ce..47a55096 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -25,10 +23,8 @@ public class DropPartitionSqsMessage extends SqsMessage { public DropPartitionSqsMessage( - String partitionLocation, - boolean isUnreferenced, - boolean isWhitelisted - ) throws IOException, URISyntaxException { + String partitionLocation, boolean isUnreferenced, boolean isWhitelisted) + throws IOException, URISyntaxException { super(DROP_PARTITION); setTableLocation(DUMMY_LOCATION); setPartitionLocation(partitionLocation); @@ -39,14 +35,17 @@ public DropPartitionSqsMessage( } public void setPartitionLocation(String partitionLocation) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); } public void setPartitionKeys(String partitionKeys) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); } public void setPartitionValues(String partitionValues) { - apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add( + EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java index f5d7d47b..edf859c2 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -22,11 +20,8 @@ public class DropTableSqsMessage extends SqsMessage { - public DropTableSqsMessage( - String tableLocation, - boolean isUnreferenced, - boolean isWhitelisted - ) throws IOException, URISyntaxException { + public DropTableSqsMessage(String tableLocation, boolean isUnreferenced, boolean isWhitelisted) + throws IOException, URISyntaxException { super(DROP_TABLE); setTableLocation(tableLocation); setUnreferenced(isUnreferenced); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java index dda2de2c..547a6acf 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -58,7 +56,8 @@ public abstract class SqsMessage { public static final String EVENT_TABLE_OLD_PARTITION_VALUES_KEY = "oldPartitionValues"; public static final String DUMMY_LOCATION = "s3://dummy-location"; - public static final String DUMMY_PARTITION_KEYS = "{ \"col_1\": \"string\", \"col_2\": \"integer\", \"col_3\": \"string\" }"; + public static final String DUMMY_PARTITION_KEYS = + "{ \"col_1\": \"string\", \"col_2\": \"integer\", \"col_3\": \"string\" }"; public static final String DUMMY_PARTITION_VALUES = "[ \"val_1\", \"val_2\", \"val_3\" ]"; public static final String EVENT_PROTOCOL_VERSION_VALUE = "1.0"; @@ -68,9 +67,11 @@ public abstract class SqsMessage { public SqsMessage(EventType eventType) throws URISyntaxException, IOException { this.eventType = eventType; - apiaryEventJsonObject = PARSER.parse(readString(Path.of(APIARY_EVENT_JSON_URL.toURI()))).getAsJsonObject(); + apiaryEventJsonObject = + PARSER.parse(readString(Path.of(APIARY_EVENT_JSON_URL.toURI()))).getAsJsonObject(); apiaryEventMessageJsonObject = new JsonObject(); - apiaryEventMessageJsonObject.add(EVENT_PROTOCOL_VERSION_KEY, new JsonPrimitive(EVENT_PROTOCOL_VERSION_VALUE)); + apiaryEventMessageJsonObject.add( + EVENT_PROTOCOL_VERSION_KEY, new JsonPrimitive(EVENT_PROTOCOL_VERSION_VALUE)); apiaryEventMessageJsonObject.add(EVENT_TYPE_KEY, new JsonPrimitive(eventType.toString())); apiaryEventMessageJsonObject.add(EVENT_DB_KEY, new JsonPrimitive(DATABASE_NAME_VALUE)); apiaryEventMessageJsonObject.add(EVENT_TABLE_NAME_KEY, new JsonPrimitive(TABLE_NAME_VALUE)); @@ -82,31 +83,41 @@ public void setTableLocation(String location) { } public void setUnreferenced(boolean isUnreferenced) { - JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); - tableParameters.add(UNREFERENCED.getTableParameterName(), new JsonPrimitive(String.valueOf(isUnreferenced))); - tableParameters.add(UNREFERENCED_DATA_RETENTION_PERIOD_PROPERTY_KEY, new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); + JsonObject tableParameters = + apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + tableParameters.add( + UNREFERENCED.getTableParameterName(), new JsonPrimitive(String.valueOf(isUnreferenced))); + tableParameters.add( + UNREFERENCED_DATA_RETENTION_PERIOD_PROPERTY_KEY, + new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); } public void setExpired(boolean isExpired) { - JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); - tableParameters.add(EXPIRED.getTableParameterName(), new JsonPrimitive(String.valueOf(isExpired))); - tableParameters.add(EXPIRED_DATA_RETENTION_PERIOD_PROPERTY_KEY, new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); + JsonObject tableParameters = + apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + tableParameters.add( + EXPIRED.getTableParameterName(), new JsonPrimitive(String.valueOf(isExpired))); + tableParameters.add( + EXPIRED_DATA_RETENTION_PERIOD_PROPERTY_KEY, new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); } public void setIceberg() { - JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + JsonObject tableParameters = + apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); tableParameters.add("table_format", new JsonPrimitive("ICEBERG")); tableParameters.add("metadata_location", new JsonPrimitive("s3://bucket/metadata")); } public void setWhitelisted(boolean isWhitelisted) { String whitelist = isWhitelisted ? eventType.toString() : ""; - JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + JsonObject tableParameters = + apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); tableParameters.add(BEEKEEPER_HIVE_EVENT_WHITELIST, new JsonPrimitive(whitelist)); } public final String getFormattedString() { - apiaryEventJsonObject.add(APIARY_EVENT_MESSAGE_KEY, new JsonPrimitive(apiaryEventMessageJsonObject.toString())); + apiaryEventJsonObject.add( + APIARY_EVENT_MESSAGE_KEY, new JsonPrimitive(apiaryEventMessageJsonObject.toString())); return apiaryEventJsonObject.toString(); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java index 2b2f0738..6c97e20f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -34,8 +32,14 @@ public class SqsMessageTest { - private static final Set COMMON_KEYS = Set.of("protocolVersion", "eventType", "tableParameters", - "dbName", "tableName", "tableLocation"); + private static final Set COMMON_KEYS = + Set.of( + "protocolVersion", + "eventType", + "tableParameters", + "dbName", + "tableName", + "tableLocation"); @Test public void testCreateTableFormat() throws IOException, URISyntaxException { @@ -46,45 +50,40 @@ public void testCreateTableFormat() throws IOException, URISyntaxException { @Test public void testAddPartitionFormat() throws IOException, URISyntaxException { - Set specificKeys = Set.of( - "partitionKeys", - "partitionValues", - "partitionLocation", - "tableParameters"); - AddPartitionSqsMessage message = new AddPartitionSqsMessage(DUMMY_LOCATION, DUMMY_PARTITION_KEYS, - DUMMY_PARTITION_VALUES, true); + Set specificKeys = + Set.of("partitionKeys", "partitionValues", "partitionLocation", "tableParameters"); + AddPartitionSqsMessage message = + new AddPartitionSqsMessage( + DUMMY_LOCATION, DUMMY_PARTITION_KEYS, DUMMY_PARTITION_VALUES, true); assertKeys(message, specificKeys, "ADD_PARTITION"); } @Test public void testAlterPartitionFormat() throws IOException, URISyntaxException { - Set specificKeys = Set.of( - "partitionKeys", - "partitionValues", - "partitionLocation", - "oldPartitionValues", - "oldPartitionLocation"); - AlterPartitionSqsMessage message = new AlterPartitionSqsMessage(DUMMY_LOCATION, DUMMY_PARTITION_KEYS, - DUMMY_PARTITION_VALUES, true); + Set specificKeys = + Set.of( + "partitionKeys", + "partitionValues", + "partitionLocation", + "oldPartitionValues", + "oldPartitionLocation"); + AlterPartitionSqsMessage message = + new AlterPartitionSqsMessage( + DUMMY_LOCATION, DUMMY_PARTITION_KEYS, DUMMY_PARTITION_VALUES, true); assertKeys(message, specificKeys, "ALTER_PARTITION"); } @Test public void testAlterTableFormat() throws IOException, URISyntaxException { - Set specificKeys = Set.of( - "oldTableName", - "oldTableLocation"); + Set specificKeys = Set.of("oldTableName", "oldTableLocation"); AlterTableSqsMessage message = new AlterTableSqsMessage(DUMMY_LOCATION, true); assertKeys(message, specificKeys, "ALTER_TABLE"); } @Test public void testDropPartitionFormat() throws IOException, URISyntaxException { - Set specificKeys = Set.of( - "partitionKeys", - "partitionValues", - "partitionLocation", - "tableParameters"); + Set specificKeys = + Set.of("partitionKeys", "partitionValues", "partitionLocation", "tableParameters"); DropPartitionSqsMessage message = new DropPartitionSqsMessage(DUMMY_LOCATION, true, true); assertKeys(message, specificKeys, "DROP_PARTITION"); } @@ -97,10 +96,13 @@ public void testDropTableFormat() throws IOException, URISyntaxException { } private void assertKeys(SqsMessage sqsMessage, Set specificKeys, String eventType) { - SortedSet mergedSet = new TreeSet<>() {{ - addAll(specificKeys); - addAll(COMMON_KEYS); - }}; + SortedSet mergedSet = + new TreeSet<>() { + { + addAll(specificKeys); + addAll(COMMON_KEYS); + } + }; JsonObject object = sqsMessage.getApiaryEventMessageJsonObject(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java index b22cf8e5..ccbbcaaf 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -38,37 +36,53 @@ public BeekeeperApiTestClient(String baseUrl) { this.httpClient = HttpClient.newHttpClient(); } - public HttpResponse getMetadata(String database, String table) throws IOException, InterruptedException { - HttpRequest request = newBuilder() - .uri( - URI.create(String.format(housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH, database, table))) - .GET() - .build(); + public HttpResponse getMetadata(String database, String table) + throws IOException, InterruptedException { + HttpRequest request = + newBuilder() + .uri( + URI.create( + String.format( + housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH, + database, + table))) + .GET() + .build(); return httpClient.send(request, BodyHandlers.ofString()); } public HttpResponse getMetadata(String database, String table, String filters) - throws IOException, InterruptedException { - HttpRequest request = newBuilder() - .uri(URI - .create(String - .format(housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH + "%s", database, table, - filters))) - .GET() - .build(); + throws IOException, InterruptedException { + HttpRequest request = + newBuilder() + .uri( + URI.create( + String.format( + housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH + "%s", + database, + table, + filters))) + .GET() + .build(); return httpClient.send(request, BodyHandlers.ofString()); } public HttpResponse getUnreferencedPaths(String database, String table, String filters) throws IOException, InterruptedException { - HttpRequest request = newBuilder() - .uri(URI - .create(String - .format(housekeepingEntityUrl + "/database/%s/table/%s" + UNREFERENCED_PATHS_PATH + "%s", database, table, - filters))) - .GET() - .build(); + HttpRequest request = + newBuilder() + .uri( + URI.create( + String.format( + housekeepingEntityUrl + + "/database/%s/table/%s" + + UNREFERENCED_PATHS_PATH + + "%s", + database, + table, + filters))) + .GET() + .build(); return httpClient.send(request, BodyHandlers.ofString()); } - } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java index 32b8241d..b8a4c2c8 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -20,6 +18,7 @@ import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.utility.DockerImageName; import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.services.s3.AmazonS3; @@ -36,31 +35,35 @@ public static MySQLContainer mySqlContainer() { } public static LocalStackContainer awsContainer(LocalStackContainer.Service service) { - return new LocalStackContainer().withServices(service); + return new LocalStackContainer(DockerImageName.parse("localstack/localstack:3.0.0")) + .withServices(service); } - public static String awsServiceEndpoint(LocalStackContainer awsContainer, LocalStackContainer.Service service) { - return awsContainer.getEndpointConfiguration(service).getServiceEndpoint(); + public static String awsServiceEndpoint( + LocalStackContainer awsContainer, LocalStackContainer.Service service) { + return awsContainer.getEndpointOverride(service).toString(); } public static String queueUrl(LocalStackContainer awsContainer, String queue) { - return awsServiceEndpoint(awsContainer, SQS) + "/queue/" + queue; + return awsServiceEndpoint(awsContainer, SQS) + "/000000000000/" + queue; } public static AmazonSQS sqsClient(LocalStackContainer awsContainer, String region) { - EndpointConfiguration endpointConfiguration = new EndpointConfiguration(awsServiceEndpoint(awsContainer, SQS), - region); - return AmazonSQSClientBuilder.standard().withEndpointConfiguration(endpointConfiguration).build(); + EndpointConfiguration endpointConfiguration = + new EndpointConfiguration(awsServiceEndpoint(awsContainer, SQS), region); + return AmazonSQSClientBuilder.standard() + .withEndpointConfiguration(endpointConfiguration) + .build(); } public static AmazonS3 s3Client(LocalStackContainer awsContainer, String region) { - EndpointConfiguration endpointConfiguration = new EndpointConfiguration(awsServiceEndpoint(awsContainer, S3), - region); + EndpointConfiguration endpointConfiguration = + new EndpointConfiguration(awsServiceEndpoint(awsContainer, S3), region); // build with disableChunkedEncoding to be able to create empty files - return AmazonS3ClientBuilder - .standard() + return AmazonS3ClientBuilder.standard() .withEndpointConfiguration(endpointConfiguration) + .withPathStyleAccessEnabled(true) .disableChunkedEncoding() .build(); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java index d64e67ae..a298d14c 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -45,19 +43,24 @@ public HiveTestUtils(HiveMetaStoreClient metastoreClient) { this.metastoreClient = metastoreClient; } - private final List DATA_COLUMNS = Arrays - .asList(new FieldSchema("id", "bigint", ""), new FieldSchema("name", "string", ""), + private final List DATA_COLUMNS = + Arrays.asList( + new FieldSchema("id", "bigint", ""), + new FieldSchema("name", "string", ""), new FieldSchema("city", "tinyint", "")); - private final List PARTITION_COLUMNS = Arrays - .asList(new FieldSchema("event_date", "string", ""), new FieldSchema("event_hour", "string", ""), + private final List PARTITION_COLUMNS = + Arrays.asList( + new FieldSchema("event_date", "string", ""), + new FieldSchema("event_hour", "string", ""), new FieldSchema("event_type", "string", "")); public Table createTable(String path, String tableName, boolean partitioned) throws TException { return createTable(path, tableName, partitioned, true); } - public Table createTable(String path, String tableName, boolean partitioned, boolean withBeekeeperProperty) + public Table createTable( + String path, String tableName, boolean partitioned, boolean withBeekeeperProperty) throws TException { Table hiveTable = new Table(); hiveTable.setDbName(DATABASE_NAME_VALUE); @@ -88,19 +91,22 @@ public Table createTable(String path, String tableName, boolean partitioned, boo * @param path Path of the table * @param hiveTable Table to add partitions to * @param partitionValues The list of partition values, e.g. ["2020-01-01", "0", "A"] - * @throws Exception May be thrown if there is a problem when trying to write the data to the file, or when the client - * adds the partition to the table. + * @throws Exception May be thrown if there is a problem when trying to write the data to the + * file, or when the client adds the partition to the table. */ - public void addPartitionsToTable(String path, Table hiveTable, List partitionValues) throws Exception { + public void addPartitionsToTable(String path, Table hiveTable, List partitionValues) + throws Exception { String eventDate = "/event_date=" + partitionValues.get(0); // 2020-01-01 String eventHour = eventDate + "/event_hour=" + partitionValues.get(1); // 0 String eventType = eventHour + "/event_type=" + partitionValues.get(2); // A URI partitionUri = URI.create(path + eventType); - metastoreClient - .add_partitions(Collections - .singletonList(newTablePartition(hiveTable, - List.of(partitionValues.get(0), partitionValues.get(1), partitionValues.get(2)), partitionUri))); + metastoreClient.add_partitions( + Collections.singletonList( + newTablePartition( + hiveTable, + List.of(partitionValues.get(0), partitionValues.get(1), partitionValues.get(2)), + partitionUri))); } private Partition newTablePartition(Table hiveTable, List values, URI location) { @@ -113,8 +119,12 @@ private Partition newTablePartition(Table hiveTable, List values, URI lo return partition; } - public Table createTableWithProperties(String path, String tableName, boolean partitioned, - Map tableProperties, boolean withBeekeeperProperty) + public Table createTableWithProperties( + String path, + String tableName, + boolean partitioned, + Map tableProperties, + boolean withBeekeeperProperty) throws TException { Table hiveTable = new Table(); hiveTable.setDbName(DATABASE_NAME_VALUE); @@ -144,8 +154,9 @@ public Table createTableWithProperties(String path, String tableName, boolean pa return hiveTable; } - public Table createTableWithDeletionProperties(String path, String tableName, boolean partitioned, - boolean shouldTableBeDeletedIfEmpty) throws TException { + public Table createTableWithDeletionProperties( + String path, String tableName, boolean partitioned, boolean shouldTableBeDeletedIfEmpty) + throws TException { Map tableProperties = new HashMap<>(); if (shouldTableBeDeletedIfEmpty) { tableProperties.put("beekeeper.expired.data.table.deletion.enabled", "true"); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index 17af29a5..aa2bc940 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -39,23 +37,28 @@ public void close() throws SQLException { connection.close(); } - public void insertToTable(String database, String table, String fields, String values) throws SQLException { - connection.createStatement().executeUpdate(format(INSERT_TO_TABLE, database, table, fields, values)); + public void insertToTable(String database, String table, String fields, String values) + throws SQLException { + connection + .createStatement() + .executeUpdate(format(INSERT_TO_TABLE, database, table, fields, values)); } - public int getTableRowCount(String database, String table, String additionalFilters) throws SQLException { + public int getTableRowCount(String database, String table, String additionalFilters) + throws SQLException { return getTableRowCount(format(SELECT_TABLE, database, table, additionalFilters)); } private int getTableRowCount(String statementString) throws SQLException { - Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, - ResultSet.CONCUR_READ_ONLY); + Statement statement = + connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY); ResultSet resultSet = statement.executeQuery(statementString); resultSet.last(); return resultSet.getRow(); } - public ResultSet getTableRows(String database, String table, String additionalFilters) throws SQLException { + public ResultSet getTableRows(String database, String table, String additionalFilters) + throws SQLException { return getTableRows(format(SELECT_TABLE, database, table, additionalFilters)); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java index aff31b49..4b6e9ce2 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -39,11 +37,12 @@ public RestResponsePage( @JsonProperty("first") boolean first, @JsonProperty("numberOfElements") int numberOfElements, @JsonProperty("empty") boolean empty) { - // If the page size is 0, we override the value to 1. This is because the jackson dependency does not allow to - // create empty pages, which makes it hard to test what happens when the table is not found and an empty page is + // If the page size is 0, we override the value to 1. This is because the jackson dependency + // does not allow to + // create empty pages, which makes it hard to test what happens when the table is not found and + // an empty page is // returned. // By overriding that value to 1, it is possible to create the 'empty' page and do the testing. super(content, PageRequest.of(number, size == 0 ? 1 : size), totalElements); } - } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java index d254e755..a7c0307a 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java index 5eb2b2a0..43c1515d 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -40,15 +38,17 @@ public class ResultSetToHousekeepingEntityMapper { public static HousekeepingPath mapToHousekeepingPath(ResultSet resultSet) throws SQLException { - return HousekeepingPath - .builder() + return HousekeepingPath.builder() .id(resultSet.getLong(ID_FIELD)) .path(resultSet.getString(PATH_FIELD)) .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) .tableName(resultSet.getString(TABLE_NAME_FIELD)) - .housekeepingStatus(HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) - .creationTimestamp(Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) - .modifiedTimestamp(Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) + .housekeepingStatus( + HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) + .creationTimestamp( + Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) + .modifiedTimestamp( + Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) .cleanupDelay(PeriodDuration.parse(resultSet.getString(CLEANUP_DELAY_FIELD))) .cleanupAttempts(resultSet.getInt(CLEANUP_ATTEMPTS_FIELD)) .clientId(resultSet.getString(CLIENT_ID_FIELD)) @@ -56,17 +56,20 @@ public static HousekeepingPath mapToHousekeepingPath(ResultSet resultSet) throws .build(); } - public static HousekeepingMetadata mapToHousekeepingMetadata(ResultSet resultSet) throws SQLException { - return HousekeepingMetadata - .builder() + public static HousekeepingMetadata mapToHousekeepingMetadata(ResultSet resultSet) + throws SQLException { + return HousekeepingMetadata.builder() .id(resultSet.getLong(ID_FIELD)) .path(resultSet.getString(PATH_FIELD)) .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) .tableName(resultSet.getString(TABLE_NAME_FIELD)) .partitionName(resultSet.getString(PARTITION_NAME_FIELD)) - .housekeepingStatus(HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) - .creationTimestamp(Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) - .modifiedTimestamp(Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) + .housekeepingStatus( + HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) + .creationTimestamp( + Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) + .modifiedTimestamp( + Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) .cleanupDelay(PeriodDuration.parse(resultSet.getString(CLEANUP_DELAY_FIELD))) .cleanupAttempts(resultSet.getInt(CLEANUP_ATTEMPTS_FIELD)) .clientId(resultSet.getString(CLIENT_ID_FIELD)) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java index c4b5f244..2f3b10d5 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java index 53893e35..5d1ec551 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.scheduler.apiary.context; @@ -29,6 +27,10 @@ import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.retry.annotation.EnableRetry; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; +import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.AmazonSQSClientBuilder; + import com.expedia.apiary.extensions.receiver.common.event.AddPartitionEvent; import com.expedia.apiary.extensions.receiver.common.event.AlterPartitionEvent; import com.expedia.apiary.extensions.receiver.common.event.AlterTableEvent; @@ -64,26 +66,42 @@ import com.hotels.hcommon.hive.metastore.client.supplier.HiveMetaStoreClientSupplier; @Configuration -@ComponentScan(basePackages = { "com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.scheduler" }) -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +@ComponentScan( + basePackages = {"com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.scheduler"}) +@EntityScan(basePackages = {"com.expediagroup.beekeeper.core"}) +@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) @EnableRetry(proxyTargetClass = true) public class CommonBeans { @Bean - public EnumMap schedulerServiceMap(List schedulerServices) { - EnumMap schedulerMap = new EnumMap<>(LifecycleEventType.class); - schedulerServices.forEach(scheduler -> schedulerMap.put(scheduler.getLifecycleEventType(), scheduler)); + public EnumMap schedulerServiceMap( + List schedulerServices) { + EnumMap schedulerMap = + new EnumMap<>(LifecycleEventType.class); + schedulerServices.forEach( + scheduler -> schedulerMap.put(scheduler.getLifecycleEventType(), scheduler)); return schedulerMap; } @Bean(name = "sqsMessageReader") - public MessageReader messageReader(@Value("${properties.apiary.queue-url}") String queueUrl) { - return new SqsMessageReader.Builder(queueUrl).build(); + public MessageReader messageReader( + @Value("${properties.apiary.queue-url}") String queueUrl, + @Value("${properties.sqs.endpoint:}") String sqsEndpoint, + @Value("${properties.sqs.region:us-east-1}") String sqsRegion) { + SqsMessageReader.Builder builder = new SqsMessageReader.Builder(queueUrl); + if (!sqsEndpoint.isEmpty()) { + AmazonSQS amazonSQS = + AmazonSQSClientBuilder.standard() + .withEndpointConfiguration(new EndpointConfiguration(sqsEndpoint, sqsRegion)) + .build(); + builder = builder.withConsumer(amazonSQS); + } + return builder.build(); } @Bean(name = "retryingMessageReader") - public MessageReader retryingMessageReader(@Qualifier("sqsMessageReader") MessageReader messageReader) { + public MessageReader retryingMessageReader( + @Qualifier("sqsMessageReader") MessageReader messageReader) { return new RetryingMessageReader(messageReader); } @@ -96,20 +114,20 @@ public HousekeepingEntityGenerator unreferencedHousekeepingPathGenerator( @Bean(name = "unreferencedHousekeepingPathMessageEventHandler") public MessageEventHandler unreferencedHousekeepingPathMessageEventHandler( @Qualifier("unreferencedHousekeepingPathGenerator") HousekeepingEntityGenerator generator) { - List> eventClasses = List.of( - AlterPartitionEvent.class, - AlterTableEvent.class, - DropPartitionEvent.class, - DropTableEvent.class - ); - - List filters = List.of( - new EventTypeListenerEventFilter(eventClasses), - new LocationOnlyUpdateListenerEventFilter(), - new TableParameterListenerEventFilter(), - new WhitelistedListenerEventFilter(), - new IcebergTableListenerEventFilter() - ); + List> eventClasses = + List.of( + AlterPartitionEvent.class, + AlterTableEvent.class, + DropPartitionEvent.class, + DropTableEvent.class); + + List filters = + List.of( + new EventTypeListenerEventFilter(eventClasses), + new LocationOnlyUpdateListenerEventFilter(), + new TableParameterListenerEventFilter(), + new WhitelistedListenerEventFilter(), + new IcebergTableListenerEventFilter()); return new MessageEventHandler(generator, filters); } @@ -124,18 +142,18 @@ public HousekeepingEntityGenerator expiredHousekeepingMetadataGenerator( @Bean(name = "expiredHousekeepingMetadataMessageEventHandler") public MessageEventHandler expiredHousekeepingMetadataMessageEventHandler( @Qualifier("expiredHousekeepingMetadataGenerator") HousekeepingEntityGenerator generator) { - List> eventClasses = List.of( - CreateTableEvent.class, - AlterTableEvent.class, - AddPartitionEvent.class, - AlterPartitionEvent.class - ); - - List filters = List.of( - new EventTypeListenerEventFilter(eventClasses), - new TableParameterListenerEventFilter(), - new IcebergTableListenerEventFilter() - ); + List> eventClasses = + List.of( + CreateTableEvent.class, + AlterTableEvent.class, + AddPartitionEvent.class, + AlterPartitionEvent.class); + + List filters = + List.of( + new EventTypeListenerEventFilter(eventClasses), + new TableParameterListenerEventFilter(), + new IcebergTableListenerEventFilter()); return new MessageEventHandler(generator, filters); } @@ -143,19 +161,21 @@ public MessageEventHandler expiredHousekeepingMetadataMessageEventHandler( @Bean public BeekeeperEventReader eventReader( @Qualifier("retryingMessageReader") MessageReader messageReader, - @Qualifier("unreferencedHousekeepingPathMessageEventHandler") MessageEventHandler unreferencedHousekeepingPathMessageEventHandler, - @Qualifier("expiredHousekeepingMetadataMessageEventHandler") MessageEventHandler expiredHousekeepingMetadataMessageEventHandler - ) { - List handlers = List.of( - unreferencedHousekeepingPathMessageEventHandler, - expiredHousekeepingMetadataMessageEventHandler - ); + @Qualifier("unreferencedHousekeepingPathMessageEventHandler") + MessageEventHandler unreferencedHousekeepingPathMessageEventHandler, + @Qualifier("expiredHousekeepingMetadataMessageEventHandler") + MessageEventHandler expiredHousekeepingMetadataMessageEventHandler) { + List handlers = + List.of( + unreferencedHousekeepingPathMessageEventHandler, + expiredHousekeepingMetadataMessageEventHandler); return new MessageReaderAdapter(messageReader, handlers); } @Bean - BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + BeekeeperHistoryService beekeeperHistoryService( + BeekeeperHistoryRepository beekeeperHistoryRepository) { return new BeekeeperHistoryService(beekeeperHistoryRepository); } @@ -172,8 +192,8 @@ public CloseableMetaStoreClientFactory metaStoreClientFactory() { } @Bean - Supplier metaStoreClientSupplier(CloseableMetaStoreClientFactory metaStoreClientFactory, - HiveConf hiveConf) { + Supplier metaStoreClientSupplier( + CloseableMetaStoreClientFactory metaStoreClientFactory, HiveConf hiveConf) { String name = "beekeeper-scheduler"; return new HiveMetaStoreClientSupplier(metaStoreClientFactory, hiveConf, name); } @@ -184,7 +204,8 @@ public PartitionIteratorFactory partitionIteratorFactory() { } @Bean(name = "hiveClientFactory") - public HiveClientFactory clientFactory(Supplier metaStoreClientSupplier, + public HiveClientFactory clientFactory( + Supplier metaStoreClientSupplier, PartitionIteratorFactory partitionIteratorFactory) { return new HiveClientFactory(metaStoreClientSupplier, partitionIteratorFactory); } diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java index 2e2441f1..7bd923d2 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.scheduler.apiary.context; @@ -80,13 +78,14 @@ void setUp() { @Test public void validateSchedulerServiceMap() { - EnumMap scheduleMap = commonBeans.schedulerServiceMap(Collections.EMPTY_LIST); + EnumMap scheduleMap = + commonBeans.schedulerServiceMap(Collections.EMPTY_LIST); assertThat(scheduleMap).isInstanceOf(EnumMap.class); } @Test public void validateMessageReader() { - MessageReader reader = commonBeans.messageReader("some_path"); + MessageReader reader = commonBeans.messageReader("some_path", "", "us-east-1"); assertThat(reader).isInstanceOf(SqsMessageReader.class); } @@ -98,57 +97,65 @@ public void validateRetryingMessageReader() { @Test public void validateUnreferencedHousekeepingPathGenerator() { - HousekeepingEntityGenerator generator = commonBeans.unreferencedHousekeepingPathGenerator("P30D"); + HousekeepingEntityGenerator generator = + commonBeans.unreferencedHousekeepingPathGenerator("P30D"); assertThat(generator).isInstanceOf(UnreferencedHousekeepingPathGenerator.class); } @Test public void validateUnreferencedHousekeepingPathMessageEventHandler() { - MessageEventHandler handler = commonBeans.unreferencedHousekeepingPathMessageEventHandler( - unreferencedHousekeepingPathGenerator); + MessageEventHandler handler = + commonBeans.unreferencedHousekeepingPathMessageEventHandler( + unreferencedHousekeepingPathGenerator); assertThat(handler).isInstanceOf(MessageEventHandler.class); } @Test public void validateExpiredHousekeepingMetadataGenerator() { HiveClientFactory mockHiveClientFactory = mock(HiveClientFactory.class); - HousekeepingEntityGenerator generator = commonBeans.expiredHousekeepingMetadataGenerator("P30D", mockHiveClientFactory); + HousekeepingEntityGenerator generator = + commonBeans.expiredHousekeepingMetadataGenerator("P30D", mockHiveClientFactory); assertThat(generator).isInstanceOf(ExpiredHousekeepingMetadataGenerator.class); } @Test public void validateExpiredHousekeepingMetadataMessageEventHandler() { - MessageEventHandler handler = commonBeans.expiredHousekeepingMetadataMessageEventHandler( - expiredHousekeepingMetadataGenerator); + MessageEventHandler handler = + commonBeans.expiredHousekeepingMetadataMessageEventHandler( + expiredHousekeepingMetadataGenerator); assertThat(handler).isInstanceOf(MessageEventHandler.class); } @Test public void validatePathEventReader() { - BeekeeperEventReader reader = commonBeans.eventReader(messageReader, mock(MessageEventHandler.class), - mock(MessageEventHandler.class)); + BeekeeperEventReader reader = + commonBeans.eventReader( + messageReader, mock(MessageEventHandler.class), mock(MessageEventHandler.class)); assertThat(reader).isInstanceOf(BeekeeperEventReader.class); } @Test public void validateUnreferencedHousekeepingPathMessageEventHandlerIncludesIcebergFilter() { - MessageEventHandler handler = commonBeans.unreferencedHousekeepingPathMessageEventHandler( - unreferencedHousekeepingPathGenerator); + MessageEventHandler handler = + commonBeans.unreferencedHousekeepingPathMessageEventHandler( + unreferencedHousekeepingPathGenerator); List filters = handler.getFilters(); assertThat(filters).hasAtLeastOneElementOfType(IcebergTableListenerEventFilter.class); } @Test public void validateExpiredHousekeepingMetadataMessageEventHandlerIncludesIcebergFilter() { - MessageEventHandler handler = commonBeans.expiredHousekeepingMetadataMessageEventHandler( - expiredHousekeepingMetadataGenerator); + MessageEventHandler handler = + commonBeans.expiredHousekeepingMetadataMessageEventHandler( + expiredHousekeepingMetadataGenerator); List filters = handler.getFilters(); assertThat(filters).hasAtLeastOneElementOfType(IcebergTableListenerEventFilter.class); } @Test public void verifyBeekeeperHistoryService() { - BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + BeekeeperHistoryService beekeeperHistoryService = + commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); } @@ -157,8 +164,8 @@ public void verifyMetaStoreClientSupplier() { CloseableMetaStoreClientFactory metaStoreClientFactory = commonBeans.metaStoreClientFactory(); HiveConf hiveConf = Mockito.mock(HiveConf.class); - Supplier metaStoreClientSupplier = commonBeans - .metaStoreClientSupplier(metaStoreClientFactory, hiveConf); + Supplier metaStoreClientSupplier = + commonBeans.metaStoreClientSupplier(metaStoreClientFactory, hiveConf); assertThat(metaStoreClientSupplier).isInstanceOf(HiveMetaStoreClientSupplier.class); } } From bf2fe8987700593ef000308e63e790e08e20a8c7 Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Tue, 24 Mar 2026 12:12:54 -0500 Subject: [PATCH 04/12] fix: fix test failures in metadata-cleanup and path-cleanup modules - PagingMetadataCleanupServiceTest: fix ArgumentCaptor type from HousekeepingPath to HousekeepingEntity to match the actual type passed to PathCleaner.cleanupPath(HousekeepingEntity) - CommonBeans (path-cleanup): remove spurious withPathStyleAccessEnabled(true) from amazonS3Test() which caused URL host to be "endpoint" instead of "bucket.endpoint" as the test expected Co-Authored-By: Claude Sonnet 4.6 --- .../PagingMetadataCleanupServiceTest.java | 236 +++++++++++------- .../path/cleanup/context/CommonBeans.java | 35 ++- 2 files changed, 155 insertions(+), 116 deletions(-) diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java index b022d3b2..17db4390 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.metadata.cleanup.service; @@ -61,8 +59,8 @@ import com.expediagroup.beekeeper.cleanup.hive.HiveClientFactory; import com.expediagroup.beekeeper.cleanup.metadata.MetadataCleaner; import com.expediagroup.beekeeper.cleanup.path.PathCleaner; +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; @@ -73,17 +71,20 @@ @ExtendWith(SpringExtension.class) @ExtendWith(MockitoExtension.class) -@ContextConfiguration(classes = { TestApplication.class }, loader = AnnotationConfigContextLoader.class) +@ContextConfiguration( + classes = {TestApplication.class}, + loader = AnnotationConfigContextLoader.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) public class PagingMetadataCleanupServiceTest { - public static final List TABLE_PATHS = List.of("s3://bucket/table", "s3://bucket/table", "s3://bucket/table"); - public static final List PARTITION_PATHS = List - .of("s3://bucket/table/1", "s3://bucket/table/2", "s3://bucket/table/3"); + public static final List TABLE_PATHS = + List.of("s3://bucket/table", "s3://bucket/table", "s3://bucket/table"); + public static final List PARTITION_PATHS = + List.of("s3://bucket/table/1", "s3://bucket/table/2", "s3://bucket/table/3"); private final LocalDateTime localNow = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC); private PagingMetadataCleanupService pagingCleanupService; private @Captor ArgumentCaptor metadataCaptor; - private @Captor ArgumentCaptor pathCaptor; + private @Captor ArgumentCaptor pathCaptor; private @Captor ArgumentCaptor hiveClientCaptor; private @Autowired HousekeepingMetadataRepository metadataRepository; private @MockBean MetadataCleaner metadataCleaner; @@ -99,7 +100,8 @@ public class PagingMetadataCleanupServiceTest { @BeforeEach public void init() { - when(metadataCleaner.tableExists(Mockito.any(), Mockito.anyString(), Mockito.anyString())).thenReturn(true); + when(metadataCleaner.tableExists(Mockito.any(), Mockito.anyString(), Mockito.anyString())) + .thenReturn(true); when(metadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(true); Map properties = new HashMap<>(); properties.put(UNREFERENCED.getTableParameterName(), "true"); @@ -108,8 +110,13 @@ public void init() { .thenReturn(properties); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); - handler = new ExpiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner, - beekeeperHistoryService); + handler = + new ExpiredMetadataHandler( + hiveClientFactory, + metadataRepository, + metadataCleaner, + pathCleaner, + beekeeperHistoryService); handlers = List.of(handler); pagingCleanupService = new PagingMetadataCleanupService(handlers, 2, false); } @@ -118,14 +125,17 @@ public void init() { public void typicalUnpartitioned() { List tables = List.of("table1", "table2", "table3"); - IntStream - .range(0, tables.size()) - .forEach(i -> metadataRepository - .save(createHousekeepingMetadata(tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); + IntStream.range(0, tables.size()) + .forEach( + i -> + metadataRepository.save( + createHousekeepingMetadata( + tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(3)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(3)) + .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0), tables.get(1), tables.get(2)); @@ -134,10 +144,13 @@ public void typicalUnpartitioned() { .extracting("path") .containsExactly(TABLE_PATHS.get(0), TABLE_PATHS.get(1), TABLE_PATHS.get(2)); - metadataRepository.findAll().forEach(housekeepingMetadata -> { - assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); - assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); - }); + metadataRepository + .findAll() + .forEach( + housekeepingMetadata -> { + assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); + assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); + }); pagingCleanupService.cleanUp(Instant.now()); verifyNoMoreInteractions(pathCleaner); @@ -149,14 +162,17 @@ public void typicalDryRunEnabled() { List tables = List.of("table1", "table2", "table3"); - IntStream - .range(0, tables.size()) - .forEach(i -> metadataRepository - .save(createHousekeepingMetadata(tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); + IntStream.range(0, tables.size()) + .forEach( + i -> + metadataRepository.save( + createHousekeepingMetadata( + tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(3)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(3)) + .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0), tables.get(1), tables.get(2)); @@ -165,24 +181,30 @@ public void typicalDryRunEnabled() { .extracting("path") .containsExactly(TABLE_PATHS.get(0), TABLE_PATHS.get(1), TABLE_PATHS.get(2)); - metadataRepository.findAll().forEach(housekeepingMetadata -> { - assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(0); - assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(SCHEDULED); - }); + metadataRepository + .findAll() + .forEach( + housekeepingMetadata -> { + assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(0); + assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(SCHEDULED); + }); } @Test public void typicalPartitioned() { List tables = List.of("table1", "table2", "table3"); - IntStream - .range(0, tables.size()) - .forEach(i -> metadataRepository - .save(createHousekeepingMetadata(tables.get(i), PARTITION_PATHS.get(i), PARTITION_NAME, SCHEDULED))); + IntStream.range(0, tables.size()) + .forEach( + i -> + metadataRepository.save( + createHousekeepingMetadata( + tables.get(i), PARTITION_PATHS.get(i), PARTITION_NAME, SCHEDULED))); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(3)).dropPartition(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(3)) + .dropPartition(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0), tables.get(1), tables.get(2)); @@ -191,10 +213,13 @@ public void typicalPartitioned() { .extracting("path") .containsExactly(PARTITION_PATHS.get(0), PARTITION_PATHS.get(1), PARTITION_PATHS.get(2)); - metadataRepository.findAll().forEach(housekeepingMetadata -> { - assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); - assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); - }); + metadataRepository + .findAll() + .forEach( + housekeepingMetadata -> { + assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); + assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); + }); pagingCleanupService.cleanUp(Instant.now()); verifyNoMoreInteractions(pathCleaner); @@ -202,13 +227,15 @@ public void typicalPartitioned() { @Test public void mixOfScheduledAndFailedPaths() { - List tables = List - .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), + List tables = + List.of( + createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED)); tables.forEach(table -> metadataRepository.save(table)); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(2)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(2)) + .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0).getTableName(), tables.get(1).getTableName()); @@ -220,8 +247,9 @@ public void mixOfScheduledAndFailedPaths() { @Test public void mixOfAllPaths() { - List tables = List - .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), + List tables = + List.of( + createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED), createHousekeepingMetadata("table3", "s3://bucket/some_foobar", null, DELETED), createHousekeepingMetadata("table4", "s3://bucket/some_foobar", null, DISABLED), @@ -230,7 +258,8 @@ public void mixOfAllPaths() { tables.forEach(path -> metadataRepository.save(path)); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(2)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(2)) + .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0).getTableName(), tables.get(1).getTableName()); @@ -242,20 +271,21 @@ public void mixOfAllPaths() { @Test public void metadataCleanerException() { - Mockito - .doNothing() + Mockito.doNothing() .doThrow(new RuntimeException("Error")) .when(metadataCleaner) .dropTable(Mockito.any(HousekeepingMetadata.class), Mockito.any(HiveClient.class)); - List tables = List - .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), + List tables = + List.of( + createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, SCHEDULED)); tables.forEach(table -> metadataRepository.save(table)); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(2)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(2)) + .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0).getTableName(), tables.get(1).getTableName()); @@ -277,40 +307,51 @@ public void metadataCleanerException() { @Test public void invalidPaths() { - List tables = List - .of(createHousekeepingMetadata("table1", "s3://invalid", null, SCHEDULED), + List tables = + List.of( + createHousekeepingMetadata("table1", "s3://invalid", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://invalid/path", "partition", SCHEDULED)); metadataRepository.saveAll(tables); pagingCleanupService.cleanUp(Instant.now()); - metadataRepository.findAll().forEach(table -> { - assertThat(table.getCleanupAttempts()).isEqualTo(0); - assertThat(table.getHousekeepingStatus()).isEqualTo(SKIPPED); - }); + metadataRepository + .findAll() + .forEach( + table -> { + assertThat(table.getCleanupAttempts()).isEqualTo(0); + assertThat(table.getHousekeepingStatus()).isEqualTo(SKIPPED); + }); } @Test @Timeout(value = 10) void doNotInfiniteLoopOnRepeatedFailures() { - List tables = List - .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, FAILED), + List tables = + List.of( + createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, FAILED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED), createHousekeepingMetadata("table3", "s3://bucket/some_foobar", null, FAILED)); - doThrow(new RuntimeException("Error")).when(metadataCleaner).dropTable(Mockito.any(), Mockito.any()); + doThrow(new RuntimeException("Error")) + .when(metadataCleaner) + .dropTable(Mockito.any(), Mockito.any()); for (int i = 0; i < 5; i++) { int finalI = i; - tables.forEach(path -> { - if (finalI == 0) { - metadataRepository.save(path); - } - }); + tables.forEach( + path -> { + if (finalI == 0) { + metadataRepository.save(path); + } + }); pagingCleanupService.cleanUp(Instant.now()); - metadataRepository.findAll().forEach(table -> { - assertThat(table.getCleanupAttempts()).isEqualTo(finalI + 1); - assertThat(table.getHousekeepingStatus()).isEqualTo(FAILED); - }); + metadataRepository + .findAll() + .forEach( + table -> { + assertThat(table.getCleanupAttempts()).isEqualTo(finalI + 1); + assertThat(table.getHousekeepingStatus()).isEqualTo(FAILED); + }); } } @@ -319,36 +360,37 @@ void doNotInfiniteLoopOnRepeatedFailures() { void doNotInfiniteLoopOnDryRunCleanup() { pagingCleanupService = new PagingMetadataCleanupService(handlers, 2, true); - List tables = List - .of(createHousekeepingMetadata("table1", "s3://some_foo", null, SCHEDULED), + List tables = + List.of( + createHousekeepingMetadata("table1", "s3://some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://some_foo", null, SCHEDULED), createHousekeepingMetadata("table3", "s3://some_foo", null, SCHEDULED)); metadataRepository.saveAll(tables); pagingCleanupService.cleanUp(Instant.now()); - metadataRepository.findAll().forEach(table -> { - assertThat(table.getCleanupAttempts()).isEqualTo(0); - assertThat(table.getHousekeepingStatus()).isEqualTo(SCHEDULED); - }); + metadataRepository + .findAll() + .forEach( + table -> { + assertThat(table.getCleanupAttempts()).isEqualTo(0); + assertThat(table.getHousekeepingStatus()).isEqualTo(SCHEDULED); + }); } private HousekeepingMetadata createHousekeepingMetadata( - String tableName, - String path, - String partitionName, - HousekeepingStatus housekeepingStatus) { - HousekeepingMetadata metadata = HousekeepingMetadata - .builder() - .path(path) - .databaseName("database") - .tableName(tableName) - .partitionName(partitionName) - .housekeepingStatus(housekeepingStatus) - .creationTimestamp(localNow) - .modifiedTimestamp(localNow) - .cleanupDelay(PeriodDuration.of(Duration.parse("P30D"))) - .cleanupAttempts(0) - .lifecycleType(EXPIRED.toString()) - .build(); + String tableName, String path, String partitionName, HousekeepingStatus housekeepingStatus) { + HousekeepingMetadata metadata = + HousekeepingMetadata.builder() + .path(path) + .databaseName("database") + .tableName(tableName) + .partitionName(partitionName) + .housekeepingStatus(housekeepingStatus) + .creationTimestamp(localNow) + .modifiedTimestamp(localNow) + .cleanupDelay(PeriodDuration.of(Duration.parse("P30D"))) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); metadata.setCleanupTimestamp(localNow); return metadata; diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java index a634454a..b7c5f8f3 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java @@ -1,16 +1,14 @@ /** - * Copyright (C) 2019-2025 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + *

http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and + *

Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.path.cleanup.context; @@ -49,9 +47,9 @@ @Configuration @EnableScheduling -@ComponentScan({ "com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.cleanup" }) -@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) -@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) +@ComponentScan({"com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.cleanup"}) +@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) +@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) public class CommonBeans { @Bean @@ -74,20 +72,18 @@ public AmazonS3 amazonS3Test() { @Bean public BytesDeletedReporter bytesDeletedReporter( - MeterRegistry meterRegistry, - @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { + MeterRegistry meterRegistry, @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { return new BytesDeletedReporter(meterRegistry, dryRunEnabled); } @Bean - public S3Client s3Client(AmazonS3 amazonS3, @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { + public S3Client s3Client( + AmazonS3 amazonS3, @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { return new S3Client(amazonS3, dryRunEnabled); } @Bean(name = "s3PathCleaner") - public PathCleaner pathCleaner( - S3Client s3Client, - BytesDeletedReporter bytesDeletedReporter) { + public PathCleaner pathCleaner(S3Client s3Client, BytesDeletedReporter bytesDeletedReporter) { return new S3PathCleaner(s3Client, new S3SentinelFilesCleaner(s3Client), bytesDeletedReporter); } @@ -112,7 +108,8 @@ public DisableTablesService disableTablesService() { } @Bean - public BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + public BeekeeperHistoryService beekeeperHistoryService( + BeekeeperHistoryRepository beekeeperHistoryRepository) { return new BeekeeperHistoryService(beekeeperHistoryRepository); } } From 7fc3263c3232df74957287d49a90d5fbcc6413f8 Mon Sep 17 00:00:00 2001 From: Milton Ortegon Date: Tue, 24 Mar 2026 16:25:43 -0500 Subject: [PATCH 05/12] fix: truncate LocalDateTime to microseconds in repository tests After Java 21 / Spring Boot 3.2.12 upgrade, LocalDateTime.now() produces nanosecond-precision timestamps but H2 (MySQL mode) stores DATETIME with microsecond precision, causing round-trip comparison failures and incorrect query boundary evaluations. Truncate CREATION_TIMESTAMP and compare() helper to MICROS to align in-memory values with DB-stored values. Co-Authored-By: Claude Opus 4.6 --- .../HousekeepingMetadataRepositoryTest.java | 11 +++++++---- .../repository/HousekeepingPathRepositoryTest.java | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java index ef6df464..6a405baa 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepositoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import static java.time.temporal.ChronoUnit.DAYS; import static java.time.temporal.ChronoUnit.HOURS; +import static java.time.temporal.ChronoUnit.MICROS; import static java.time.temporal.ChronoUnit.MONTHS; import static org.assertj.core.api.Assertions.assertThat; @@ -71,7 +72,7 @@ public class HousekeepingMetadataRepositoryTest { private static final String DATABASE_NAME = "database"; private static final String TABLE_NAME = "table"; private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; - private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")).truncatedTo(MICROS); private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); private static final LocalDateTime CLEANUP_TIMESTAMP = CREATION_TIMESTAMP.plus(CLEANUP_DELAY); @@ -525,8 +526,10 @@ private void compare(HousekeepingMetadata expected, HousekeepingMetadata actual) assertThat(actual.getTableName()).isEqualTo(expected.getTableName()); assertThat(actual.getPartitionName()).isEqualTo(expected.getPartitionName()); assertThat(actual.getHousekeepingStatus()).isEqualTo(expected.getHousekeepingStatus()); - assertThat(actual.getCreationTimestamp()).isEqualTo(expected.getCreationTimestamp()); - assertThat(actual.getModifiedTimestamp()).isEqualTo(expected.getModifiedTimestamp()); + assertThat(actual.getCreationTimestamp().truncatedTo(MICROS)) + .isEqualTo(expected.getCreationTimestamp().truncatedTo(MICROS)); + assertThat(actual.getModifiedTimestamp().truncatedTo(MICROS)) + .isEqualTo(expected.getModifiedTimestamp().truncatedTo(MICROS)); assertThat(actual.getCleanupDelay()).isEqualTo(expected.getCleanupDelay()); assertThat(actual.getCleanupAttempts()).isEqualTo(expected.getCleanupAttempts()); assertThat(actual.getLifecycleType()).isEqualTo(expected.getLifecycleType()); diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java index 6aba9e9d..a79f2621 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepositoryTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2026 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ import static java.time.temporal.ChronoUnit.DAYS; import static java.time.temporal.ChronoUnit.HOURS; +import static java.time.temporal.ChronoUnit.MICROS; import static java.time.temporal.ChronoUnit.MONTHS; import static org.assertj.core.api.Assertions.assertThat; @@ -64,7 +65,7 @@ @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) public class HousekeepingPathRepositoryTest { - private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + private static final LocalDateTime CREATION_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")).truncatedTo(MICROS); private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("PT1H"); private static final LocalDateTime CLEANUP_TIMESTAMP = CREATION_TIMESTAMP.plus(CLEANUP_DELAY); From 64779f1c54a53cf0ce833f5cdb41665aa27cba76 Mon Sep 17 00:00:00 2001 From: Milton Ortegon Date: Tue, 24 Mar 2026 16:51:59 -0500 Subject: [PATCH 06/12] fix: truncate LocalDateTime to microseconds in vacuum tool repository test Same nanosecond vs microsecond precision issue as in beekeeper-core tests. H2 (MySQL mode) truncates DATETIME to microseconds, causing the cleanupTimestamp round-trip comparison to fail after Java 21 upgrade. Co-Authored-By: Claude Opus 4.6 --- .gitignore | 2 + .../api/conf/SwaggerConfiguration.java | 16 +- .../api/controller/BeekeeperController.java | 107 +++------ .../api/error/BeekeeperExceptionHandler.java | 24 +- .../HousekeepingMetadataResponse.java | 27 ++- .../beekeeper/api/TestApplication.java | 14 +- .../beekeeper/cleanup/TestApplication.java | 14 +- .../beekeeper/cleanup/aws/S3ClientTest.java | 48 ++-- .../cleanup/aws/S3DryRunPathCleanerTest.java | 54 ++--- .../cleanup/aws/S3PathCleanerTest.java | 82 +++---- .../aws/S3SentinelFilesCleanerTest.java | 36 ++- .../core/model/HousekeepingPath.java | 15 +- .../BeekeeperHistoryRepository.java | 6 +- .../HousekeepingMetadataRepository.java | 119 ++++----- .../HousekeepingPathRepository.java | 17 +- ...rDryRunMetadataCleanupIntegrationTest.java | 104 ++++---- ...eeperDryRunPathCleanupIntegrationTest.java | 34 +-- ...etadataSchedulerApiaryIntegrationTest.java | 111 ++++----- .../BeekeeperIntegrationTestBase.java | 225 +++++++----------- .../integration/CommonTestVariables.java | 19 +- .../model/AddPartitionSqsMessage.java | 32 +-- .../model/AlterPartitionSqsMessage.java | 42 ++-- .../model/AlterTableSqsMessage.java | 32 ++- .../model/CreateTableSqsMessage.java | 29 ++- .../model/DropPartitionSqsMessage.java | 31 +-- .../model/DropTableSqsMessage.java | 23 +- .../integration/model/SqsMessage.java | 53 ++--- .../integration/model/SqsMessageTest.java | 78 +++--- .../utils/BeekeeperApiTestClient.java | 76 +++--- .../integration/utils/HiveTestUtils.java | 61 ++--- .../integration/utils/MySqlTestUtils.java | 33 ++- .../integration/utils/RestResponsePage.java | 23 +- .../ResultSetToBeekeeperHistoryMapper.java | 16 +- .../ResultSetToHousekeepingEntityMapper.java | 43 ++-- .../integration/utils/TestAppender.java | 16 +- .../path/cleanup/context/CommonBeans.java | 35 +-- .../repository/BeekeeperRepositoryTest.java | 4 +- 37 files changed, 741 insertions(+), 960 deletions(-) diff --git a/.gitignore b/.gitignore index 43147406..4cd5ca05 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ target/ # HMS data metastore_db derby.log + +CLAUDE.md diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java index 212a05d9..b082ad24 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/conf/SwaggerConfiguration.java @@ -1,14 +1,16 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.conf; @@ -21,7 +23,7 @@ import io.swagger.v3.oas.annotations.servers.Server; @Configuration -@OpenAPIDefinition(servers = {@Server(url = "/", description = "Default Server URL")}) +@OpenAPIDefinition(servers = { @Server(url = "/", description = "Default Server URL") }) public class SwaggerConfiguration { @Bean diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index d7b6a03c..3690683e 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -1,14 +1,16 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.controller; @@ -49,9 +51,7 @@ public BeekeeperController(HousekeepingEntityService housekeepingEntityService) this.housekeepingEntityService = housekeepingEntityService; } - @RequestMapping( - value = "/database/{databaseName}/table/{tableName}/metadata", - method = RequestMethod.GET) + @RequestMapping(value = "/database/{databaseName}/table/{tableName}/metadata", method = RequestMethod.GET) @Parameter(name = "tableName", in = ParameterIn.PATH) @Parameter(name = "databaseName", in = ParameterIn.PATH) @Parameter(name = "path", in = ParameterIn.QUERY) @@ -65,42 +65,17 @@ public BeekeeperController(HousekeepingEntityService housekeepingEntityService) public ResponseEntity> getAllMetadata( @PathVariable String databaseName, @PathVariable String tableName, - @Parameter(hidden = true) - @And( - value = { - @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), - @Spec( - path = "databaseName", - pathVars = "databaseName", - spec = EqualIgnoreCase.class), - @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), - @Spec( - path = "partitionName", - params = "partition_name", - spec = EqualIgnoreCase.class), - @Spec( - path = "housekeepingStatus", - params = "housekeeping_status", - spec = EqualIgnoreCase.class), - @Spec( - path = "lifecycleType", - params = "lifecycle_type", - spec = EqualIgnoreCase.class), - @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), - @Spec( - path = "cleanupTimestamp", - params = "deleted_after", - spec = GreaterThan.class), - @Spec( - path = "creationTimestamp", - params = "registered_before", - spec = LessThan.class), - @Spec( - path = "creationTimestamp", - params = "registered_after", - spec = GreaterThan.class) - }) - Specification spec, + @Parameter(hidden = true) @And(value = { + @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), + @Spec(path = "databaseName", pathVars = "databaseName", spec = EqualIgnoreCase.class), + @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), + @Spec(path = "partitionName", params = "partition_name", spec = EqualIgnoreCase.class), + @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), + @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), + @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), + @Spec(path = "cleanupTimestamp", params = "deleted_after", spec = GreaterThan.class), + @Spec(path = "creationTimestamp", params = "registered_before", spec = LessThan.class), + @Spec(path = "creationTimestamp", params = "registered_after", spec = GreaterThan.class) }) Specification spec, @ParameterObject Pageable pageable) { return ResponseEntity.ok(housekeepingEntityService.getAllMetadata(spec, pageable)); } @@ -121,43 +96,19 @@ public ResponseEntity> getAllMetadata( public ResponseEntity> getAllPaths( @PathVariable String databaseName, @PathVariable String tableName, - @Parameter(hidden = true) - @And( - value = { + @Parameter(hidden = true) @And(value = { @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), - @Spec( - path = "databaseName", - pathVars = "databaseName", - spec = EqualIgnoreCase.class), + @Spec(path = "databaseName", pathVars = "databaseName", spec = EqualIgnoreCase.class), @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), - @Spec( - path = "partitionName", - params = "partition_name", - spec = EqualIgnoreCase.class), - @Spec( - path = "housekeepingStatus", - params = "housekeeping_status", - spec = EqualIgnoreCase.class), - @Spec( - path = "lifecycleType", - params = "lifecycle_type", - spec = EqualIgnoreCase.class), + @Spec(path = "partitionName", params = "partition_name", spec = EqualIgnoreCase.class), + @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), + @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), - @Spec( - path = "cleanupTimestamp", - params = "deleted_after", - spec = GreaterThan.class), - @Spec( - path = "creationTimestamp", - params = "registered_before", - spec = LessThan.class), - @Spec( - path = "creationTimestamp", - params = "registered_after", - spec = GreaterThan.class) - }) - Specification spec, + @Spec(path = "cleanupTimestamp", params = "deleted_after", spec = GreaterThan.class), + @Spec(path = "creationTimestamp", params = "registered_before", spec = LessThan.class), + @Spec(path = "creationTimestamp", params = "registered_after", spec = GreaterThan.class) }) Specification spec, @ParameterObject Pageable pageable) { return ResponseEntity.ok(housekeepingEntityService.getAllPaths(spec, pageable)); } + } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java index 0c5b2e26..0a04224b 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java @@ -1,44 +1,46 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.error; import java.time.LocalDateTime; +import jakarta.servlet.http.HttpServletRequest; + import org.springframework.data.mapping.PropertyReferenceException; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestControllerAdvice; -import jakarta.servlet.http.HttpServletRequest; - @RestControllerAdvice public class BeekeeperExceptionHandler { /** * Handles invalid sort parameters. * - * @param exception the exception is thrown when an invalid property is referenced + * @param exception the exception is thrown when an invalid property is referenced * @param request the HTTP request * @return a ResponseEntity containing the error response */ + @ExceptionHandler(PropertyReferenceException.class) public ResponseEntity handlePropertyReferenceException( PropertyReferenceException exception, HttpServletRequest request) { - ErrorResponse errorResponse = - ErrorResponse.builder() + ErrorResponse errorResponse = ErrorResponse.builder() .timestamp(LocalDateTime.now().toString()) .status(HttpStatus.BAD_REQUEST.value()) .error(HttpStatus.BAD_REQUEST.getReasonPhrase()) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java index 855bea4a..c662c45f 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponse.java @@ -1,24 +1,27 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api.response; import java.time.LocalDateTime; -import org.hibernate.annotations.UpdateTimestamp; - import jakarta.persistence.EnumType; import jakarta.persistence.Enumerated; + +import org.hibernate.annotations.UpdateTimestamp; + import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Value; @@ -40,9 +43,12 @@ public class HousekeepingMetadataResponse { HousekeepingStatus housekeepingStatus; - @EqualsAndHashCode.Exclude LocalDateTime creationTimestamp; + @EqualsAndHashCode.Exclude + LocalDateTime creationTimestamp; - @EqualsAndHashCode.Exclude @UpdateTimestamp LocalDateTime modifiedTimestamp; + @EqualsAndHashCode.Exclude + @UpdateTimestamp + LocalDateTime modifiedTimestamp; LocalDateTime cleanupTimestamp; @@ -51,4 +57,5 @@ public class HousekeepingMetadataResponse { int cleanupAttempts; String lifecycleType; + } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java index 31f3eb76..4bf3eeed 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/TestApplication.java @@ -1,14 +1,16 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.api; diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java index dd0bdc0f..2e35ee49 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/TestApplication.java @@ -1,14 +1,16 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup; diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java index 0561baff..0f65db6e 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3ClientTest.java @@ -1,14 +1,16 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -56,9 +58,8 @@ class S3ClientTest { private AmazonS3 amazonS3; @Rule - public static LocalStackContainer awsContainer = - new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) - .withServices(S3); + public static LocalStackContainer awsContainer = new LocalStackContainer( + DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); static { awsContainer.start(); @@ -68,12 +69,11 @@ class S3ClientTest { @BeforeEach void setUp() { - amazonS3 = - AmazonS3ClientBuilder.standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) - .build(); + amazonS3 = AmazonS3ClientBuilder + .standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")).build(); amazonS3.createBucket(bucket); emptyBucket(bucket); assertThat(amazonS3.listObjectsV2(bucket).getObjectSummaries()).isEmpty(); @@ -85,16 +85,16 @@ private void emptyBucket(String bucket) { ListObjectsV2Result listObjectsV2Result; String continuationToken = null; do { - ListObjectsV2Request request = - new ListObjectsV2Request() - .withBucketName(bucket) - .withContinuationToken(continuationToken); + ListObjectsV2Request request = new ListObjectsV2Request() + .withBucketName(bucket) + .withContinuationToken(continuationToken); listObjectsV2Result = amazonS3.listObjectsV2(request); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket); - List keys = - listObjectsV2Result.getObjectSummaries().stream() - .map(S3ObjectSummary::getKey) - .collect(Collectors.toList()); + List keys = listObjectsV2Result + .getObjectSummaries() + .stream() + .map(S3ObjectSummary::getKey) + .collect(Collectors.toList()); if (keys.size() > 0) { amazonS3.deleteObjects(deleteObjectsRequest.withKeys(keys.toArray(new String[] {}))); } @@ -212,7 +212,7 @@ void deleteObjectsInDirectory() { } @ParameterizedTest - @ValueSource(ints = {500, 1000, 1500}) + @ValueSource(ints = { 500, 1000, 1500 }) void splitDeleteObjectsInDirectory(final int totalObjects) { ArrayList keys = new ArrayList<>(); for (int i = 1; i <= totalObjects; i++) { diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java index 326c555b..cc605cfa 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3DryRunPathCleanerTest.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2024 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -62,35 +64,32 @@ class S3DryRunPathCleanerTest { private S3PathCleaner s3DryRunPathCleaner; @Container - public static LocalStackContainer awsContainer = - new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) - .withServices(S3); + public static LocalStackContainer awsContainer = new LocalStackContainer( + DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); @BeforeEach void setUp() { String S3_ENDPOINT = awsContainer.getEndpointOverride(S3).toString(); - amazonS3 = - AmazonS3ClientBuilder.standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) - .build(); + amazonS3 = AmazonS3ClientBuilder + .standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) + .build(); amazonS3.createBucket(bucket); amazonS3 .listObjectsV2(bucket) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(bucket, object.getKey())); S3Client s3Client = new S3Client(amazonS3, dryRunEnabled); - s3DryRunPathCleaner = - new S3PathCleaner(s3Client, new S3SentinelFilesCleaner(s3Client), bytesDeletedReporter); - housekeepingPath = - HousekeepingPath.builder() - .path(absolutePath) - .tableName(tableName) - .databaseName(databaseName) - .creationTimestamp(LocalDateTime.now()) - .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) - .build(); + s3DryRunPathCleaner = new S3PathCleaner(s3Client, new S3SentinelFilesCleaner(s3Client), bytesDeletedReporter); + housekeepingPath = HousekeepingPath + .builder() + .path(absolutePath) + .tableName(tableName) + .databaseName(databaseName) + .creationTimestamp(LocalDateTime.now()) + .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) + .build(); } @Test @@ -208,7 +207,6 @@ void deleteTable() { @Test void pathDoesNotExist() { - assertThatCode(() -> s3DryRunPathCleaner.cleanupPath(housekeepingPath)) - .doesNotThrowAnyException(); + assertThatCode(() -> s3DryRunPathCleaner.cleanupPath(housekeepingPath)).doesNotThrowAnyException(); } } diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java index b3675e99..8157a90c 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java @@ -1,14 +1,16 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -80,9 +82,8 @@ class S3PathCleanerTest { private S3PathCleaner s3PathCleaner; @Rule - public static LocalStackContainer awsContainer = - new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) - .withServices(S3); + public static LocalStackContainer awsContainer = new LocalStackContainer( + DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); static { awsContainer.start(); @@ -92,12 +93,11 @@ class S3PathCleanerTest { @BeforeEach void setUp() { - amazonS3 = - AmazonS3ClientBuilder.standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) - .build(); + amazonS3 = AmazonS3ClientBuilder + .standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) + .build(); amazonS3.createBucket(bucket); amazonS3 .listObjectsV2(bucket) @@ -109,14 +109,14 @@ void setUp() { s3PathCleaner = new S3PathCleaner(s3Client, s3SentinelFilesCleaner, bytesDeletedReporter); String tableName = "table"; String databaseName = "database"; - housekeepingPath = - HousekeepingPath.builder() - .path(absolutePath) - .tableName(tableName) - .databaseName(databaseName) - .creationTimestamp(LocalDateTime.now()) - .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) - .build(); + housekeepingPath = HousekeepingPath + .builder() + .path(absolutePath) + .tableName(tableName) + .databaseName(databaseName) + .creationTimestamp(LocalDateTime.now()) + .cleanupDelay(PeriodDuration.of(Duration.ofDays(1))) + .build(); } @Test @@ -128,8 +128,7 @@ void typicalForDirectory() { assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isFalse(); - verify(bytesDeletedReporter) - .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -155,8 +154,7 @@ void directoryWithTrailingSlash() { assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isFalse(); - verify(bytesDeletedReporter) - .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -169,8 +167,7 @@ void typicalForFile() { s3PathCleaner.cleanupPath(housekeepingPath); assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isTrue(); - verify(bytesDeletedReporter) - .reportTaggable(content.getBytes().length, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter).reportTaggable(content.getBytes().length, housekeepingPath, FileSystemType.S3); } @Test @@ -184,8 +181,7 @@ void typicalWithSentinelFile() { assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, key2)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, partition1Sentinel)).isFalse(); - verify(bytesDeletedReporter) - .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -248,8 +244,7 @@ void deleteTable() { assertThat(amazonS3.doesObjectExist(bucket, partition1Sentinel)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, parentSentinelFile)).isFalse(); assertThat(amazonS3.doesObjectExist(bucket, tableSentinelFile)).isFalse(); - verify(bytesDeletedReporter) - .reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); + verify(bytesDeletedReporter).reportTaggable(content.getBytes().length * 2, housekeepingPath, FileSystemType.S3); } @Test @@ -260,9 +255,7 @@ void pathDoesNotExist() { @Test void sentinelFilesCleanerThrowsException() { S3SentinelFilesCleaner s3SentinelFilesCleaner = mock(S3SentinelFilesCleaner.class); - doThrow(IllegalArgumentException.class) - .when(s3SentinelFilesCleaner) - .deleteSentinelFiles(absolutePath); + doThrow(IllegalArgumentException.class).when(s3SentinelFilesCleaner).deleteSentinelFiles(absolutePath); amazonS3.putObject(bucket, key1, content); @@ -348,9 +341,7 @@ void noBytesDeletedMetricWhenFileDeletionFails() { void noBytesDeletedMetricWhenDirectoryDeletionFails() { S3Client mockS3Client = mock(S3Client.class); s3PathCleaner = new S3PathCleaner(mockS3Client, s3SentinelFilesCleaner, bytesDeletedReporter); - doThrow(AmazonServiceException.class) - .when(mockS3Client) - .listObjects(bucket, keyRootAsDirectory); + doThrow(AmazonServiceException.class).when(mockS3Client).listObjects(bucket, keyRootAsDirectory); assertThatExceptionOfType(AmazonServiceException.class) .isThrownBy(() -> s3PathCleaner.cleanupPath(housekeepingPath)); @@ -365,11 +356,8 @@ void reportBytesDeletedWhenDirectoryDeletionPartiallyFails() { s3PathCleaner = new S3PathCleaner(mockS3Client, s3SentinelFilesCleaner, bytesDeletedReporter); assertThatExceptionOfType(BeekeeperException.class) .isThrownBy(() -> s3PathCleaner.cleanupPath(housekeepingPath)) - .withMessage( - format( - "Not all files could be deleted at path \"%s/%s\"; deleted 1/2 objects. " - + "Objects not deleted: 'table/id1/partition_1/file2'.", - bucket, keyRootAsDirectory)); + .withMessage(format("Not all files could be deleted at path \"%s/%s\"; deleted 1/2 objects. " + + "Objects not deleted: 'table/id1/partition_1/file2'.", bucket, keyRootAsDirectory)); verify(bytesDeletedReporter).reportTaggable(100L, housekeepingPath, FileSystemType.S3); } @@ -392,10 +380,8 @@ private void mockOneOutOfTwoObjectsDeleted(AmazonS3 mockAmazonS3) { s3ObjectSummary2.setKey(key2); s3ObjectSummary2.setSize(50L); ListObjectsV2Result listObjectsV2Result = mock(ListObjectsV2Result.class); - when(listObjectsV2Result.getObjectSummaries()) - .thenReturn(List.of(s3ObjectSummary, s3ObjectSummary2)); - when(mockAmazonS3.listObjectsV2(any(ListObjectsV2Request.class))) - .thenReturn(listObjectsV2Result); + when(listObjectsV2Result.getObjectSummaries()).thenReturn(List.of(s3ObjectSummary, s3ObjectSummary2)); + when(mockAmazonS3.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(listObjectsV2Result); DeleteObjectsResult.DeletedObject deletedObject = new DeleteObjectsResult.DeletedObject(); deletedObject.setKey(key1); when(mockAmazonS3.deleteObjects(any(DeleteObjectsRequest.class))) diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java index 01922796..1c481a35 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java @@ -1,14 +1,16 @@ /** * Copyright (C) 2019-2026 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.cleanup.aws; @@ -45,9 +47,8 @@ class S3SentinelFilesCleanerTest { private AmazonS3 amazonS3; @Rule - public static LocalStackContainer awsContainer = - new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14.2")) - .withServices(S3); + public static LocalStackContainer awsContainer = new LocalStackContainer( + DockerImageName.parse("localstack/localstack:0.14.2")).withServices(S3); static { awsContainer.start(); @@ -57,15 +58,13 @@ class S3SentinelFilesCleanerTest { @BeforeEach void setUp() { - amazonS3 = - AmazonS3ClientBuilder.standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) - .build(); + amazonS3 =AmazonS3ClientBuilder + .standard() + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")).build(); amazonS3.createBucket(bucket); - amazonS3 - .listObjectsV2(bucket) + amazonS3.listObjectsV2(bucket) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(bucket, object.getKey())); S3Client s3Client = new S3Client(amazonS3, false); @@ -117,8 +116,7 @@ void nonEmptySentinelFile() { @Test void sentinelFileDoesntExist() { amazonS3.putObject(bucket, "table/partition_1", "content"); - assertThatCode(() -> s3SentinelFilesCleaner.deleteSentinelFiles(partition1AbsolutePath)) - .doesNotThrowAnyException(); + assertThatCode(() -> s3SentinelFilesCleaner.deleteSentinelFiles(partition1AbsolutePath)).doesNotThrowAnyException(); } @Test diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java index 78cce53e..7d7da4d4 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingPath.java @@ -125,18 +125,8 @@ public MetricTag getMetricTag() { public String toString() { return format( "%s(path=%s, databaseName=%s, tableName=%s, housekeepingStatus=%s, creationTimestamp=%s, modifiedTimestamp=%s, cleanupTimestamp=%s, cleanupDelay=%s, cleanupAttempts=%s, clientId=%s, lifecycleType=%s)", - HousekeepingPath.class.getSimpleName(), - path, - databaseName, - tableName, - housekeepingStatus, - creationTimestamp, - modifiedTimestamp, - cleanupTimestamp, - cleanupDelay, - cleanupAttempts, - clientId, - lifecycleType); + HousekeepingPath.class.getSimpleName(), path, databaseName, tableName, housekeepingStatus, creationTimestamp, + modifiedTimestamp, cleanupTimestamp, cleanupDelay, cleanupAttempts, clientId, lifecycleType); } private LocalDateTime configureCleanupTimestamp() { @@ -148,4 +138,5 @@ private LocalDateTime configureCleanupTimestamp() { } return creationTimestamp.plus(cleanupDelay); } + } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java index 8fa5042d..99636269 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java @@ -23,10 +23,8 @@ import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; -public interface BeekeeperHistoryRepository - extends PagingAndSortingRepository, - CrudRepository, - JpaSpecificationExecutor { +public interface BeekeeperHistoryRepository extends PagingAndSortingRepository, + CrudRepository, JpaSpecificationExecutor { @Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle") Slice findRecordsByLifecycleType( diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index 9c3581ef..6bc47d6a 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -29,36 +29,31 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; public interface HousekeepingMetadataRepository - extends PagingAndSortingRepository, - CrudRepository, + extends PagingAndSortingRepository, CrudRepository, JpaSpecificationExecutor { - @Query( - value = - "from HousekeepingMetadata t where t.cleanupTimestamp <= :instant " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " - + "and t.modifiedTimestamp <= :instant and t.cleanupAttempts < 10 order by t.modifiedTimestamp") + @Query(value = "from HousekeepingMetadata t where t.cleanupTimestamp <= :instant " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " + + "and t.modifiedTimestamp <= :instant and t.cleanupAttempts < 10 order by t.modifiedTimestamp") Slice findRecordsForCleanupByModifiedTimestamp( @Param("instant") LocalDateTime instant, Pageable pageable); /** * Returns the record that matches the inputs given, if there is one. * - * @implNote To get the record for a partitioned table both the input value and the value of the - * partitionName of the current record must be NULL. + * @implNote To get the record for a partitioned table both the input value and the value of the partitionName of the + * current record must be NULL. * @param databaseName * @param tableName * @param partitionName * @return */ - @Query( - value = - "from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.partitionName = :partitionName or (:partitionName is NULL and t.partitionName is NULL)) " - // To handle special null case - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query(value = "from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.partitionName = :partitionName or (:partitionName is NULL and t.partitionName is NULL)) " + // To handle special null case + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") Optional findRecordForCleanupByDbTableAndPartitionName( @Param("databaseName") String databaseName, @Param("tableName") String tableName, @@ -71,94 +66,80 @@ Optional findRecordForCleanupByDbTableAndPartitionName( * @param tableName * @return */ - @Query( - value = - "select max(cleanupTimestamp) from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query(value = "select max(cleanupTimestamp) from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") LocalDateTime findMaximumCleanupTimestampForDbAndTable( @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method returns the count of all records for a database and table name pair where the - * partitionName is not null. + * This method returns the count of all records for a database and table name pair where the partitionName is not + * null. * * @param databaseName * @param tableName * @return A count of the number of partitions on this table. */ - @Query( - value = - "select count(partitionName) from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query(value = "select count(partitionName) from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") Long countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull( @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method is used for dry runs since the entries are not being updated. It counts the number - * of partitions on a table which have not yet expired, i.e. they will not be cleaned up in this - * instant. + * This method is used for dry runs since the entries are not being updated. It counts the number of partitions on a + * table which have not yet expired, i.e. they will not be cleaned up in this instant. * * @param instant * @param databaseName * @param tableName * @return A count of the number of existing partitions on this table */ - @Query( - value = - "select count(partitionName) from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " - + "and t.cleanupTimestamp >= :instant") + @Query(value = "select count(partitionName) from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " + + "and t.cleanupTimestamp >= :instant") Long countRecordsForDryRunWherePartitionIsNotNullOrExpired( @Param("instant") LocalDateTime instant, @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method deletes the rows for scheduled or failed partitions for the specified {@code - * databaseName} and {@code tableName}. + * This method deletes the rows for scheduled or failed partitions for the specified {@code databaseName} and + * {@code tableName}. * * @param databaseName * @param tableName */ @Modifying - @Query( - value = - "delete from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and t.partitionName is not NULL " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query(value = "delete from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and t.partitionName is not NULL " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") void deleteScheduledOrFailedPartitionRecordsForTable( @Param("databaseName") String databaseName, @Param("tableName") String tableName); /** - * This method returns all table records where the partition name is NULL and the status is - * `SCHEDULED` or `FAILED` + * This method returns all table records where the partition name is NULL and the status is SCHEDULED` or `FAILED` */ - @Query( - value = - "from HousekeepingMetadata t " - + "where t.partitionName is NULL " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query(value = "from HousekeepingMetadata t " + + "where t.partitionName is NULL " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") List findActiveTables(); /** - * This method deletes the rows which have "DELETED" or "DISABLED" status and are older than the - * specified {@code instant}. + * This method deletes the rows which have "DELETED" or "DISABLED" status and are older than the specified + * {@code instant}. * * @param instant */ @Modifying - @Query( - value = - "delete from HousekeepingMetadata t where t.cleanupTimestamp < :instant " - + "and (t.housekeepingStatus = 'DELETED' or t.housekeepingStatus = 'DISABLED')") + @Query(value = "delete from HousekeepingMetadata t where t.cleanupTimestamp < :instant " + + "and (t.housekeepingStatus = 'DELETED' or t.housekeepingStatus = 'DISABLED')") void cleanUpOldDeletedRecords(@Param("instant") LocalDateTime instant); /** @@ -168,13 +149,11 @@ void deleteScheduledOrFailedPartitionRecordsForTable( * @param tableName * @return List of records that match the inputs given. */ - @Query( - value = - "from HousekeepingMetadata t " - + "where t.databaseName = :databaseName " - + "and t.tableName = :tableName " - + "and t.partitionName IS NOT NULL " - + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") + @Query(value = "from HousekeepingMetadata t " + + "where t.databaseName = :databaseName " + + "and t.tableName = :tableName " + + "and t.partitionName IS NOT NULL " + + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") List findRecordsForCleanupByDbAndTableName( @Param("databaseName") String databaseName, @Param("tableName") String tableName); } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java index 4cb8639a..ae3460e8 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java @@ -29,22 +29,17 @@ @Repository public interface HousekeepingPathRepository - extends PagingAndSortingRepository, - CrudRepository, + extends PagingAndSortingRepository, CrudRepository, JpaSpecificationExecutor { - @Query( - value = - "from HousekeepingPath p where p.cleanupTimestamp <= :instant " - + "and (p.housekeepingStatus = 'SCHEDULED' or p.housekeepingStatus = 'FAILED') " - + "and p.modifiedTimestamp <= :instant and p.cleanupAttempts < 10") + @Query(value = "from HousekeepingPath p where p.cleanupTimestamp <= :instant " + + "and (p.housekeepingStatus = 'SCHEDULED' or p.housekeepingStatus = 'FAILED') " + + "and p.modifiedTimestamp <= :instant and p.cleanupAttempts < 10") Slice findRecordsForCleanup( @Param("instant") LocalDateTime instant, Pageable pageable); @Modifying - @Query( - value = - "delete from HousekeepingPath p where p.cleanupTimestamp < :instant " - + "and p.housekeepingStatus = 'DELETED'") + @Query(value = "delete from HousekeepingPath p where p.cleanupTimestamp < :instant " + + "and p.housekeepingStatus = 'DELETED'") void cleanUpOldDeletedRecords(@Param("instant") LocalDateTime instant); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java index 3ba0415a..91f730ef 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java @@ -79,10 +79,8 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; private static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; private static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; - private static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = - "com.amazonaws.services.s3.disableGetObjectMD5Validation"; - private static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = - "com.amazonaws.services.s3.disablePutObjectMD5Validation"; + private static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disableGetObjectMD5Validation"; + private static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disablePutObjectMD5Validation"; private static final String BUCKET = "test-path-bucket"; private static final String TABLE_DATA = "1\tadam\tlondon\n2\tsusan\tglasgow\n"; @@ -96,18 +94,22 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private static final String PARTITIONED_TABLE_PATH = ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; private static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1"; - private static final String PARTITION_PATH = - PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; - private static final String PARTITIONED_TABLE_OBJECT_KEY = - DATABASE_NAME_VALUE + "/" + PARTITIONED_TABLE_NAME + "/id1"; - - private static final String PARTITIONED_OBJECT_KEY = - DATABASE_NAME_VALUE + "/some_location/id1/" + PARTITION_NAME + "/file1"; - - private static final String UNPARTITIONED_TABLE_PATH = - ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; - private static final String UNPARTITIONED_TABLE_OBJECT_KEY = - DATABASE_NAME_VALUE + "/" + UNPARTITIONED_TABLE_NAME + "/id1"; + private static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; + private static final String PARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE + + "/" + + PARTITIONED_TABLE_NAME + + "/id1"; + + private static final String PARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE + + "/some_location/id1/" + + PARTITION_NAME + + "/file1"; + + private static final String UNPARTITIONED_TABLE_PATH = ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; + private static final String UNPARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE + + "/" + + UNPARTITIONED_TABLE_NAME + + "/id1"; private static final String S3_CLIENT_CLASS_NAME = "S3Client"; private static final String HIVE_CLIENT_CLASS_NAME = "HiveClient"; @@ -119,16 +121,16 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private final ExecutorService executorService = Executors.newFixedThreadPool(1); private final TestAppender appender = new TestAppender(); - private Map metastoreProperties = - ImmutableMap.builder() - .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) - .put(ACCESS_KEY, S3_ACCESS_KEY) - .put(SECRET_KEY, S3_SECRET_KEY) - .build(); + private Map metastoreProperties = ImmutableMap + .builder() + .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) + .put(ACCESS_KEY, S3_ACCESS_KEY) + .put(SECRET_KEY, S3_SECRET_KEY) + .build(); @RegisterExtension - public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = - new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, metastoreProperties); + public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( + DATABASE_NAME_VALUE, metastoreProperties); private HiveTestUtils hiveTestUtils; private HiveMetaStoreClient metastoreClient; @@ -138,8 +140,7 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, "test"); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, "true"); - System.setProperty( - AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); + System.setProperty(AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); System.setProperty(AWS_DISABLE_GET_VALIDATION_PROPERTY, "true"); System.setProperty(AWS_DISABLE_PUT_VALIDATION_PROPERTY, "true"); @@ -185,14 +186,11 @@ void stop() throws InterruptedException { @Test public void dryRunDropUnpartitionedTable() throws TException, SQLException { - hiveTestUtils.createTableWithDeletionProperties( - UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, true); + hiveTestUtils.createTableWithDeletionProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, true); amazonS3.putObject(BUCKET, UNPARTITIONED_TABLE_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> logsContainLine(UNPARTITIONED_TABLE_OBJECT_KEY)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLine(UNPARTITIONED_TABLE_OBJECT_KEY)); assertHiveClientLogs(1); assertS3ClientLogs(1); @@ -202,9 +200,7 @@ public void dryRunDropUnpartitionedTable() throws TException, SQLException { @Test public void dryRunDropPartitionedTable() throws Exception { - Table table = - hiveTestUtils.createTableWithDeletionProperties( - PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); + Table table = hiveTestUtils.createTableWithDeletionProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -222,16 +218,13 @@ public void dryRunDropPartitionedTable() throws Exception { @Test public void dryRunDontDropPartitionedTable() throws Exception { - Table table = - hiveTestUtils.createTableWithDeletionProperties( - PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); + Table table = hiveTestUtils.createTableWithDeletionProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); String partition2Name = "event_date=2020-01-01/event_hour=1/event_type=B"; String partition2Path = PARTITION_ROOT_PATH + "/" + partition2Name + "/file1"; - String partition2ObjectKey = - DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; + String partition2ObjectKey = DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -239,8 +232,7 @@ public void dryRunDontDropPartitionedTable() throws Exception { insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); insertExpiredMetadata(PARTITION_PATH, PARTITION_NAME); - insertExpiredMetadata( - TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); + insertExpiredMetadata(TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLine(PARTITIONED_OBJECT_KEY)); @@ -254,9 +246,7 @@ public void dryRunDontDropPartitionedTable() throws Exception { @Test public void dryRunMetrics() throws Exception { - Table table = - hiveTestUtils.createTableWithDeletionProperties( - PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); + Table table = hiveTestUtils.createTableWithDeletionProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); @@ -265,9 +255,7 @@ public void dryRunMetrics() throws Exception { insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); insertExpiredMetadata(PARTITION_PATH, PARTITION_NAME); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> logsContainLine(PARTITION_PATH.replace("s3a://", ""))); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLine(PARTITION_PATH.replace("s3a://", ""))); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isTrue(); @@ -276,24 +264,17 @@ public void dryRunMetrics() throws Exception { } private void assertMetrics() { - Set meterRegistry = - ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); + Set meterRegistry = ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); meterRegistry.forEach( registry -> { List meters = registry.getMeters(); - assertThat(meters) - .extracting("id", Meter.Id.class) - .extracting("name") - .contains( - "metadata-cleanup-job", - "hive-table-deleted", - "hive-partition-deleted", - "hive-table-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, - "hive-partition-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, - "s3-paths-deleted", - "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); - }); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name") + .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + +DeletedMetadataReporter.DRY_RUN_METRIC_NAME, + "hive-partition-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, "s3-paths-deleted", "s3-" + +BytesDeletedReporter.DRY_RUN_METRIC_NAME); + }); } private boolean logsContainLine(String messageFragment) { @@ -327,4 +308,5 @@ private void assertHiveClientLogs(int expected) { } assertThat(logsFromHiveClient).isEqualTo(expected); } + } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java index e14165ea..b34e0364 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java @@ -66,13 +66,11 @@ public class BeekeeperDryRunPathCleanupIntegrationTest extends BeekeeperIntegrat private static final String OBJECT_KEY_ROOT = DB_AND_TABLE_PREFIX + "/id1/partition1"; private static final String OBJECT_KEY1 = DB_AND_TABLE_PREFIX + "/id1/partition1/file1"; private static final String OBJECT_KEY2 = DB_AND_TABLE_PREFIX + "/id1/partition1/file2"; - private static final String OBJECT_KEY_SENTINEL = - DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; + private static final String OBJECT_KEY_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; private static final String ABSOLUTE_PATH = "s3://" + BUCKET + "/" + OBJECT_KEY_ROOT; private static final String OBJECT_KEY_OTHER = DB_AND_TABLE_PREFIX + "/id1/partition10/file1"; - private static final String OBJECT_KEY_OTHER_SENTINEL = - DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; + private static final String OBJECT_KEY_OTHER_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; private static final String SPRING_PROFILES_ACTIVE = "test"; private static final String SCHEDULER_DELAY_MS = "5000"; @@ -83,7 +81,6 @@ public class BeekeeperDryRunPathCleanupIntegrationTest extends BeekeeperIntegrat @Container private static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); - private static AmazonS3 amazonS3; private final ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -94,8 +91,7 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, SPRING_PROFILES_ACTIVE); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, DRY_RUN_ENABLED); - System.setProperty( - AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); + System.setProperty(AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); amazonS3 = ContainerTestUtils.s3Client(S3_CONTAINER, AWS_REGION); amazonS3.createBucket(new CreateBucketRequest(BUCKET, AWS_REGION)); @@ -113,8 +109,7 @@ public static void teardown() { @BeforeEach public void setup() { - amazonS3 - .listObjectsV2(BUCKET) + amazonS3.listObjectsV2(BUCKET) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {})); @@ -139,9 +134,7 @@ public void filesNotDeletedInDirectory() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isTrue(); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY2)).isTrue(); @@ -159,9 +152,7 @@ public void logsForDirectory() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertS3ClientLogs(3); } @@ -209,9 +200,7 @@ public void logsForNonEmptyParentDeletion() throws SQLException { amazonS3.putObject(BUCKET, tableSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertS3ClientLogs(3); } @@ -234,13 +223,8 @@ public void metrics() throws SQLException { private boolean assertMetrics() { MeterRegistry meterRegistry = BeekeeperPathCleanup.meterRegistry(); List meters = meterRegistry.getMeters(); - assertThat(meters) - .extracting("id", Meter.Id.class) - .extracting("name") - .contains( - "path-cleanup-job", - "s3-paths-deleted", - "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name") + .contains("path-cleanup-job", "s3-paths-deleted", "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); return true; } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index a45d80d5..14faca94 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -80,8 +80,7 @@ import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension; @Testcontainers -public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest - extends BeekeeperIntegrationTestBase { +public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { protected static final int TIMEOUT = 30; protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; @@ -94,8 +93,7 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest protected static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; protected static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; - protected static final String PARTITION_KEYS = - "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; + protected static final String PARTITION_KEYS = "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; protected static final String PARTITION_A_VALUES = "[ \"2020-01-01\", \"0\" ]"; protected static final String PARTITION_B_VALUES = "[ \"2020-01-01\", \"1\" ]"; protected static final String PARTITION_A_NAME = "event_date=2020-01-01/event_hour=0"; @@ -114,7 +112,6 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest @Container protected static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); - @Container protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); @@ -125,16 +122,16 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest protected static AmazonS3 amazonS3; - private Map metastoreProperties = - ImmutableMap.builder() + private static Map metastoreProperties = ImmutableMap + .builder() .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) .put(ACCESS_KEY, S3_ACCESS_KEY) .put(SECRET_KEY, S3_SECRET_KEY) .build(); @RegisterExtension - protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = - new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, metastoreProperties); + protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( + DATABASE_NAME_VALUE, metastoreProperties); @BeforeAll public static void init() { @@ -142,8 +139,7 @@ public static void init() { ACTUAL_QUEUE_URL = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); amazonSQS.createQueue(QUEUE); System.setProperty(APIARY_QUEUE_URL_PROPERTY, ACTUAL_QUEUE_URL); - System.setProperty( - SQS_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(SQS_CONTAINER, SQS)); + System.setProperty(SQS_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(SQS_CONTAINER, SQS)); System.setProperty(SQS_REGION_PROPERTY, AWS_REGION); amazonS3 = ContainerTestUtils.s3Client(S3_CONTAINER, AWS_REGION); @@ -183,8 +179,7 @@ public void stop() throws InterruptedException { } @Test - public void expiredMetadataCreateTableEvent() - throws SQLException, IOException, URISyntaxException { + public void expiredMetadataCreateTableEvent() throws SQLException, IOException, URISyntaxException { CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); @@ -195,8 +190,7 @@ public void expiredMetadataCreateTableEvent() } @Test - public void expiredMetadataAlterTableEvent() - throws SQLException, IOException, URISyntaxException { + public void expiredMetadataAlterTableEvent() throws SQLException, IOException, URISyntaxException { insertExpiredMetadata(LOCATION_A + "-old", null); AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage(LOCATION_A, true); @@ -209,10 +203,9 @@ public void expiredMetadataAlterTableEvent() } @Test - public void expiredMetadataAddPartitionEvent() - throws SQLException, IOException, URISyntaxException { - AddPartitionSqsMessage addPartitionSqsMessage = - new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); + public void expiredMetadataAddPartitionEvent() throws SQLException, IOException, URISyntaxException { + AddPartitionSqsMessage addPartitionSqsMessage = new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, + PARTITION_A_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage.getFormattedString())); // creating entry for table @@ -227,12 +220,11 @@ public void expiredMetadataAddPartitionEvent() } @Test - public void expiredMetadataMultipleAddPartitionEvents() - throws SQLException, IOException, URISyntaxException { - AddPartitionSqsMessage addPartitionSqsMessage = - new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); - AddPartitionSqsMessage addPartitionSqsMessage2 = - new AddPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, PARTITION_B_VALUES, true); + public void expiredMetadataMultipleAddPartitionEvents() throws SQLException, IOException, URISyntaxException { + AddPartitionSqsMessage addPartitionSqsMessage = new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, + PARTITION_A_VALUES, true); + AddPartitionSqsMessage addPartitionSqsMessage2 = new AddPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, + PARTITION_B_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage.getFormattedString())); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage2.getFormattedString())); @@ -249,12 +241,11 @@ public void expiredMetadataMultipleAddPartitionEvents() } @Test - public void expiredMetadataAlterPartitionTableEvent() - throws SQLException, IOException, URISyntaxException { + public void expiredMetadataAlterPartitionTableEvent() throws SQLException, IOException, URISyntaxException { insertExpiredMetadata(LOCATION_A + "-old", PARTITION_A_NAME); - AlterPartitionSqsMessage alterPartitionSqsMessage = - new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); + AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, + PARTITION_A_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUpdatedExpiredMetadataRowCount() == 1); @@ -264,15 +255,14 @@ public void expiredMetadataAlterPartitionTableEvent() } @Test - public void expiredMetadataMultipleAlterPartitionTableEvents() - throws SQLException, IOException, URISyntaxException { + public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLException, IOException, URISyntaxException { insertExpiredMetadata(LOCATION_A + "-old", PARTITION_A_NAME); insertExpiredMetadata(LOCATION_B + "-old", PARTITION_B_NAME); - AlterPartitionSqsMessage alterPartitionSqsMessage = - new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, PARTITION_A_VALUES, true); - AlterPartitionSqsMessage alterPartitionSqsMessage2 = - new AlterPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, PARTITION_B_VALUES, true); + AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, + PARTITION_A_VALUES, true); + AlterPartitionSqsMessage alterPartitionSqsMessage2 = new AlterPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, + PARTITION_B_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage2.getFormattedString())); @@ -284,8 +274,7 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() } @Test - public void expiredMetadataCreateIcebergTableEvent() - throws SQLException, IOException, URISyntaxException { + public void expiredMetadataCreateIcebergTableEvent() throws SQLException, IOException, URISyntaxException { CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true, true); amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); @@ -300,9 +289,7 @@ public void testEventAddedToHistoryTable() throws SQLException, IOException, URI CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); List beekeeperHistory = getBeekeeperHistory(EXPIRED); BeekeeperHistory history = beekeeperHistory.get(0); @@ -321,7 +308,8 @@ void scheduleExistingPartitionsWhenPropertiesExpireInTable() throws Exception { AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage(LOCATION_A, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 3); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 3); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); List expiredMetadata = getExpiredMetadata(); @@ -335,14 +323,13 @@ void scheduleMissingPartitionsWhenPropertiesExpireInTable() throws Exception { hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); insertExpiredMetadata(PARTITION_ROOT_PATH, null); - insertExpiredMetadata( - PARTITION_ROOT_PATH + PARTITION_A_NAME + "/event_type=C", - PARTITION_A_NAME + "/event_type=C"); + insertExpiredMetadata(PARTITION_ROOT_PATH + PARTITION_A_NAME + "/event_type=C",PARTITION_A_NAME + "/event_type=C"); AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage(PARTITION_ROOT_PATH, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 4); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getExpiredMetadataRowCount() == 4); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); List expiredMetadata = getExpiredMetadata(); @@ -353,9 +340,7 @@ void scheduleMissingPartitionsWhenPropertiesExpireInTable() throws Exception { public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(HEALTHCHECK_URI); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } @Test @@ -371,25 +356,23 @@ protected SendMessageRequest sendMessageRequest(String payload) { return new SendMessageRequest(ACTUAL_QUEUE_URL, payload); } - protected void assertExpiredMetadata( - HousekeepingMetadata actual, String expectedPath, String partitionName) { + protected void assertExpiredMetadata(HousekeepingMetadata actual, String expectedPath, String partitionName) { assertHousekeepingMetadata(actual, expectedPath, partitionName); assertMetrics(); } public void assertHousekeepingMetadata( - HousekeepingMetadata actual, String expectedPath, String expectedPartitionName) { + HousekeepingMetadata actual, + String expectedPath, + String expectedPartitionName) { assertThat(actual.getPath()).isEqualTo(expectedPath); assertThat(actual.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); assertThat(actual.getTableName()).isEqualTo(TABLE_NAME_VALUE); assertThat(actual.getPartitionName()).isEqualTo(expectedPartitionName); assertThat(actual.getHousekeepingStatus()).isEqualTo(SCHEDULED); - assertThat(actual.getCreationTimestamp()) - .isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); - assertThat(actual.getModifiedTimestamp()) - .isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); - assertThat(actual.getCleanupTimestamp()) - .isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); + assertThat(actual.getCreationTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); + assertThat(actual.getModifiedTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE.withNano(0)); + assertThat(actual.getCleanupTimestamp()).isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); assertThat(actual.getCleanupDelay()).isEqualTo(PeriodDuration.parse(SHORT_CLEANUP_DELAY_VALUE)); assertThat(actual.getCleanupAttempts()).isEqualTo(CLEANUP_ATTEMPTS_VALUE); assertThat(actual.getClientId()).isEqualTo(CLIENT_ID_VALUE); @@ -397,16 +380,12 @@ public void assertHousekeepingMetadata( } public void assertMetrics() { - Set meterRegistry = - ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()).getRegistries(); + Set meterRegistry = ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()) + .getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach( - registry -> { + meterRegistry.forEach(registry -> { List meters = registry.getMeters(); - assertThat(meters) - .extracting("id", Meter.Id.class) - .extracting("name") - .contains(SCHEDULED_EXPIRED_METRIC); - }); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name").contains(SCHEDULED_EXPIRED_METRIC); + }); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 105eb30f..de63bbad 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2025 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -92,64 +94,30 @@ public abstract class BeekeeperIntegrationTestBase { // FIELDS TO INSERT INTO BEEKEEPER TABLES private Long id = 1L; - private static final String HOUSEKEEPING_PATH_FIELDS = - String.join( - ",", - ID_FIELD, - PATH_FIELD, - DATABASE_NAME_FIELD, - TABLE_NAME_FIELD, - HOUSEKEEPING_STATUS_FIELD, - CREATION_TIMESTAMP_FIELD, - MODIFIED_TIMESTAMP_FIELD, - CLEANUP_TIMESTAMP_FIELD, - CLEANUP_DELAY_FIELD, - CLEANUP_ATTEMPTS_FIELD, - CLIENT_ID_FIELD, - LIFECYCLE_TYPE_FIELD); - private static final String HOUSEKEEPING_METADATA_FIELDS = - String.join( - ",", - ID_FIELD, - PATH_FIELD, - DATABASE_NAME_FIELD, - TABLE_NAME_FIELD, - PARTITION_NAME_FIELD, - HOUSEKEEPING_STATUS_FIELD, - CREATION_TIMESTAMP_FIELD, - MODIFIED_TIMESTAMP_FIELD, - CLEANUP_TIMESTAMP_FIELD, - CLEANUP_DELAY_FIELD, - CLEANUP_ATTEMPTS_FIELD, - CLIENT_ID_FIELD, - LIFECYCLE_TYPE_FIELD); - private static final String BEEKEEPER_HISTORY_FIELDS = - String.join( - ",", - ID_FIELD, - EVENT_TIMESTAMP_FIELD, - DATABASE_NAME_FIELD, - TABLE_NAME_FIELD, - LIFECYCLE_TYPE_FIELD, - HOUSEKEEPING_STATUS_FIELD, - EVENT_DETAILS_FIELD); - private static final String LIFE_CYCLE_FILTER = - "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; - private static final String LIFE_CYCLE_AND_UPDATE_FILTER = - "WHERE " - + LIFECYCLE_TYPE_FIELD - + " = '%s'" - + " AND " - + MODIFIED_TIMESTAMP_FIELD - + " > " - + CREATION_TIMESTAMP_FIELD - + " ORDER BY " - + PATH_FIELD; + private static final String HOUSEKEEPING_PATH_FIELDS = String + .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, + CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, + CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); + private static final String HOUSEKEEPING_METADATA_FIELDS = String + .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, PARTITION_NAME_FIELD, + HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, + CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); + private static final String BEEKEEPER_HISTORY_FIELDS = String.join(",", ID_FIELD, EVENT_TIMESTAMP_FIELD, + DATABASE_NAME_FIELD, TABLE_NAME_FIELD, LIFECYCLE_TYPE_FIELD, HOUSEKEEPING_STATUS_FIELD, EVENT_DETAILS_FIELD); + private static final String LIFE_CYCLE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; + private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE " + + LIFECYCLE_TYPE_FIELD + + " = '%s'" + + " AND " + + MODIFIED_TIMESTAMP_FIELD + + " > " + + CREATION_TIMESTAMP_FIELD + + " ORDER BY " + + PATH_FIELD; // MySQL DB CONTAINER AND UTILS @Container private static final MySQLContainer MY_SQL_CONTAINER = ContainerTestUtils.mySqlContainer(); - protected static MySqlTestUtils mySQLTestUtils; protected final ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -196,105 +164,75 @@ protected void insertUnreferencedPath(String path) throws SQLException { } protected void insertUnreferencedPath(HousekeepingPath housekeepingPath) throws SQLException { - housekeepingPath.setCleanupTimestamp( - housekeepingPath.getCleanupTimestamp().minus(Duration.ofDays(1))); - String values = - Stream.of( - housekeepingPath.getId().toString(), - housekeepingPath.getPath(), - housekeepingPath.getDatabaseName(), - housekeepingPath.getTableName(), - housekeepingPath.getHousekeepingStatus().toString(), - housekeepingPath.getCreationTimestamp().toString(), - housekeepingPath.getModifiedTimestamp().toString(), - housekeepingPath.getCleanupTimestamp().toString(), - housekeepingPath.getCleanupDelay().toString(), - String.valueOf(housekeepingPath.getCleanupAttempts()), - housekeepingPath.getClientId(), - housekeepingPath.getLifecycleType()) - .map(s -> s == null ? null : "\"" + s + "\"") - .collect(Collectors.joining(", ")); - - mySQLTestUtils.insertToTable( - BEEKEEPER_DB_NAME, - BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, - HOUSEKEEPING_PATH_FIELDS, - values); + housekeepingPath.setCleanupTimestamp(housekeepingPath.getCleanupTimestamp().minus(Duration.ofDays(1))); + String values = Stream + .of(housekeepingPath.getId().toString(), housekeepingPath.getPath(), housekeepingPath.getDatabaseName(), + housekeepingPath.getTableName(), housekeepingPath.getHousekeepingStatus().toString(), + housekeepingPath.getCreationTimestamp().toString(), housekeepingPath.getModifiedTimestamp().toString(), + housekeepingPath.getCleanupTimestamp().toString(), housekeepingPath.getCleanupDelay().toString(), + String.valueOf(housekeepingPath.getCleanupAttempts()), housekeepingPath.getClientId(), + housekeepingPath.getLifecycleType()) + .map(s -> s == null ? null : "\"" + s + "\"") + .collect(Collectors.joining(", ")); + + mySQLTestUtils + .insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, HOUSEKEEPING_PATH_FIELDS, values); } protected void insertExpiredMetadata(String path, String partitionName) throws SQLException { insertExpiredMetadata(TABLE_NAME_VALUE, path, partitionName, SHORT_CLEANUP_DELAY_VALUE); } - protected void insertExpiredMetadata( - String tableName, String path, String partitionName, String cleanupDelay) + protected void insertExpiredMetadata(String tableName, String path, String partitionName, String cleanupDelay) throws SQLException { - HousekeepingMetadata metadata = - createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); + HousekeepingMetadata metadata = createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); insertExpiredMetadata(metadata); } protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLException { - String values = - Stream.of( - metadata.getId().toString(), - metadata.getPath(), - metadata.getDatabaseName(), - metadata.getTableName(), - metadata.getPartitionName(), - metadata.getHousekeepingStatus().toString(), - metadata.getCreationTimestamp().toString(), - metadata.getModifiedTimestamp().toString(), - metadata.getCleanupTimestamp().toString(), - metadata.getCleanupDelay().toString(), - String.valueOf(metadata.getCleanupAttempts()), - metadata.getClientId(), - metadata.getLifecycleType()) - .map(s -> s == null ? null : "\"" + s + "\"") - .collect(Collectors.joining(", ")); - - mySQLTestUtils.insertToTable( - BEEKEEPER_DB_NAME, - BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, - HOUSEKEEPING_METADATA_FIELDS, - values); + String values = Stream + .of(metadata.getId().toString(), metadata.getPath(), metadata.getDatabaseName(), metadata.getTableName(), + metadata.getPartitionName(), metadata.getHousekeepingStatus().toString(), + metadata.getCreationTimestamp().toString(), metadata.getModifiedTimestamp().toString(), + metadata.getCleanupTimestamp().toString(), metadata.getCleanupDelay().toString(), + String.valueOf(metadata.getCleanupAttempts()), metadata.getClientId(), metadata.getLifecycleType()) + .map(s -> s == null ? null : "\"" + s + "\"") + .collect(Collectors.joining(", ")); + + mySQLTestUtils + .insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, HOUSEKEEPING_METADATA_FIELDS, + values); } protected int getUnreferencedPathsRowCount() throws SQLException { - return mySQLTestUtils.getTableRowCount( - BEEKEEPER_DB_NAME, - BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, - format(LIFE_CYCLE_FILTER, UNREFERENCED)); + return mySQLTestUtils + .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, + format(LIFE_CYCLE_FILTER, UNREFERENCED)); } protected int getExpiredMetadataRowCount() throws SQLException { - return mySQLTestUtils.getTableRowCount( - BEEKEEPER_DB_NAME, - BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, - format(LIFE_CYCLE_FILTER, EXPIRED)); + return mySQLTestUtils + .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, + format(LIFE_CYCLE_FILTER, EXPIRED)); } protected int getUpdatedExpiredMetadataRowCount() throws SQLException { - return mySQLTestUtils.getTableRowCount( - BEEKEEPER_DB_NAME, - BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, - format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED)); + return mySQLTestUtils + .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, + format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED)); } - protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) - throws SQLException { + protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException { String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; - return mySQLTestUtils.getTableRowCount( - BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, format(filter, lifecycleEventType)); + return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); } protected List getUnreferencedPaths() throws SQLException { List paths = new ArrayList<>(); - ResultSet resultSet = - mySQLTestUtils.getTableRows( - BEEKEEPER_DB_NAME, - BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, + ResultSet resultSet = mySQLTestUtils + .getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, format(LIFE_CYCLE_FILTER, UNREFERENCED)); while (resultSet.next()) { @@ -306,10 +244,8 @@ protected List getUnreferencedPaths() throws SQLException { protected List getExpiredMetadata() throws SQLException { List metadata = new ArrayList<>(); - ResultSet resultSet = - mySQLTestUtils.getTableRows( - BEEKEEPER_DB_NAME, - BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, + ResultSet resultSet = mySQLTestUtils + .getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME, format(LIFE_CYCLE_FILTER, EXPIRED)); while (resultSet.next()) { @@ -319,13 +255,11 @@ protected List getExpiredMetadata() throws SQLException { return metadata; } - protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) - throws SQLException { + protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException { String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; List history = new ArrayList<>(); - ResultSet resultSet = - mySQLTestUtils.getTableRows( - BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, format(filter, lifecycleEventType)); + ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); while (resultSet.next()) { history.add(mapToBeekeeperHistory(resultSet)); @@ -334,9 +268,9 @@ protected List getBeekeeperHistory(LifecycleEventType lifecycl return history; } - private HousekeepingPath createHousekeepingPath( - String path, LifecycleEventType lifecycleEventType) { - return HousekeepingPath.builder() + private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType lifecycleEventType) { + return HousekeepingPath + .builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -357,7 +291,8 @@ private HousekeepingMetadata createHousekeepingMetadata( String partitionName, LifecycleEventType lifecycleEventType, String cleanupDelay) { - return HousekeepingMetadata.builder() + return HousekeepingMetadata + .builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java index ff2084d3..acce7868 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2025 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration; @@ -45,8 +47,7 @@ private CommonTestVariables() {} // HOUSEKEEPINGENTITY DEFAULT VALUES public static final String DATABASE_NAME_VALUE = "some_database"; public static final String TABLE_NAME_VALUE = "some_table"; - public static final LocalDateTime CREATION_TIMESTAMP_VALUE = - LocalDateTime.now(UTC).minus(1L, ChronoUnit.MINUTES); + public static final LocalDateTime CREATION_TIMESTAMP_VALUE = LocalDateTime.now(UTC).minus(1L, ChronoUnit.MINUTES); public static final String SHORT_CLEANUP_DELAY_VALUE = "PT1S"; public static final String LONG_CLEANUP_DELAY_VALUE = "P3D"; public static final int CLEANUP_ATTEMPTS_VALUE = 0; diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java index 2ff4deed..acdf5876 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AddPartitionSqsMessage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2020 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -23,8 +25,11 @@ public class AddPartitionSqsMessage extends SqsMessage { public AddPartitionSqsMessage( - String partitionLocation, String partitionKeys, String partitionValues, boolean isExpired) - throws IOException, URISyntaxException { + String partitionLocation, + String partitionKeys, + String partitionValues, + boolean isExpired + ) throws IOException, URISyntaxException { super(ADD_PARTITION); setTableLocation(DUMMY_LOCATION); setPartitionLocation(partitionLocation); @@ -34,17 +39,14 @@ public AddPartitionSqsMessage( } private void setPartitionLocation(String partitionLocation) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); } private void setPartitionKeys(String partitionKeys) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); } private void setPartitionValues(String partitionValues) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java index 218a6067..86c73929 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterPartitionSqsMessage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2020 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -27,8 +29,8 @@ public AlterPartitionSqsMessage( String partitionLocation, String oldPartitionLocation, boolean isUnreferenced, - boolean isWhitelisted) - throws IOException, URISyntaxException { + boolean isWhitelisted + ) throws IOException, URISyntaxException { super(ALTER_PARTITION); setTableLocation(tableLocation); setPartitionLocation(partitionLocation); @@ -41,8 +43,11 @@ public AlterPartitionSqsMessage( } public AlterPartitionSqsMessage( - String partitionLocation, String partitionKeys, String partitionValues, boolean isExpired) - throws IOException, URISyntaxException { + String partitionLocation, + String partitionKeys, + String partitionValues, + boolean isExpired + ) throws IOException, URISyntaxException { super(ALTER_PARTITION); setTableLocation(DUMMY_LOCATION); setPartitionLocation(partitionLocation); @@ -54,27 +59,22 @@ public AlterPartitionSqsMessage( } public void setPartitionLocation(String partitionLocation) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); } public void setOldPartitionLocation(String oldPartitionLocation) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_OLD_PARTITION_LOCATION_KEY, new JsonPrimitive(oldPartitionLocation)); + apiaryEventMessageJsonObject.add(EVENT_TABLE_OLD_PARTITION_LOCATION_KEY, new JsonPrimitive(oldPartitionLocation)); } public void setPartitionKeys(String partitionKeys) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); } public void setPartitionValues(String partitionValues) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); } public void setOldPartitionValues(String oldPartitionValues) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_OLD_PARTITION_VALUES_KEY, PARSER.parse(oldPartitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add(EVENT_TABLE_OLD_PARTITION_VALUES_KEY, PARSER.parse(oldPartitionValues).getAsJsonArray()); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java index ac21492f..9fbff756 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/AlterTableSqsMessage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2020 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -25,8 +27,11 @@ public class AlterTableSqsMessage extends SqsMessage { public AlterTableSqsMessage( - String tableLocation, String oldTableLocation, boolean isUnreferenced, boolean isWhitelisted) - throws IOException, URISyntaxException { + String tableLocation, + String oldTableLocation, + boolean isUnreferenced, + boolean isWhitelisted + ) throws IOException, URISyntaxException { super(ALTER_TABLE); setTableLocation(tableLocation); setOldTableLocation(oldTableLocation); @@ -35,8 +40,10 @@ public AlterTableSqsMessage( setWhitelisted(isWhitelisted); } - public AlterTableSqsMessage(String tableLocation, boolean isExpired) - throws IOException, URISyntaxException { + public AlterTableSqsMessage( + String tableLocation, + boolean isExpired + ) throws IOException, URISyntaxException { super(ALTER_TABLE); setTableLocation(tableLocation); setOldTableLocation(DUMMY_LOCATION); @@ -45,8 +52,7 @@ public AlterTableSqsMessage(String tableLocation, boolean isExpired) } public void setOldTableLocation(String oldTableLocation) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_OLD_LOCATION_KEY, new JsonPrimitive(oldTableLocation)); + apiaryEventMessageJsonObject.add(EVENT_TABLE_OLD_LOCATION_KEY, new JsonPrimitive(oldTableLocation)); } public void setOldTableName(String oldTableName) { diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java index 3c39331f..5c70e846 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/CreateTableSqsMessage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2025 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -20,15 +22,20 @@ public class CreateTableSqsMessage extends SqsMessage { - public CreateTableSqsMessage(String tableLocation, boolean isExpired) - throws IOException, URISyntaxException { + public CreateTableSqsMessage( + String tableLocation, + boolean isExpired + ) throws IOException, URISyntaxException { super(CREATE_TABLE); setTableLocation(tableLocation); setExpired(isExpired); } - public CreateTableSqsMessage(String tableLocation, boolean isIceberg, boolean isExpired) - throws IOException, URISyntaxException { + public CreateTableSqsMessage( + String tableLocation, + boolean isIceberg, + boolean isExpired + ) throws IOException, URISyntaxException { super(CREATE_TABLE); setTableLocation(tableLocation); setExpired(isExpired); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java index 47a55096..536999ce 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropPartitionSqsMessage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2020 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -23,8 +25,10 @@ public class DropPartitionSqsMessage extends SqsMessage { public DropPartitionSqsMessage( - String partitionLocation, boolean isUnreferenced, boolean isWhitelisted) - throws IOException, URISyntaxException { + String partitionLocation, + boolean isUnreferenced, + boolean isWhitelisted + ) throws IOException, URISyntaxException { super(DROP_PARTITION); setTableLocation(DUMMY_LOCATION); setPartitionLocation(partitionLocation); @@ -35,17 +39,14 @@ public DropPartitionSqsMessage( } public void setPartitionLocation(String partitionLocation) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_LOCATION_KEY, new JsonPrimitive(partitionLocation)); } public void setPartitionKeys(String partitionKeys) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_KEYS_KEY, PARSER.parse(partitionKeys).getAsJsonObject()); } public void setPartitionValues(String partitionValues) { - apiaryEventMessageJsonObject.add( - EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); + apiaryEventMessageJsonObject.add(EVENT_TABLE_PARTITION_VALUES_KEY, PARSER.parse(partitionValues).getAsJsonArray()); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java index edf859c2..f5d7d47b 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/DropTableSqsMessage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2020 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -20,8 +22,11 @@ public class DropTableSqsMessage extends SqsMessage { - public DropTableSqsMessage(String tableLocation, boolean isUnreferenced, boolean isWhitelisted) - throws IOException, URISyntaxException { + public DropTableSqsMessage( + String tableLocation, + boolean isUnreferenced, + boolean isWhitelisted + ) throws IOException, URISyntaxException { super(DROP_TABLE); setTableLocation(tableLocation); setUnreferenced(isUnreferenced); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java index 547a6acf..dda2de2c 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2025 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -56,8 +58,7 @@ public abstract class SqsMessage { public static final String EVENT_TABLE_OLD_PARTITION_VALUES_KEY = "oldPartitionValues"; public static final String DUMMY_LOCATION = "s3://dummy-location"; - public static final String DUMMY_PARTITION_KEYS = - "{ \"col_1\": \"string\", \"col_2\": \"integer\", \"col_3\": \"string\" }"; + public static final String DUMMY_PARTITION_KEYS = "{ \"col_1\": \"string\", \"col_2\": \"integer\", \"col_3\": \"string\" }"; public static final String DUMMY_PARTITION_VALUES = "[ \"val_1\", \"val_2\", \"val_3\" ]"; public static final String EVENT_PROTOCOL_VERSION_VALUE = "1.0"; @@ -67,11 +68,9 @@ public abstract class SqsMessage { public SqsMessage(EventType eventType) throws URISyntaxException, IOException { this.eventType = eventType; - apiaryEventJsonObject = - PARSER.parse(readString(Path.of(APIARY_EVENT_JSON_URL.toURI()))).getAsJsonObject(); + apiaryEventJsonObject = PARSER.parse(readString(Path.of(APIARY_EVENT_JSON_URL.toURI()))).getAsJsonObject(); apiaryEventMessageJsonObject = new JsonObject(); - apiaryEventMessageJsonObject.add( - EVENT_PROTOCOL_VERSION_KEY, new JsonPrimitive(EVENT_PROTOCOL_VERSION_VALUE)); + apiaryEventMessageJsonObject.add(EVENT_PROTOCOL_VERSION_KEY, new JsonPrimitive(EVENT_PROTOCOL_VERSION_VALUE)); apiaryEventMessageJsonObject.add(EVENT_TYPE_KEY, new JsonPrimitive(eventType.toString())); apiaryEventMessageJsonObject.add(EVENT_DB_KEY, new JsonPrimitive(DATABASE_NAME_VALUE)); apiaryEventMessageJsonObject.add(EVENT_TABLE_NAME_KEY, new JsonPrimitive(TABLE_NAME_VALUE)); @@ -83,41 +82,31 @@ public void setTableLocation(String location) { } public void setUnreferenced(boolean isUnreferenced) { - JsonObject tableParameters = - apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); - tableParameters.add( - UNREFERENCED.getTableParameterName(), new JsonPrimitive(String.valueOf(isUnreferenced))); - tableParameters.add( - UNREFERENCED_DATA_RETENTION_PERIOD_PROPERTY_KEY, - new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); + JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + tableParameters.add(UNREFERENCED.getTableParameterName(), new JsonPrimitive(String.valueOf(isUnreferenced))); + tableParameters.add(UNREFERENCED_DATA_RETENTION_PERIOD_PROPERTY_KEY, new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); } public void setExpired(boolean isExpired) { - JsonObject tableParameters = - apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); - tableParameters.add( - EXPIRED.getTableParameterName(), new JsonPrimitive(String.valueOf(isExpired))); - tableParameters.add( - EXPIRED_DATA_RETENTION_PERIOD_PROPERTY_KEY, new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); + JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + tableParameters.add(EXPIRED.getTableParameterName(), new JsonPrimitive(String.valueOf(isExpired))); + tableParameters.add(EXPIRED_DATA_RETENTION_PERIOD_PROPERTY_KEY, new JsonPrimitive(SHORT_CLEANUP_DELAY_VALUE)); } public void setIceberg() { - JsonObject tableParameters = - apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); tableParameters.add("table_format", new JsonPrimitive("ICEBERG")); tableParameters.add("metadata_location", new JsonPrimitive("s3://bucket/metadata")); } public void setWhitelisted(boolean isWhitelisted) { String whitelist = isWhitelisted ? eventType.toString() : ""; - JsonObject tableParameters = - apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); + JsonObject tableParameters = apiaryEventMessageJsonObject.getAsJsonObject(EVENT_TABLE_PARAMETERS_KEY); tableParameters.add(BEEKEEPER_HIVE_EVENT_WHITELIST, new JsonPrimitive(whitelist)); } public final String getFormattedString() { - apiaryEventJsonObject.add( - APIARY_EVENT_MESSAGE_KEY, new JsonPrimitive(apiaryEventMessageJsonObject.toString())); + apiaryEventJsonObject.add(APIARY_EVENT_MESSAGE_KEY, new JsonPrimitive(apiaryEventMessageJsonObject.toString())); return apiaryEventJsonObject.toString(); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java index 6c97e20f..2b2f0738 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/model/SqsMessageTest.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2020 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.model; @@ -32,14 +34,8 @@ public class SqsMessageTest { - private static final Set COMMON_KEYS = - Set.of( - "protocolVersion", - "eventType", - "tableParameters", - "dbName", - "tableName", - "tableLocation"); + private static final Set COMMON_KEYS = Set.of("protocolVersion", "eventType", "tableParameters", + "dbName", "tableName", "tableLocation"); @Test public void testCreateTableFormat() throws IOException, URISyntaxException { @@ -50,40 +46,45 @@ public void testCreateTableFormat() throws IOException, URISyntaxException { @Test public void testAddPartitionFormat() throws IOException, URISyntaxException { - Set specificKeys = - Set.of("partitionKeys", "partitionValues", "partitionLocation", "tableParameters"); - AddPartitionSqsMessage message = - new AddPartitionSqsMessage( - DUMMY_LOCATION, DUMMY_PARTITION_KEYS, DUMMY_PARTITION_VALUES, true); + Set specificKeys = Set.of( + "partitionKeys", + "partitionValues", + "partitionLocation", + "tableParameters"); + AddPartitionSqsMessage message = new AddPartitionSqsMessage(DUMMY_LOCATION, DUMMY_PARTITION_KEYS, + DUMMY_PARTITION_VALUES, true); assertKeys(message, specificKeys, "ADD_PARTITION"); } @Test public void testAlterPartitionFormat() throws IOException, URISyntaxException { - Set specificKeys = - Set.of( - "partitionKeys", - "partitionValues", - "partitionLocation", - "oldPartitionValues", - "oldPartitionLocation"); - AlterPartitionSqsMessage message = - new AlterPartitionSqsMessage( - DUMMY_LOCATION, DUMMY_PARTITION_KEYS, DUMMY_PARTITION_VALUES, true); + Set specificKeys = Set.of( + "partitionKeys", + "partitionValues", + "partitionLocation", + "oldPartitionValues", + "oldPartitionLocation"); + AlterPartitionSqsMessage message = new AlterPartitionSqsMessage(DUMMY_LOCATION, DUMMY_PARTITION_KEYS, + DUMMY_PARTITION_VALUES, true); assertKeys(message, specificKeys, "ALTER_PARTITION"); } @Test public void testAlterTableFormat() throws IOException, URISyntaxException { - Set specificKeys = Set.of("oldTableName", "oldTableLocation"); + Set specificKeys = Set.of( + "oldTableName", + "oldTableLocation"); AlterTableSqsMessage message = new AlterTableSqsMessage(DUMMY_LOCATION, true); assertKeys(message, specificKeys, "ALTER_TABLE"); } @Test public void testDropPartitionFormat() throws IOException, URISyntaxException { - Set specificKeys = - Set.of("partitionKeys", "partitionValues", "partitionLocation", "tableParameters"); + Set specificKeys = Set.of( + "partitionKeys", + "partitionValues", + "partitionLocation", + "tableParameters"); DropPartitionSqsMessage message = new DropPartitionSqsMessage(DUMMY_LOCATION, true, true); assertKeys(message, specificKeys, "DROP_PARTITION"); } @@ -96,13 +97,10 @@ public void testDropTableFormat() throws IOException, URISyntaxException { } private void assertKeys(SqsMessage sqsMessage, Set specificKeys, String eventType) { - SortedSet mergedSet = - new TreeSet<>() { - { - addAll(specificKeys); - addAll(COMMON_KEYS); - } - }; + SortedSet mergedSet = new TreeSet<>() {{ + addAll(specificKeys); + addAll(COMMON_KEYS); + }}; JsonObject object = sqsMessage.getApiaryEventMessageJsonObject(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java index ccbbcaaf..b22cf8e5 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/BeekeeperApiTestClient.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -36,53 +38,37 @@ public BeekeeperApiTestClient(String baseUrl) { this.httpClient = HttpClient.newHttpClient(); } - public HttpResponse getMetadata(String database, String table) - throws IOException, InterruptedException { - HttpRequest request = - newBuilder() - .uri( - URI.create( - String.format( - housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH, - database, - table))) - .GET() - .build(); + public HttpResponse getMetadata(String database, String table) throws IOException, InterruptedException { + HttpRequest request = newBuilder() + .uri( + URI.create(String.format(housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH, database, table))) + .GET() + .build(); return httpClient.send(request, BodyHandlers.ofString()); } public HttpResponse getMetadata(String database, String table, String filters) - throws IOException, InterruptedException { - HttpRequest request = - newBuilder() - .uri( - URI.create( - String.format( - housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH + "%s", - database, - table, - filters))) - .GET() - .build(); + throws IOException, InterruptedException { + HttpRequest request = newBuilder() + .uri(URI + .create(String + .format(housekeepingEntityUrl + "/database/%s/table/%s" + METADATA_PATH + "%s", database, table, + filters))) + .GET() + .build(); return httpClient.send(request, BodyHandlers.ofString()); } public HttpResponse getUnreferencedPaths(String database, String table, String filters) throws IOException, InterruptedException { - HttpRequest request = - newBuilder() - .uri( - URI.create( - String.format( - housekeepingEntityUrl - + "/database/%s/table/%s" - + UNREFERENCED_PATHS_PATH - + "%s", - database, - table, - filters))) - .GET() - .build(); + HttpRequest request = newBuilder() + .uri(URI + .create(String + .format(housekeepingEntityUrl + "/database/%s/table/%s" + UNREFERENCED_PATHS_PATH + "%s", database, table, + filters))) + .GET() + .build(); return httpClient.send(request, BodyHandlers.ofString()); } + } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java index a298d14c..d64e67ae 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/HiveTestUtils.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2025 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -43,24 +45,19 @@ public HiveTestUtils(HiveMetaStoreClient metastoreClient) { this.metastoreClient = metastoreClient; } - private final List DATA_COLUMNS = - Arrays.asList( - new FieldSchema("id", "bigint", ""), - new FieldSchema("name", "string", ""), + private final List DATA_COLUMNS = Arrays + .asList(new FieldSchema("id", "bigint", ""), new FieldSchema("name", "string", ""), new FieldSchema("city", "tinyint", "")); - private final List PARTITION_COLUMNS = - Arrays.asList( - new FieldSchema("event_date", "string", ""), - new FieldSchema("event_hour", "string", ""), + private final List PARTITION_COLUMNS = Arrays + .asList(new FieldSchema("event_date", "string", ""), new FieldSchema("event_hour", "string", ""), new FieldSchema("event_type", "string", "")); public Table createTable(String path, String tableName, boolean partitioned) throws TException { return createTable(path, tableName, partitioned, true); } - public Table createTable( - String path, String tableName, boolean partitioned, boolean withBeekeeperProperty) + public Table createTable(String path, String tableName, boolean partitioned, boolean withBeekeeperProperty) throws TException { Table hiveTable = new Table(); hiveTable.setDbName(DATABASE_NAME_VALUE); @@ -91,22 +88,19 @@ public Table createTable( * @param path Path of the table * @param hiveTable Table to add partitions to * @param partitionValues The list of partition values, e.g. ["2020-01-01", "0", "A"] - * @throws Exception May be thrown if there is a problem when trying to write the data to the - * file, or when the client adds the partition to the table. + * @throws Exception May be thrown if there is a problem when trying to write the data to the file, or when the client + * adds the partition to the table. */ - public void addPartitionsToTable(String path, Table hiveTable, List partitionValues) - throws Exception { + public void addPartitionsToTable(String path, Table hiveTable, List partitionValues) throws Exception { String eventDate = "/event_date=" + partitionValues.get(0); // 2020-01-01 String eventHour = eventDate + "/event_hour=" + partitionValues.get(1); // 0 String eventType = eventHour + "/event_type=" + partitionValues.get(2); // A URI partitionUri = URI.create(path + eventType); - metastoreClient.add_partitions( - Collections.singletonList( - newTablePartition( - hiveTable, - List.of(partitionValues.get(0), partitionValues.get(1), partitionValues.get(2)), - partitionUri))); + metastoreClient + .add_partitions(Collections + .singletonList(newTablePartition(hiveTable, + List.of(partitionValues.get(0), partitionValues.get(1), partitionValues.get(2)), partitionUri))); } private Partition newTablePartition(Table hiveTable, List values, URI location) { @@ -119,12 +113,8 @@ private Partition newTablePartition(Table hiveTable, List values, URI lo return partition; } - public Table createTableWithProperties( - String path, - String tableName, - boolean partitioned, - Map tableProperties, - boolean withBeekeeperProperty) + public Table createTableWithProperties(String path, String tableName, boolean partitioned, + Map tableProperties, boolean withBeekeeperProperty) throws TException { Table hiveTable = new Table(); hiveTable.setDbName(DATABASE_NAME_VALUE); @@ -154,9 +144,8 @@ public Table createTableWithProperties( return hiveTable; } - public Table createTableWithDeletionProperties( - String path, String tableName, boolean partitioned, boolean shouldTableBeDeletedIfEmpty) - throws TException { + public Table createTableWithDeletionProperties(String path, String tableName, boolean partitioned, + boolean shouldTableBeDeletedIfEmpty) throws TException { Map tableProperties = new HashMap<>(); if (shouldTableBeDeletedIfEmpty) { tableProperties.put("beekeeper.expired.data.table.deletion.enabled", "true"); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java index aa2bc940..17af29a5 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/MySqlTestUtils.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -37,28 +39,23 @@ public void close() throws SQLException { connection.close(); } - public void insertToTable(String database, String table, String fields, String values) - throws SQLException { - connection - .createStatement() - .executeUpdate(format(INSERT_TO_TABLE, database, table, fields, values)); + public void insertToTable(String database, String table, String fields, String values) throws SQLException { + connection.createStatement().executeUpdate(format(INSERT_TO_TABLE, database, table, fields, values)); } - public int getTableRowCount(String database, String table, String additionalFilters) - throws SQLException { + public int getTableRowCount(String database, String table, String additionalFilters) throws SQLException { return getTableRowCount(format(SELECT_TABLE, database, table, additionalFilters)); } private int getTableRowCount(String statementString) throws SQLException { - Statement statement = - connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY); + Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, + ResultSet.CONCUR_READ_ONLY); ResultSet resultSet = statement.executeQuery(statementString); resultSet.last(); return resultSet.getRow(); } - public ResultSet getTableRows(String database, String table, String additionalFilters) - throws SQLException { + public ResultSet getTableRows(String database, String table, String additionalFilters) throws SQLException { return getTableRows(format(SELECT_TABLE, database, table, additionalFilters)); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java index 4b6e9ce2..aff31b49 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/RestResponsePage.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2021 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -37,12 +39,11 @@ public RestResponsePage( @JsonProperty("first") boolean first, @JsonProperty("numberOfElements") int numberOfElements, @JsonProperty("empty") boolean empty) { - // If the page size is 0, we override the value to 1. This is because the jackson dependency - // does not allow to - // create empty pages, which makes it hard to test what happens when the table is not found and - // an empty page is + // If the page size is 0, we override the value to 1. This is because the jackson dependency does not allow to + // create empty pages, which makes it hard to test what happens when the table is not found and an empty page is // returned. // By overriding that value to 1, it is possible to create the 'empty' page and do the testing. super(content, PageRequest.of(number, size == 0 ? 1 : size), totalElements); } + } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java index a7c0307a..d254e755 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2025 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java index 43c1515d..5eb2b2a0 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToHousekeepingEntityMapper.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2023 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; @@ -38,17 +40,15 @@ public class ResultSetToHousekeepingEntityMapper { public static HousekeepingPath mapToHousekeepingPath(ResultSet resultSet) throws SQLException { - return HousekeepingPath.builder() + return HousekeepingPath + .builder() .id(resultSet.getLong(ID_FIELD)) .path(resultSet.getString(PATH_FIELD)) .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) .tableName(resultSet.getString(TABLE_NAME_FIELD)) - .housekeepingStatus( - HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) - .creationTimestamp( - Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) - .modifiedTimestamp( - Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) + .housekeepingStatus(HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) + .creationTimestamp(Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) + .modifiedTimestamp(Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) .cleanupDelay(PeriodDuration.parse(resultSet.getString(CLEANUP_DELAY_FIELD))) .cleanupAttempts(resultSet.getInt(CLEANUP_ATTEMPTS_FIELD)) .clientId(resultSet.getString(CLIENT_ID_FIELD)) @@ -56,20 +56,17 @@ public static HousekeepingPath mapToHousekeepingPath(ResultSet resultSet) throws .build(); } - public static HousekeepingMetadata mapToHousekeepingMetadata(ResultSet resultSet) - throws SQLException { - return HousekeepingMetadata.builder() + public static HousekeepingMetadata mapToHousekeepingMetadata(ResultSet resultSet) throws SQLException { + return HousekeepingMetadata + .builder() .id(resultSet.getLong(ID_FIELD)) .path(resultSet.getString(PATH_FIELD)) .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) .tableName(resultSet.getString(TABLE_NAME_FIELD)) .partitionName(resultSet.getString(PARTITION_NAME_FIELD)) - .housekeepingStatus( - HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) - .creationTimestamp( - Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) - .modifiedTimestamp( - Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) + .housekeepingStatus(HousekeepingStatus.valueOf(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))) + .creationTimestamp(Timestamp.valueOf(resultSet.getString(CREATION_TIMESTAMP_FIELD)).toLocalDateTime()) + .modifiedTimestamp(Timestamp.valueOf(resultSet.getString(MODIFIED_TIMESTAMP_FIELD)).toLocalDateTime()) .cleanupDelay(PeriodDuration.parse(resultSet.getString(CLEANUP_DELAY_FIELD))) .cleanupAttempts(resultSet.getInt(CLEANUP_ATTEMPTS_FIELD)) .clientId(resultSet.getString(CLIENT_ID_FIELD)) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java index 2f3b10d5..c4b5f244 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/TestAppender.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2020 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.integration.utils; diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java index b7c5f8f3..a634454a 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java @@ -1,14 +1,16 @@ /** - * Copyright (C) 2019-2026 Expedia, Inc. + * Copyright (C) 2019-2025 Expedia, Inc. * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - *

http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ package com.expediagroup.beekeeper.path.cleanup.context; @@ -47,9 +49,9 @@ @Configuration @EnableScheduling -@ComponentScan({"com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.cleanup"}) -@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) -@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) +@ComponentScan({ "com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.cleanup" }) +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) public class CommonBeans { @Bean @@ -72,18 +74,20 @@ public AmazonS3 amazonS3Test() { @Bean public BytesDeletedReporter bytesDeletedReporter( - MeterRegistry meterRegistry, @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { + MeterRegistry meterRegistry, + @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { return new BytesDeletedReporter(meterRegistry, dryRunEnabled); } @Bean - public S3Client s3Client( - AmazonS3 amazonS3, @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { + public S3Client s3Client(AmazonS3 amazonS3, @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { return new S3Client(amazonS3, dryRunEnabled); } @Bean(name = "s3PathCleaner") - public PathCleaner pathCleaner(S3Client s3Client, BytesDeletedReporter bytesDeletedReporter) { + public PathCleaner pathCleaner( + S3Client s3Client, + BytesDeletedReporter bytesDeletedReporter) { return new S3PathCleaner(s3Client, new S3SentinelFilesCleaner(s3Client), bytesDeletedReporter); } @@ -108,8 +112,7 @@ public DisableTablesService disableTablesService() { } @Bean - public BeekeeperHistoryService beekeeperHistoryService( - BeekeeperHistoryRepository beekeeperHistoryRepository) { + public BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { return new BeekeeperHistoryService(beekeeperHistoryRepository); } } diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java index 881aba7e..fce3eb0f 100644 --- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java +++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperRepositoryTest.java @@ -21,6 +21,8 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; +import static java.time.temporal.ChronoUnit.MICROS; + import java.time.LocalDateTime; import java.time.ZoneId; import java.util.List; @@ -109,7 +111,7 @@ public void notNullableField() { } private HousekeepingPath createEntityHousekeepingPath() { - LocalDateTime creationTimestamp = LocalDateTime.now(ZoneId.of("UTC")); + LocalDateTime creationTimestamp = LocalDateTime.now(ZoneId.of("UTC")).truncatedTo(MICROS); return HousekeepingPath .builder() .path("path") From d5fdd594372a051d9490efc61f3850ec8a9ff1ea Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Thu, 26 Mar 2026 14:20:43 -0500 Subject: [PATCH 07/12] Clean up --- .../api/controller/BeekeeperController.java | 24 +- .../api/error/BeekeeperExceptionHandler.java | 13 +- .../cleanup/aws/S3PathCleanerTest.java | 6 +- .../aws/S3SentinelFilesCleanerTest.java | 2 +- .../HousekeepingMetadataRepository.java | 14 +- .../HousekeepingPathRepository.java | 3 +- .../beekeeper/core/TestApplication.java | 4 +- ...rDryRunMetadataCleanupIntegrationTest.java | 32 +-- ...eeperDryRunPathCleanupIntegrationTest.java | 8 +- ...etadataSchedulerApiaryIntegrationTest.java | 36 ++- .../BeekeeperPathCleanupIntegrationTest.java | 52 ++--- ...cedPathSchedulerApiaryIntegrationTest.java | 103 +++------ .../api/BeekeeperApiIntegrationTest.java | 191 +++++----------- .../integration/utils/ContainerTestUtils.java | 18 +- .../metadata/cleanup/TestApplication.java | 4 +- .../PagingMetadataCleanupServiceTest.java | 206 +++++++----------- .../path/cleanup/TestApplication.java | 4 +- .../scheduler/apiary/context/CommonBeans.java | 91 ++++---- .../apiary/context/CommonBeansTest.java | 38 ++-- .../beekeeper/vacuum/CommonBeans.java | 28 +-- .../vacuum/ConsistencyCheckTest.java | 10 +- 21 files changed, 341 insertions(+), 546 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index 3690683e..ca4304cd 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -80,9 +80,7 @@ public ResponseEntity> getAllMetadata( return ResponseEntity.ok(housekeepingEntityService.getAllMetadata(spec, pageable)); } - @RequestMapping( - value = "/database/{databaseName}/table/{tableName}/unreferenced-paths", - method = RequestMethod.GET) + @RequestMapping(value = "/database/{databaseName}/table/{tableName}/unreferenced-paths", method = RequestMethod.GET) @Parameter(name = "tableName", in = ParameterIn.PATH) @Parameter(name = "databaseName", in = ParameterIn.PATH) @Parameter(name = "path", in = ParameterIn.QUERY) @@ -97,16 +95,16 @@ public ResponseEntity> getAllPaths( @PathVariable String databaseName, @PathVariable String tableName, @Parameter(hidden = true) @And(value = { - @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), - @Spec(path = "databaseName", pathVars = "databaseName", spec = EqualIgnoreCase.class), - @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), - @Spec(path = "partitionName", params = "partition_name", spec = EqualIgnoreCase.class), - @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), - @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), - @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), - @Spec(path = "cleanupTimestamp", params = "deleted_after", spec = GreaterThan.class), - @Spec(path = "creationTimestamp", params = "registered_before", spec = LessThan.class), - @Spec(path = "creationTimestamp", params = "registered_after", spec = GreaterThan.class) }) Specification spec, + @Spec(path = "tableName", pathVars = "tableName", spec = EqualIgnoreCase.class), + @Spec(path = "databaseName", pathVars = "databaseName", spec = EqualIgnoreCase.class), + @Spec(path = "path", params = "path", spec = EqualIgnoreCase.class), + @Spec(path = "partitionName", params = "partition_name", spec = EqualIgnoreCase.class), + @Spec(path = "housekeepingStatus", params = "housekeeping_status", spec = EqualIgnoreCase.class), + @Spec(path = "lifecycleType", params = "lifecycle_type", spec = EqualIgnoreCase.class), + @Spec(path = "cleanupTimestamp", params = "deleted_before", spec = LessThan.class), + @Spec(path = "cleanupTimestamp", params = "deleted_after", spec = GreaterThan.class), + @Spec(path = "creationTimestamp", params = "registered_before", spec = LessThan.class), + @Spec(path = "creationTimestamp", params = "registered_after", spec = GreaterThan.class) }) Specification spec, @ParameterObject Pageable pageable) { return ResponseEntity.ok(housekeepingEntityService.getAllPaths(spec, pageable)); } diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java index 0a04224b..963f1179 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/error/BeekeeperExceptionHandler.java @@ -41,13 +41,14 @@ public ResponseEntity handlePropertyReferenceException( PropertyReferenceException exception, HttpServletRequest request) { ErrorResponse errorResponse = ErrorResponse.builder() - .timestamp(LocalDateTime.now().toString()) - .status(HttpStatus.BAD_REQUEST.value()) - .error(HttpStatus.BAD_REQUEST.getReasonPhrase()) - .message(exception.getMessage()) - .path(request.getRequestURI()) - .build(); + .timestamp(LocalDateTime.now().toString()) + .status(HttpStatus.BAD_REQUEST.value()) + .error(HttpStatus.BAD_REQUEST.getReasonPhrase()) + .message(exception.getMessage()) + .path(request.getRequestURI()) + .build(); return new ResponseEntity<>(errorResponse, HttpStatus.BAD_REQUEST); } + } diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java index 8157a90c..c958c134 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java @@ -95,9 +95,9 @@ class S3PathCleanerTest { void setUp() { amazonS3 = AmazonS3ClientBuilder .standard() - .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) - .build(); + .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(S3_ENDPOINT, "region")) + .build(); amazonS3.createBucket(bucket); amazonS3 .listObjectsV2(bucket) diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java index 1c481a35..559d82cc 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3SentinelFilesCleanerTest.java @@ -58,7 +58,7 @@ class S3SentinelFilesCleanerTest { @BeforeEach void setUp() { - amazonS3 =AmazonS3ClientBuilder + amazonS3 = AmazonS3ClientBuilder .standard() .withCredentials(new BasicAWSCredentialsProvider("accesskey", "secretkey")) .withEndpointConfiguration( diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java index 6bc47d6a..81d27827 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingMetadataRepository.java @@ -36,7 +36,8 @@ public interface HousekeepingMetadataRepository + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED') " + "and t.modifiedTimestamp <= :instant and t.cleanupAttempts < 10 order by t.modifiedTimestamp") Slice findRecordsForCleanupByModifiedTimestamp( - @Param("instant") LocalDateTime instant, Pageable pageable); + @Param("instant") LocalDateTime instant, + Pageable pageable); /** * Returns the record that matches the inputs given, if there is one. @@ -86,7 +87,8 @@ LocalDateTime findMaximumCleanupTimestampForDbAndTable( + "and t.tableName = :tableName " + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") Long countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull( - @Param("databaseName") String databaseName, @Param("tableName") String tableName); + @Param("databaseName") String databaseName, + @Param("tableName") String tableName); /** * This method is used for dry runs since the entries are not being updated. It counts the number of partitions on a @@ -121,10 +123,11 @@ Long countRecordsForDryRunWherePartitionIsNotNullOrExpired( + "and t.partitionName is not NULL " + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") void deleteScheduledOrFailedPartitionRecordsForTable( - @Param("databaseName") String databaseName, @Param("tableName") String tableName); + @Param("databaseName") String databaseName, + @Param("tableName") String tableName); /** - * This method returns all table records where the partition name is NULL and the status is SCHEDULED` or `FAILED` + * This method returns all table records where the partition name is NULL and the status is `SCHEDULED` or `FAILED` */ @Query(value = "from HousekeepingMetadata t " + "where t.partitionName is NULL " @@ -155,5 +158,6 @@ void deleteScheduledOrFailedPartitionRecordsForTable( + "and t.partitionName IS NOT NULL " + "and (t.housekeepingStatus = 'SCHEDULED' or t.housekeepingStatus = 'FAILED')") List findRecordsForCleanupByDbAndTableName( - @Param("databaseName") String databaseName, @Param("tableName") String tableName); + @Param("databaseName") String databaseName, + @Param("tableName") String tableName); } diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java index ae3460e8..04397025 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/HousekeepingPathRepository.java @@ -35,8 +35,7 @@ public interface HousekeepingPathRepository @Query(value = "from HousekeepingPath p where p.cleanupTimestamp <= :instant " + "and (p.housekeepingStatus = 'SCHEDULED' or p.housekeepingStatus = 'FAILED') " + "and p.modifiedTimestamp <= :instant and p.cleanupAttempts < 10") - Slice findRecordsForCleanup( - @Param("instant") LocalDateTime instant, Pageable pageable); + Slice findRecordsForCleanup(@Param("instant") LocalDateTime instant, Pageable pageable); @Modifying @Query(value = "delete from HousekeepingPath p where p.cleanupTimestamp < :instant " diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java index 09c4c013..8c78b910 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/TestApplication.java @@ -26,8 +26,8 @@ @SpringBootApplication @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") -@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) -@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) public class TestApplication { @PostConstruct diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java index 91f730ef..ef609ef2 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java @@ -96,20 +96,20 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1"; private static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; private static final String PARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE - + "/" - + PARTITIONED_TABLE_NAME - + "/id1"; + + "/" + + PARTITIONED_TABLE_NAME + + "/id1"; private static final String PARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE - + "/some_location/id1/" - + PARTITION_NAME - + "/file1"; + + "/some_location/id1/" + + PARTITION_NAME + + "/file1"; private static final String UNPARTITIONED_TABLE_PATH = ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; private static final String UNPARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE - + "/" - + UNPARTITIONED_TABLE_NAME - + "/id1"; + + "/" + + UNPARTITIONED_TABLE_NAME + + "/id1"; private static final String S3_CLIENT_CLASS_NAME = "S3Client"; private static final String HIVE_CLIENT_CLASS_NAME = "HiveClient"; @@ -121,7 +121,7 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private final ExecutorService executorService = Executors.newFixedThreadPool(1); private final TestAppender appender = new TestAppender(); - private Map metastoreProperties = ImmutableMap + private static Map metastoreProperties = ImmutableMap .builder() .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) .put(ACCESS_KEY, S3_ACCESS_KEY) @@ -220,7 +220,8 @@ public void dryRunDropPartitionedTable() throws Exception { public void dryRunDontDropPartitionedTable() throws Exception { Table table = hiveTestUtils.createTableWithDeletionProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); - hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); + hiveTestUtils + .addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); String partition2Name = "event_date=2020-01-01/event_hour=1/event_type=B"; String partition2Path = PARTITION_ROOT_PATH + "/" + partition2Name + "/file1"; @@ -266,11 +267,10 @@ public void dryRunMetrics() throws Exception { private void assertMetrics() { Set meterRegistry = ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach( - registry -> { - List meters = registry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name") - .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + + meterRegistry.forEach(registry -> { + List meters = registry.getMeters(); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name") + .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, "hive-partition-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, "s3-paths-deleted", "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java index b34e0364..01ef194c 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunPathCleanupIntegrationTest.java @@ -181,9 +181,7 @@ public void logsForParentDeletion() throws SQLException { amazonS3.putObject(BUCKET, tableSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> logsContainLineFromS3Client(parentSentinel)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(parentSentinel)); assertS3ClientLogs(4); } @@ -211,9 +209,7 @@ public void metrics() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> logsContainLineFromS3Client(OBJECT_KEY_SENTINEL)); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isTrue(); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_SENTINEL)).isTrue(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index 14faca94..f78e973b 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -123,11 +123,11 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek protected static AmazonS3 amazonS3; private static Map metastoreProperties = ImmutableMap - .builder() - .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) - .put(ACCESS_KEY, S3_ACCESS_KEY) - .put(SECRET_KEY, S3_SECRET_KEY) - .build(); + .builder() + .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) + .put(ACCESS_KEY, S3_ACCESS_KEY) + .put(SECRET_KEY, S3_SECRET_KEY) + .build(); @RegisterExtension protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( @@ -164,9 +164,7 @@ public void setup() { hiveTestUtils = new HiveTestUtils(metastoreClient); amazonSQS.purgeQueue(new PurgeQueueRequest(ACTUAL_QUEUE_URL)); - amazonS3 - .listObjectsV2(BUCKET) - .getObjectSummaries() + amazonS3.listObjectsV2(BUCKET).getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); await().atMost(Duration.ofMinutes(1)).until(BeekeeperSchedulerApiary::isRunning); @@ -222,9 +220,9 @@ public void expiredMetadataAddPartitionEvent() throws SQLException, IOException, @Test public void expiredMetadataMultipleAddPartitionEvents() throws SQLException, IOException, URISyntaxException { AddPartitionSqsMessage addPartitionSqsMessage = new AddPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, - PARTITION_A_VALUES, true); + PARTITION_A_VALUES, true); AddPartitionSqsMessage addPartitionSqsMessage2 = new AddPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, - PARTITION_B_VALUES, true); + PARTITION_B_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage.getFormattedString())); amazonSQS.sendMessage(sendMessageRequest(addPartitionSqsMessage2.getFormattedString())); @@ -260,9 +258,9 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLExcepti insertExpiredMetadata(LOCATION_B + "-old", PARTITION_B_NAME); AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage(LOCATION_A, PARTITION_KEYS, - PARTITION_A_VALUES, true); + PARTITION_A_VALUES, true); AlterPartitionSqsMessage alterPartitionSqsMessage2 = new AlterPartitionSqsMessage(LOCATION_B, PARTITION_KEYS, - PARTITION_B_VALUES, true); + PARTITION_B_VALUES, true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage2.getFormattedString())); @@ -323,7 +321,7 @@ void scheduleMissingPartitionsWhenPropertiesExpireInTable() throws Exception { hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); insertExpiredMetadata(PARTITION_ROOT_PATH, null); - insertExpiredMetadata(PARTITION_ROOT_PATH + PARTITION_A_NAME + "/event_type=C",PARTITION_A_NAME + "/event_type=C"); + insertExpiredMetadata(PARTITION_ROOT_PATH + PARTITION_A_NAME + "/event_type=C", PARTITION_A_NAME + "/event_type=C"); AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage(PARTITION_ROOT_PATH, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); @@ -340,16 +338,16 @@ void scheduleMissingPartitionsWhenPropertiesExpireInTable() throws Exception { public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(HEALTHCHECK_URI); - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } @Test public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } protected SendMessageRequest sendMessageRequest(String payload) { @@ -384,8 +382,8 @@ public void assertMetrics() { .getRegistries(); assertThat(meterRegistry).hasSize(2); meterRegistry.forEach(registry -> { - List meters = registry.getMeters(); - assertThat(meters).extracting("id", Meter.Id.class).extracting("name").contains(SCHEDULED_EXPIRED_METRIC); + List meters = registry.getMeters(); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name").contains(SCHEDULED_EXPIRED_METRIC); }); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index f7cd36be..1a503114 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -69,13 +69,11 @@ public class BeekeeperPathCleanupIntegrationTest extends BeekeeperIntegrationTes protected static final String OBJECT_KEY_ROOT = DB_AND_TABLE_PREFIX + "/id1/partition1"; protected static final String OBJECT_KEY1 = DB_AND_TABLE_PREFIX + "/id1/partition1/file1"; protected static final String OBJECT_KEY2 = DB_AND_TABLE_PREFIX + "/id1/partition1/file2"; - protected static final String OBJECT_KEY_SENTINEL = - DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; + protected static final String OBJECT_KEY_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; protected static final String ABSOLUTE_PATH = "s3://" + BUCKET + "/" + OBJECT_KEY_ROOT; protected static final String OBJECT_KEY_OTHER = DB_AND_TABLE_PREFIX + "/id1/partition10/file1"; - protected static final String OBJECT_KEY_OTHER_SENTINEL = - DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; + protected static final String OBJECT_KEY_OTHER_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; protected static final String SPRING_PROFILES_ACTIVE = "test"; protected static final String SCHEDULER_DELAY_MS = "5000"; @@ -86,7 +84,6 @@ public class BeekeeperPathCleanupIntegrationTest extends BeekeeperIntegrationTes @Container protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); - protected static AmazonS3 amazonS3; protected final ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -96,8 +93,7 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, SPRING_PROFILES_ACTIVE); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, DRY_RUN_ENABLED); - System.setProperty( - AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); + System.setProperty(AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); amazonS3 = ContainerTestUtils.s3Client(S3_CONTAINER, AWS_REGION); amazonS3.createBucket(new CreateBucketRequest(BUCKET, AWS_REGION)); @@ -115,8 +111,7 @@ public static void teardown() { @BeforeEach public void setup() { - amazonS3 - .listObjectsV2(BUCKET) + amazonS3.listObjectsV2(BUCKET) .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {})); @@ -137,8 +132,7 @@ public void cleanupPathsForFile() throws SQLException { String path = "s3://" + BUCKET + "/" + OBJECT_KEY1; insertUnreferencedPath(path); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -156,8 +150,7 @@ public void cleanupPathsForDirectory() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -196,8 +189,7 @@ public void cleanupPathsForDirectoryWithTrailingSlash() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH + "/"); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -219,8 +211,7 @@ public void cleanupSentinelForParent() throws SQLException { amazonS3.putObject(BUCKET, databaseSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -263,9 +254,7 @@ public void testEventAddedToHistoryTable() throws SQLException { String path = "s3://" + BUCKET + "/" + OBJECT_KEY1; insertUnreferencedPath(path); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); // deleting a file shouldn't delete a folder sentinel @@ -286,8 +275,7 @@ public void metrics() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -296,16 +284,12 @@ public void metrics() throws SQLException { } private void assertMetrics() { - Set meterRegistry = - ((CompositeMeterRegistry) BeekeeperPathCleanup.meterRegistry()).getRegistries(); + Set meterRegistry = ((CompositeMeterRegistry) BeekeeperPathCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach( - registry -> { - List meters = registry.getMeters(); - assertThat(meters) - .extracting("id", Meter.Id.class) - .extracting("name") - .contains("path-cleanup-job", "s3-paths-deleted", "s3-" + METRIC_NAME); + meterRegistry.forEach(registry -> { + List meters = registry.getMeters(); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name") + .contains("path-cleanup-job", "s3-paths-deleted", "s3-" + METRIC_NAME); }); } @@ -313,8 +297,7 @@ private void assertMetrics() { public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(HEALTHCHECK_URI); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } @@ -322,8 +305,7 @@ public void healthCheck() { public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 33155357..2f4576ed 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -71,8 +71,7 @@ import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension; @Testcontainers -public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest - extends BeekeeperIntegrationTestBase { +public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { protected static final int TIMEOUT = 30; protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; @@ -85,13 +84,12 @@ public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest @Container protected static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); - protected static AmazonSQS amazonSQS; protected static String ACTUAL_QUEUE_URL; @RegisterExtension - protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = - new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, emptyMap()); + protected ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( + DATABASE_NAME_VALUE, emptyMap()); protected static final String SQS_ENDPOINT_PROPERTY = "properties.sqs.endpoint"; protected static final String SQS_REGION_PROPERTY = "properties.sqs.region"; @@ -102,8 +100,7 @@ public static void init() { ACTUAL_QUEUE_URL = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); amazonSQS.createQueue(QUEUE); System.setProperty(APIARY_QUEUE_URL_PROPERTY, ACTUAL_QUEUE_URL); - System.setProperty( - SQS_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(SQS_CONTAINER, SQS)); + System.setProperty(SQS_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(SQS_CONTAINER, SQS)); System.setProperty(SQS_REGION_PROPERTY, AWS_REGION); } @@ -133,9 +130,8 @@ public void stop() throws InterruptedException { @Test public void unreferencedAlterTableEvent() throws SQLException, IOException, URISyntaxException { - AlterTableSqsMessage alterTableSqsMessage = - new AlterTableSqsMessage( - "s3://bucket/tableLocation", "s3://bucket/oldTableLocation", true, true); + AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", + "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 1); @@ -144,11 +140,9 @@ public void unreferencedAlterTableEvent() throws SQLException, IOException, URIS } @Test - public void unreferencedMultipleAlterTableEvents() - throws SQLException, IOException, URISyntaxException { - AlterTableSqsMessage alterTableSqsMessage = - new AlterTableSqsMessage( - "s3://bucket/tableLocation", "s3://bucket/oldTableLocation", true, true); + public void unreferencedMultipleAlterTableEvents() throws SQLException, IOException, URISyntaxException { + AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", + "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); alterTableSqsMessage.setTableLocation("s3://bucket/tableLocation2"); alterTableSqsMessage.setOldTableLocation("s3://bucket/tableLocation"); @@ -161,15 +155,10 @@ public void unreferencedMultipleAlterTableEvents() } @Test - public void unreferencedAlterPartitionEvent() - throws SQLException, IOException, URISyntaxException { - AlterPartitionSqsMessage alterPartitionSqsMessage = - new AlterPartitionSqsMessage( - "s3://bucket/table/expiredTableLocation", - "s3://bucket/table/partitionLocation", - "s3://bucket/table/unreferencedPartitionLocation", - true, - true); + public void unreferencedAlterPartitionEvent() throws SQLException, IOException, URISyntaxException { + AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage( + "s3://bucket/table/expiredTableLocation", "s3://bucket/table/partitionLocation", + "s3://bucket/table/unreferencedPartitionLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); alterPartitionSqsMessage.setTableLocation("s3://bucket/table/expiredTableLocation2"); alterPartitionSqsMessage.setPartitionLocation("s3://bucket/table/partitionLocation2"); @@ -179,41 +168,30 @@ public void unreferencedAlterPartitionEvent() List unreferencedPaths = getUnreferencedPaths(); assertUnreferencedPath(unreferencedPaths.get(0), "s3://bucket/table/partitionLocation"); - assertUnreferencedPath( - unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); + assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); } @Test public void unreferencedMultipleAlterPartitionEvent() throws IOException, SQLException, URISyntaxException { - List.of( - new AlterPartitionSqsMessage( - "s3://bucket/table/expiredTableLocation", - "s3://bucket/table/partitionLocation", - "s3://bucket/table/unreferencedPartitionLocation", - true, - true), - new AlterPartitionSqsMessage( - "s3://bucket/table/expiredTableLocation2", - "s3://bucket/table/partitionLocation2", - "s3://bucket/table/partitionLocation", - true, - true)) + List + .of(new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation", + "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), + new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation2", + "s3://bucket/table/partitionLocation2", "s3://bucket/table/partitionLocation", true, true)) .forEach(msg -> amazonSQS.sendMessage(sendMessageRequest(msg.getFormattedString()))); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); List unreferencedPaths = getUnreferencedPaths(); assertUnreferencedPath(unreferencedPaths.get(0), "s3://bucket/table/partitionLocation"); - assertUnreferencedPath( - unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); + assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/table/unreferencedPartitionLocation"); } @Test - public void unreferencedDropPartitionEvent() - throws SQLException, IOException, URISyntaxException { - DropPartitionSqsMessage dropPartitionSqsMessage = - new DropPartitionSqsMessage("s3://bucket/table/partitionLocation", true, true); + public void unreferencedDropPartitionEvent() throws SQLException, IOException, URISyntaxException { + DropPartitionSqsMessage dropPartitionSqsMessage = new DropPartitionSqsMessage("s3://bucket/table/partitionLocation", + true, true); amazonSQS.sendMessage(sendMessageRequest(dropPartitionSqsMessage.getFormattedString())); dropPartitionSqsMessage.setPartitionLocation("s3://bucket/table/partitionLocation2"); amazonSQS.sendMessage(sendMessageRequest(dropPartitionSqsMessage.getFormattedString())); @@ -226,8 +204,7 @@ public void unreferencedDropPartitionEvent() @Test public void unreferencedDropTableEvent() throws SQLException, IOException, URISyntaxException { - DropTableSqsMessage dropTableSqsMessage = - new DropTableSqsMessage("s3://bucket/tableLocation", true, true); + DropTableSqsMessage dropTableSqsMessage = new DropTableSqsMessage("s3://bucket/tableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(dropTableSqsMessage.getFormattedString())); dropTableSqsMessage.setTableLocation("s3://bucket/tableLocation2"); amazonSQS.sendMessage(sendMessageRequest(dropTableSqsMessage.getFormattedString())); @@ -240,14 +217,11 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy @Test public void testEventAddedToHistoryTable() throws IOException, URISyntaxException, SQLException { - AlterTableSqsMessage alterTableSqsMessage = - new AlterTableSqsMessage( - "s3://bucket/tableLocation", "s3://bucket/oldTableLocation", true, true); + AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", + "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); BeekeeperHistory history = beekeeperHistory.get(0); @@ -270,9 +244,7 @@ public void healthCheck() { public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await() - .atMost(30, TimeUnit.SECONDS) - .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); + await().atMost(30, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } protected SendMessageRequest sendMessageRequest(String payload) { @@ -291,8 +263,7 @@ public void assertHousekeepingEntity(HousekeepingPath actual, String expectedPat assertThat(actual.getHousekeepingStatus()).isEqualTo(SCHEDULED); assertThat(actual.getCreationTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE); assertThat(actual.getModifiedTimestamp()).isAfterOrEqualTo(CREATION_TIMESTAMP_VALUE); - assertThat(actual.getCleanupTimestamp()) - .isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); + assertThat(actual.getCleanupTimestamp()).isEqualTo(actual.getCreationTimestamp().plus(actual.getCleanupDelay())); assertThat(actual.getCleanupDelay()).isEqualTo(PeriodDuration.parse(SHORT_CLEANUP_DELAY_VALUE)); assertThat(actual.getCleanupAttempts()).isEqualTo(CLEANUP_ATTEMPTS_VALUE); assertThat(actual.getClientId()).isEqualTo(CLIENT_ID_VALUE); @@ -300,16 +271,12 @@ public void assertHousekeepingEntity(HousekeepingPath actual, String expectedPat } public void assertMetrics() { - Set meterRegistry = - ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()).getRegistries(); + Set meterRegistry = ((CompositeMeterRegistry) BeekeeperSchedulerApiary.meterRegistry()) + .getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach( - registry -> { - List meters = registry.getMeters(); - assertThat(meters) - .extracting("id", Meter.Id.class) - .extracting("name") - .contains(SCHEDULED_ORPHANED_METRIC); - }); + meterRegistry.forEach(registry -> { + List meters = registry.getMeters(); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name").contains(SCHEDULED_ORPHANED_METRIC); + }); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index eab87e1a..74183730 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -64,11 +64,8 @@ public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { public ObjectMapper geObjectMapper() { - return new ObjectMapper() - .registerModule(new ParameterNamesModule()) - .registerModule(new JavaTimeModule()); + return new ObjectMapper().registerModule(new ParameterNamesModule()).registerModule(new JavaTimeModule()); } - private static final Logger log = LoggerFactory.getLogger(BeekeeperApiIntegrationTest.class); protected static ConfigurableApplicationContext context; protected BeekeeperApiTestClient testClient; @@ -76,78 +73,22 @@ public ObjectMapper geObjectMapper() { private Long id = 1L; protected static final String someTable = "some_table"; protected static final String someDatabase = "some_database"; - protected static final String pathA = - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; - protected static final String pathB = - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B"; - protected static final String pathC = - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C"; + protected static final String pathA = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; + protected static final String pathB = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B"; + protected static final String pathC = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C"; protected static final Duration duration = Duration.parse("P3D"); protected static final int pageSize = 1; protected static final String partitionA = "event_date=2020-01-01/event_hour=0/event_type=A"; protected static final String partitionB = "event_date=2020-01-01/event_hour=0/event_type=B"; protected static final String partitionC = "event_date=2020-01-01/event_hour=0/event_type=C"; - protected final HousekeepingPath testPathA = - createHousekeepingPath( - someTable, - pathA, - LifecycleEventType.EXPIRED, - duration.toString(), - HousekeepingStatus.FAILED); - protected final HousekeepingPath testPathB = - createHousekeepingPath( - someTable, - pathB, - LifecycleEventType.UNREFERENCED, - duration.toString(), - HousekeepingStatus.FAILED); - protected final HousekeepingPath testPathC = - createHousekeepingPath( - someTable, - pathC, - LifecycleEventType.UNREFERENCED, - duration.toString(), - HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataA = - createHousekeepingMetadata( - someTable, - pathA, - partitionA, - LifecycleEventType.EXPIRED, - duration.toString(), - HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataB = - createHousekeepingMetadata( - someTable, - pathB, - partitionB, - LifecycleEventType.EXPIRED, - duration.toString(), - HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataC = - createHousekeepingMetadata( - someTable, - pathC, - partitionC, - LifecycleEventType.EXPIRED, - duration.toString(), - HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataD = - createHousekeepingMetadata( - someTable, - pathC, - partitionB, - LifecycleEventType.UNREFERENCED, - duration.toString(), - HousekeepingStatus.SCHEDULED); - protected final HousekeepingMetadata testMetadataE = - createHousekeepingMetadata( - someTable, - pathC, - partitionC, - LifecycleEventType.UNREFERENCED, - duration.toString(), - HousekeepingStatus.SCHEDULED); + protected final HousekeepingPath testPathA = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString(), HousekeepingStatus.FAILED); + protected final HousekeepingPath testPathB = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); + protected final HousekeepingPath testPathC = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataA = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataB = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataC = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataD = createHousekeepingMetadata(someTable, pathC, partitionB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.SCHEDULED); + protected final HousekeepingMetadata testMetadataE = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString(),HousekeepingStatus.SCHEDULED); @BeforeEach public void beforeEach() throws IOException { @@ -189,30 +130,26 @@ public void testGetMetadataWhenTableNotFoundReturnsEmptyList() } @Test - public void testGetMetadataWhenThereIsFiltering() - throws SQLException, InterruptedException, IOException { + public void testGetMetadataWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { testMetadataA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testMetadataA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - for (HousekeepingMetadata testMetadata : - Arrays.asList(testMetadataA, testMetadataD, testMetadataE)) { + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataD, testMetadataE)) { insertExpiredMetadata(testMetadata); } - String filters = - "?housekeeping_status=FAILED" - + "&lifecycle_type=EXPIRED" - + "&deleted_before=2000-05-05T10:41:20" - + "®istered_before=2000-04-04T10:41:20" - + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A" - + "&partition_name=event_date=2020-01-01/event_hour=0/event_type=A"; + String filters = "?housekeeping_status=FAILED" + + "&lifecycle_type=EXPIRED" + + "&deleted_before=2000-05-05T10:41:20" + + "®istered_before=2000-04-04T10:41:20" + + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A" + + "&partition_name=event_date=2020-01-01/event_hour=0/event_type=A"; HttpResponse response = testClient.getMetadata(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = - mapper.readValue( - body, new TypeReference>() {}); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThatMetadataEqualsResponse(testMetadataA, result.get(0)); @@ -220,27 +157,24 @@ public void testGetMetadataWhenThereIsFiltering() } @Test - public void testGetPathsWhenThereIsFiltering() - throws SQLException, InterruptedException, IOException { + public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { testPathA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testPathA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { insertUnreferencedPath(testPath); } - String filters = - "?housekeeping_status=FAILED" - + "&lifecycle_type=EXPIRED" - + "&deleted_before=2000-05-05T10:41:20" - + "®istered_before=2000-04-04T10:41:20" - + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; - - HttpResponse response = - testClient.getUnreferencedPaths(someDatabase, someTable, filters); + String filters = "?housekeeping_status=FAILED" + + "&lifecycle_type=EXPIRED" + + "&deleted_before=2000-05-05T10:41:20" + + "®istered_before=2000-04-04T10:41:20" + + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; + + HttpResponse response = testClient.getUnreferencedPaths(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = - mapper.readValue(body, new TypeReference>() {}); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThat(responsePage.getTotalElements()).isEqualTo(1L); assertThatPathsEqualsResponse(testPathA, result.get(0)); @@ -254,13 +188,12 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc } String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; - HttpResponse response = - testClient.getUnreferencedPaths(someDatabase, someTable, filters); + HttpResponse response = testClient.getUnreferencedPaths(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = - mapper.readValue(body, new TypeReference>() {}); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); @@ -269,8 +202,7 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc @Test public void testMetadataPageable() throws SQLException, InterruptedException, IOException { - for (HousekeepingMetadata testMetadata : - Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } @@ -279,9 +211,8 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = - mapper.readValue( - body, new TypeReference>() {}); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); @@ -292,8 +223,7 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO @Disabled @Test public void manualTest() throws SQLException, InterruptedException { - for (HousekeepingMetadata testMetadata : - Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } @@ -307,35 +237,27 @@ public void manualTest() throws SQLException, InterruptedException { private void assertThatMetadataEqualsResponse( HousekeepingMetadata housekeepingMetadata, HousekeepingMetadataResponse housekeepingMetadataResponse) { - assertThat(housekeepingMetadata.getDatabaseName()) - .isEqualTo(housekeepingMetadataResponse.getDatabaseName()); - assertThat(housekeepingMetadata.getTableName()) - .isEqualTo(housekeepingMetadataResponse.getTableName()); + assertThat(housekeepingMetadata.getDatabaseName()).isEqualTo(housekeepingMetadataResponse.getDatabaseName()); + assertThat(housekeepingMetadata.getTableName()).isEqualTo(housekeepingMetadataResponse.getTableName()); assertThat(housekeepingMetadata.getPath()).isEqualTo(housekeepingMetadataResponse.getPath()); assertThat(housekeepingMetadata.getHousekeepingStatus()) .isEqualTo(housekeepingMetadataResponse.getHousekeepingStatus()); assertThat(housekeepingMetadata.getCleanupDelay().toString()) .isEqualTo(housekeepingMetadataResponse.getCleanupDelay()); - assertThat(housekeepingMetadata.getCleanupAttempts()) - .isEqualTo(housekeepingMetadataResponse.getCleanupAttempts()); - assertThat(housekeepingMetadata.getLifecycleType()) - .isEqualTo(housekeepingMetadataResponse.getLifecycleType()); + assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(housekeepingMetadataResponse.getCleanupAttempts()); + assertThat(housekeepingMetadata.getLifecycleType()).isEqualTo(housekeepingMetadataResponse.getLifecycleType()); } private void assertThatPathsEqualsResponse( - HousekeepingPath housekeepingPath, HousekeepingPathResponse housekeepingPathResponse) { - assertThat(housekeepingPath.getDatabaseName()) - .isEqualTo(housekeepingPathResponse.getDatabaseName()); + HousekeepingPath housekeepingPath, + HousekeepingPathResponse housekeepingPathResponse) { + assertThat(housekeepingPath.getDatabaseName()).isEqualTo(housekeepingPathResponse.getDatabaseName()); assertThat(housekeepingPath.getTableName()).isEqualTo(housekeepingPathResponse.getTableName()); assertThat(housekeepingPath.getPath()).isEqualTo(housekeepingPathResponse.getPath()); - assertThat(housekeepingPath.getHousekeepingStatus()) - .isEqualTo(housekeepingPathResponse.getHousekeepingStatus()); - assertThat(housekeepingPath.getCleanupDelay().toString()) - .isEqualTo(housekeepingPathResponse.getCleanupDelay()); - assertThat(housekeepingPath.getCleanupAttempts()) - .isEqualTo(housekeepingPathResponse.getCleanupAttempts()); - assertThat(housekeepingPath.getLifecycleType()) - .isEqualTo(housekeepingPathResponse.getLifecycleType()); + assertThat(housekeepingPath.getHousekeepingStatus()).isEqualTo(housekeepingPathResponse.getHousekeepingStatus()); + assertThat(housekeepingPath.getCleanupDelay().toString()).isEqualTo(housekeepingPathResponse.getCleanupDelay()); + assertThat(housekeepingPath.getCleanupAttempts()).isEqualTo(housekeepingPathResponse.getCleanupAttempts()); + assertThat(housekeepingPath.getLifecycleType()).isEqualTo(housekeepingPathResponse.getLifecycleType()); } private HousekeepingMetadata createHousekeepingMetadata( @@ -345,7 +267,8 @@ private HousekeepingMetadata createHousekeepingMetadata( LifecycleEventType lifecycleEventType, String cleanupDelay, HousekeepingStatus housekeepingStatus) { - return HousekeepingMetadata.builder() + return HousekeepingMetadata + .builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -367,7 +290,8 @@ private HousekeepingPath createHousekeepingPath( LifecycleEventType lifecycleEventType, String cleanupDelay, HousekeepingStatus housekeepingStatus) { - return HousekeepingPath.builder() + return HousekeepingPath + .builder() .id(id++) .path(path) .databaseName(DATABASE_NAME_VALUE) @@ -395,11 +319,10 @@ public void testInvalidSortParameter() throws SQLException, IOException, Interru ErrorResponse errorResponse = mapper.readValue(body, ErrorResponse.class); assertThat(errorResponse.getStatus()).isEqualTo(HttpStatus.BAD_REQUEST.value()); - assertThat(errorResponse.getMessage()) - .isEqualTo("No property 'nonExistentProperty' found for type 'HousekeepingMetadata'"); + assertThat(errorResponse.getMessage()).isEqualTo("No property 'nonExistentProperty' found for type 'HousekeepingMetadata'"); assertThat(errorResponse.getError()).isEqualTo("Bad Request"); - assertThat(errorResponse.getPath()) - .contains("/api/v1/database/some_database/table/some_table/metadata"); + assertThat(errorResponse.getPath()).contains("/api/v1/database/some_database/table/some_table/metadata"); assertThat(errorResponse.getTimestamp()).isNotNull(); } + } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java index b8a4c2c8..38f9bb7b 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java @@ -39,8 +39,7 @@ public static LocalStackContainer awsContainer(LocalStackContainer.Service servi .withServices(service); } - public static String awsServiceEndpoint( - LocalStackContainer awsContainer, LocalStackContainer.Service service) { + public static String awsServiceEndpoint(LocalStackContainer awsContainer, LocalStackContainer.Service service) { return awsContainer.getEndpointOverride(service).toString(); } @@ -49,19 +48,18 @@ public static String queueUrl(LocalStackContainer awsContainer, String queue) { } public static AmazonSQS sqsClient(LocalStackContainer awsContainer, String region) { - EndpointConfiguration endpointConfiguration = - new EndpointConfiguration(awsServiceEndpoint(awsContainer, SQS), region); - return AmazonSQSClientBuilder.standard() - .withEndpointConfiguration(endpointConfiguration) - .build(); + EndpointConfiguration endpointConfiguration = new EndpointConfiguration(awsServiceEndpoint(awsContainer, SQS), + region); + return AmazonSQSClientBuilder.standard().withEndpointConfiguration(endpointConfiguration).build(); } public static AmazonS3 s3Client(LocalStackContainer awsContainer, String region) { - EndpointConfiguration endpointConfiguration = - new EndpointConfiguration(awsServiceEndpoint(awsContainer, S3), region); + EndpointConfiguration endpointConfiguration = new EndpointConfiguration(awsServiceEndpoint(awsContainer, S3), + region); // build with disableChunkedEncoding to be able to create empty files - return AmazonS3ClientBuilder.standard() + return AmazonS3ClientBuilder + .standard() .withEndpointConfiguration(endpointConfiguration) .withPathStyleAccessEnabled(true) .disableChunkedEncoding() diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java index 439687e1..8ffaedad 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/TestApplication.java @@ -26,8 +26,8 @@ @SpringBootApplication @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") -@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) -@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) public class TestApplication { @PostConstruct diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java index 17db4390..3a181b7e 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java @@ -77,10 +77,9 @@ @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) public class PagingMetadataCleanupServiceTest { - public static final List TABLE_PATHS = - List.of("s3://bucket/table", "s3://bucket/table", "s3://bucket/table"); - public static final List PARTITION_PATHS = - List.of("s3://bucket/table/1", "s3://bucket/table/2", "s3://bucket/table/3"); + public static final List TABLE_PATHS = List.of("s3://bucket/table", "s3://bucket/table", "s3://bucket/table"); + public static final List PARTITION_PATHS = List + .of("s3://bucket/table/1", "s3://bucket/table/2", "s3://bucket/table/3"); private final LocalDateTime localNow = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC); private PagingMetadataCleanupService pagingCleanupService; private @Captor ArgumentCaptor metadataCaptor; @@ -100,8 +99,7 @@ public class PagingMetadataCleanupServiceTest { @BeforeEach public void init() { - when(metadataCleaner.tableExists(Mockito.any(), Mockito.anyString(), Mockito.anyString())) - .thenReturn(true); + when(metadataCleaner.tableExists(Mockito.any(), Mockito.anyString(), Mockito.anyString())).thenReturn(true); when(metadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(true); Map properties = new HashMap<>(); properties.put(UNREFERENCED.getTableParameterName(), "true"); @@ -110,13 +108,8 @@ public void init() { .thenReturn(properties); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); - handler = - new ExpiredMetadataHandler( - hiveClientFactory, - metadataRepository, - metadataCleaner, - pathCleaner, - beekeeperHistoryService); + handler = new ExpiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner, + beekeeperHistoryService); handlers = List.of(handler); pagingCleanupService = new PagingMetadataCleanupService(handlers, 2, false); } @@ -125,17 +118,14 @@ public void init() { public void typicalUnpartitioned() { List tables = List.of("table1", "table2", "table3"); - IntStream.range(0, tables.size()) - .forEach( - i -> - metadataRepository.save( - createHousekeepingMetadata( - tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); + IntStream + .range(0, tables.size()) + .forEach(i -> metadataRepository + .save(createHousekeepingMetadata(tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(3)) - .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(3)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0), tables.get(1), tables.get(2)); @@ -144,13 +134,10 @@ public void typicalUnpartitioned() { .extracting("path") .containsExactly(TABLE_PATHS.get(0), TABLE_PATHS.get(1), TABLE_PATHS.get(2)); - metadataRepository - .findAll() - .forEach( - housekeepingMetadata -> { - assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); - assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); - }); + metadataRepository.findAll().forEach(housekeepingMetadata -> { + assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); + assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); + }); pagingCleanupService.cleanUp(Instant.now()); verifyNoMoreInteractions(pathCleaner); @@ -162,17 +149,14 @@ public void typicalDryRunEnabled() { List tables = List.of("table1", "table2", "table3"); - IntStream.range(0, tables.size()) - .forEach( - i -> - metadataRepository.save( - createHousekeepingMetadata( - tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); + IntStream + .range(0, tables.size()) + .forEach(i -> metadataRepository + .save(createHousekeepingMetadata(tables.get(i), TABLE_PATHS.get(i), null, SCHEDULED))); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(3)) - .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(3)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0), tables.get(1), tables.get(2)); @@ -181,30 +165,24 @@ public void typicalDryRunEnabled() { .extracting("path") .containsExactly(TABLE_PATHS.get(0), TABLE_PATHS.get(1), TABLE_PATHS.get(2)); - metadataRepository - .findAll() - .forEach( - housekeepingMetadata -> { - assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(0); - assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(SCHEDULED); - }); + metadataRepository.findAll().forEach(housekeepingMetadata -> { + assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(0); + assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(SCHEDULED); + }); } @Test public void typicalPartitioned() { List tables = List.of("table1", "table2", "table3"); - IntStream.range(0, tables.size()) - .forEach( - i -> - metadataRepository.save( - createHousekeepingMetadata( - tables.get(i), PARTITION_PATHS.get(i), PARTITION_NAME, SCHEDULED))); + IntStream + .range(0, tables.size()) + .forEach(i -> metadataRepository + .save(createHousekeepingMetadata(tables.get(i), PARTITION_PATHS.get(i), PARTITION_NAME, SCHEDULED))); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(3)) - .dropPartition(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(3)).dropPartition(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0), tables.get(1), tables.get(2)); @@ -213,13 +191,10 @@ public void typicalPartitioned() { .extracting("path") .containsExactly(PARTITION_PATHS.get(0), PARTITION_PATHS.get(1), PARTITION_PATHS.get(2)); - metadataRepository - .findAll() - .forEach( - housekeepingMetadata -> { - assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); - assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); - }); + metadataRepository.findAll().forEach(housekeepingMetadata -> { + assertThat(housekeepingMetadata.getCleanupAttempts()).isEqualTo(1); + assertThat(housekeepingMetadata.getHousekeepingStatus()).isEqualTo(DELETED); + }); pagingCleanupService.cleanUp(Instant.now()); verifyNoMoreInteractions(pathCleaner); @@ -227,15 +202,13 @@ public void typicalPartitioned() { @Test public void mixOfScheduledAndFailedPaths() { - List tables = - List.of( - createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), + List tables = List + .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED)); tables.forEach(table -> metadataRepository.save(table)); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(2)) - .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(2)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0).getTableName(), tables.get(1).getTableName()); @@ -247,9 +220,8 @@ public void mixOfScheduledAndFailedPaths() { @Test public void mixOfAllPaths() { - List tables = - List.of( - createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), + List tables =List + .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED), createHousekeepingMetadata("table3", "s3://bucket/some_foobar", null, DELETED), createHousekeepingMetadata("table4", "s3://bucket/some_foobar", null, DISABLED), @@ -258,8 +230,7 @@ public void mixOfAllPaths() { tables.forEach(path -> metadataRepository.save(path)); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(2)) - .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(2)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0).getTableName(), tables.get(1).getTableName()); @@ -271,21 +242,20 @@ public void mixOfAllPaths() { @Test public void metadataCleanerException() { - Mockito.doNothing() + Mockito + .doNothing() .doThrow(new RuntimeException("Error")) .when(metadataCleaner) .dropTable(Mockito.any(HousekeepingMetadata.class), Mockito.any(HiveClient.class)); - List tables = - List.of( - createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), + List tables = List + .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, SCHEDULED)); tables.forEach(table -> metadataRepository.save(table)); pagingCleanupService.cleanUp(Instant.now()); - verify(metadataCleaner, times(2)) - .dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); + verify(metadataCleaner, times(2)).dropTable(metadataCaptor.capture(), hiveClientCaptor.capture()); assertThat(metadataCaptor.getAllValues()) .extracting("tableName") .containsExactly(tables.get(0).getTableName(), tables.get(1).getTableName()); @@ -307,51 +277,41 @@ public void metadataCleanerException() { @Test public void invalidPaths() { - List tables = - List.of( - createHousekeepingMetadata("table1", "s3://invalid", null, SCHEDULED), + List tables = List + .of(createHousekeepingMetadata("table1", "s3://invalid", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://invalid/path", "partition", SCHEDULED)); metadataRepository.saveAll(tables); pagingCleanupService.cleanUp(Instant.now()); - metadataRepository - .findAll() - .forEach( - table -> { - assertThat(table.getCleanupAttempts()).isEqualTo(0); - assertThat(table.getHousekeepingStatus()).isEqualTo(SKIPPED); - }); + metadataRepository.findAll().forEach(table -> { + assertThat(table.getCleanupAttempts()).isEqualTo(0); + assertThat(table.getHousekeepingStatus()).isEqualTo(SKIPPED); + }); } @Test @Timeout(value = 10) void doNotInfiniteLoopOnRepeatedFailures() { - List tables = - List.of( + List tables = List + .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, FAILED), createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, FAILED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED), createHousekeepingMetadata("table3", "s3://bucket/some_foobar", null, FAILED)); - doThrow(new RuntimeException("Error")) - .when(metadataCleaner) - .dropTable(Mockito.any(), Mockito.any()); + doThrow(new RuntimeException("Error")).when(metadataCleaner).dropTable(Mockito.any(), Mockito.any()); for (int i = 0; i < 5; i++) { int finalI = i; - tables.forEach( - path -> { - if (finalI == 0) { - metadataRepository.save(path); - } - }); + tables.forEach(path -> { + if (finalI == 0) { + metadataRepository.save(path); + } + }); pagingCleanupService.cleanUp(Instant.now()); - metadataRepository - .findAll() - .forEach( - table -> { - assertThat(table.getCleanupAttempts()).isEqualTo(finalI + 1); - assertThat(table.getHousekeepingStatus()).isEqualTo(FAILED); - }); + metadataRepository.findAll().forEach(table -> { + assertThat(table.getCleanupAttempts()).isEqualTo(finalI + 1); + assertThat(table.getHousekeepingStatus()).isEqualTo(FAILED); + }); } } @@ -360,37 +320,33 @@ void doNotInfiniteLoopOnRepeatedFailures() { void doNotInfiniteLoopOnDryRunCleanup() { pagingCleanupService = new PagingMetadataCleanupService(handlers, 2, true); - List tables = - List.of( - createHousekeepingMetadata("table1", "s3://some_foo", null, SCHEDULED), + List tables = List + .of(createHousekeepingMetadata("table1", "s3://some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://some_foo", null, SCHEDULED), createHousekeepingMetadata("table3", "s3://some_foo", null, SCHEDULED)); metadataRepository.saveAll(tables); pagingCleanupService.cleanUp(Instant.now()); - metadataRepository - .findAll() - .forEach( - table -> { - assertThat(table.getCleanupAttempts()).isEqualTo(0); - assertThat(table.getHousekeepingStatus()).isEqualTo(SCHEDULED); - }); + metadataRepository.findAll().forEach(table -> { + assertThat(table.getCleanupAttempts()).isEqualTo(0); + assertThat(table.getHousekeepingStatus()).isEqualTo(SCHEDULED); + }); } private HousekeepingMetadata createHousekeepingMetadata( String tableName, String path, String partitionName, HousekeepingStatus housekeepingStatus) { - HousekeepingMetadata metadata = - HousekeepingMetadata.builder() - .path(path) - .databaseName("database") - .tableName(tableName) - .partitionName(partitionName) - .housekeepingStatus(housekeepingStatus) - .creationTimestamp(localNow) - .modifiedTimestamp(localNow) - .cleanupDelay(PeriodDuration.of(Duration.parse("P30D"))) - .cleanupAttempts(0) - .lifecycleType(EXPIRED.toString()) - .build(); + HousekeepingMetadata metadata = HousekeepingMetadata + .builder() + .path(path) + .databaseName("database") + .tableName(tableName) + .partitionName(partitionName) + .housekeepingStatus(housekeepingStatus) + .creationTimestamp(localNow) + .modifiedTimestamp(localNow) + .cleanupDelay(PeriodDuration.of(Duration.parse("P30D"))) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); metadata.setCleanupTimestamp(localNow); return metadata; diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java index 6a3afe68..1cb68316 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/TestApplication.java @@ -26,8 +26,8 @@ @SpringBootApplication @EnableConfigurationProperties @ComponentScan("com.expediagroup.beekeeper.core") -@EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) -@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core.model" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) public class TestApplication { @PostConstruct diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java index 5d1ec551..1f93e896 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java @@ -66,20 +66,16 @@ import com.hotels.hcommon.hive.metastore.client.supplier.HiveMetaStoreClientSupplier; @Configuration -@ComponentScan( - basePackages = {"com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.scheduler"}) +@ComponentScan(basePackages = {"com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.scheduler"}) @EntityScan(basePackages = {"com.expediagroup.beekeeper.core"}) @EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) @EnableRetry(proxyTargetClass = true) public class CommonBeans { @Bean - public EnumMap schedulerServiceMap( - List schedulerServices) { - EnumMap schedulerMap = - new EnumMap<>(LifecycleEventType.class); - schedulerServices.forEach( - scheduler -> schedulerMap.put(scheduler.getLifecycleEventType(), scheduler)); + public EnumMap schedulerServiceMap(List schedulerServices) { + EnumMap schedulerMap = new EnumMap<>(LifecycleEventType.class); + schedulerServices.forEach(scheduler -> schedulerMap.put(scheduler.getLifecycleEventType(), scheduler)); return schedulerMap; } @@ -100,8 +96,7 @@ public MessageReader messageReader( } @Bean(name = "retryingMessageReader") - public MessageReader retryingMessageReader( - @Qualifier("sqsMessageReader") MessageReader messageReader) { + public MessageReader retryingMessageReader(@Qualifier("sqsMessageReader") MessageReader messageReader) { return new RetryingMessageReader(messageReader); } @@ -114,20 +109,20 @@ public HousekeepingEntityGenerator unreferencedHousekeepingPathGenerator( @Bean(name = "unreferencedHousekeepingPathMessageEventHandler") public MessageEventHandler unreferencedHousekeepingPathMessageEventHandler( @Qualifier("unreferencedHousekeepingPathGenerator") HousekeepingEntityGenerator generator) { - List> eventClasses = - List.of( - AlterPartitionEvent.class, - AlterTableEvent.class, - DropPartitionEvent.class, - DropTableEvent.class); - - List filters = - List.of( - new EventTypeListenerEventFilter(eventClasses), - new LocationOnlyUpdateListenerEventFilter(), - new TableParameterListenerEventFilter(), - new WhitelistedListenerEventFilter(), - new IcebergTableListenerEventFilter()); + List> eventClasses = List.of( + AlterPartitionEvent.class, + AlterTableEvent.class, + DropPartitionEvent.class, + DropTableEvent.class + ); + + List filters = List.of( + new EventTypeListenerEventFilter(eventClasses), + new LocationOnlyUpdateListenerEventFilter(), + new TableParameterListenerEventFilter(), + new WhitelistedListenerEventFilter(), + new IcebergTableListenerEventFilter() + ); return new MessageEventHandler(generator, filters); } @@ -142,18 +137,18 @@ public HousekeepingEntityGenerator expiredHousekeepingMetadataGenerator( @Bean(name = "expiredHousekeepingMetadataMessageEventHandler") public MessageEventHandler expiredHousekeepingMetadataMessageEventHandler( @Qualifier("expiredHousekeepingMetadataGenerator") HousekeepingEntityGenerator generator) { - List> eventClasses = - List.of( - CreateTableEvent.class, - AlterTableEvent.class, - AddPartitionEvent.class, - AlterPartitionEvent.class); - - List filters = - List.of( - new EventTypeListenerEventFilter(eventClasses), - new TableParameterListenerEventFilter(), - new IcebergTableListenerEventFilter()); + List> eventClasses = List.of( + CreateTableEvent.class, + AlterTableEvent.class, + AddPartitionEvent.class, + AlterPartitionEvent.class + ); + + List filters = List.of( + new EventTypeListenerEventFilter(eventClasses), + new TableParameterListenerEventFilter(), + new IcebergTableListenerEventFilter() + ); return new MessageEventHandler(generator, filters); } @@ -161,21 +156,18 @@ public MessageEventHandler expiredHousekeepingMetadataMessageEventHandler( @Bean public BeekeeperEventReader eventReader( @Qualifier("retryingMessageReader") MessageReader messageReader, - @Qualifier("unreferencedHousekeepingPathMessageEventHandler") - MessageEventHandler unreferencedHousekeepingPathMessageEventHandler, - @Qualifier("expiredHousekeepingMetadataMessageEventHandler") - MessageEventHandler expiredHousekeepingMetadataMessageEventHandler) { - List handlers = - List.of( - unreferencedHousekeepingPathMessageEventHandler, - expiredHousekeepingMetadataMessageEventHandler); + @Qualifier("unreferencedHousekeepingPathMessageEventHandler") MessageEventHandler unreferencedHousekeepingPathMessageEventHandler, + @Qualifier("expiredHousekeepingMetadataMessageEventHandler") MessageEventHandler expiredHousekeepingMetadataMessageEventHandler) { + List handlers = List.of( + unreferencedHousekeepingPathMessageEventHandler, + expiredHousekeepingMetadataMessageEventHandler + ); return new MessageReaderAdapter(messageReader, handlers); } @Bean - BeekeeperHistoryService beekeeperHistoryService( - BeekeeperHistoryRepository beekeeperHistoryRepository) { + BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { return new BeekeeperHistoryService(beekeeperHistoryRepository); } @@ -192,8 +184,8 @@ public CloseableMetaStoreClientFactory metaStoreClientFactory() { } @Bean - Supplier metaStoreClientSupplier( - CloseableMetaStoreClientFactory metaStoreClientFactory, HiveConf hiveConf) { + Supplier metaStoreClientSupplier(CloseableMetaStoreClientFactory metaStoreClientFactory, + HiveConf hiveConf) { String name = "beekeeper-scheduler"; return new HiveMetaStoreClientSupplier(metaStoreClientFactory, hiveConf, name); } @@ -204,8 +196,7 @@ public PartitionIteratorFactory partitionIteratorFactory() { } @Bean(name = "hiveClientFactory") - public HiveClientFactory clientFactory( - Supplier metaStoreClientSupplier, + public HiveClientFactory clientFactory(Supplier metaStoreClientSupplier, PartitionIteratorFactory partitionIteratorFactory) { return new HiveClientFactory(metaStoreClientSupplier, partitionIteratorFactory); } diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java index 7bd923d2..aef53bb0 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java @@ -78,8 +78,7 @@ void setUp() { @Test public void validateSchedulerServiceMap() { - EnumMap scheduleMap = - commonBeans.schedulerServiceMap(Collections.EMPTY_LIST); + EnumMap scheduleMap = commonBeans.schedulerServiceMap(Collections.EMPTY_LIST); assertThat(scheduleMap).isInstanceOf(EnumMap.class); } @@ -104,58 +103,51 @@ public void validateUnreferencedHousekeepingPathGenerator() { @Test public void validateUnreferencedHousekeepingPathMessageEventHandler() { - MessageEventHandler handler = - commonBeans.unreferencedHousekeepingPathMessageEventHandler( - unreferencedHousekeepingPathGenerator); + MessageEventHandler handler = commonBeans.unreferencedHousekeepingPathMessageEventHandler( + unreferencedHousekeepingPathGenerator); assertThat(handler).isInstanceOf(MessageEventHandler.class); } @Test public void validateExpiredHousekeepingMetadataGenerator() { HiveClientFactory mockHiveClientFactory = mock(HiveClientFactory.class); - HousekeepingEntityGenerator generator = - commonBeans.expiredHousekeepingMetadataGenerator("P30D", mockHiveClientFactory); + HousekeepingEntityGenerator generator = commonBeans.expiredHousekeepingMetadataGenerator("P30D", mockHiveClientFactory); assertThat(generator).isInstanceOf(ExpiredHousekeepingMetadataGenerator.class); } @Test public void validateExpiredHousekeepingMetadataMessageEventHandler() { - MessageEventHandler handler = - commonBeans.expiredHousekeepingMetadataMessageEventHandler( - expiredHousekeepingMetadataGenerator); + MessageEventHandler handler = commonBeans.expiredHousekeepingMetadataMessageEventHandler( + expiredHousekeepingMetadataGenerator); assertThat(handler).isInstanceOf(MessageEventHandler.class); } @Test public void validatePathEventReader() { - BeekeeperEventReader reader = - commonBeans.eventReader( - messageReader, mock(MessageEventHandler.class), mock(MessageEventHandler.class)); + BeekeeperEventReader reader = commonBeans.eventReader(messageReader, mock(MessageEventHandler.class), + mock(MessageEventHandler.class)); assertThat(reader).isInstanceOf(BeekeeperEventReader.class); } @Test public void validateUnreferencedHousekeepingPathMessageEventHandlerIncludesIcebergFilter() { - MessageEventHandler handler = - commonBeans.unreferencedHousekeepingPathMessageEventHandler( - unreferencedHousekeepingPathGenerator); + MessageEventHandler handler = commonBeans.unreferencedHousekeepingPathMessageEventHandler( + unreferencedHousekeepingPathGenerator); List filters = handler.getFilters(); assertThat(filters).hasAtLeastOneElementOfType(IcebergTableListenerEventFilter.class); } @Test public void validateExpiredHousekeepingMetadataMessageEventHandlerIncludesIcebergFilter() { - MessageEventHandler handler = - commonBeans.expiredHousekeepingMetadataMessageEventHandler( - expiredHousekeepingMetadataGenerator); + MessageEventHandler handler = commonBeans.expiredHousekeepingMetadataMessageEventHandler( + expiredHousekeepingMetadataGenerator); List filters = handler.getFilters(); assertThat(filters).hasAtLeastOneElementOfType(IcebergTableListenerEventFilter.class); } @Test public void verifyBeekeeperHistoryService() { - BeekeeperHistoryService beekeeperHistoryService = - commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); } @@ -164,8 +156,8 @@ public void verifyMetaStoreClientSupplier() { CloseableMetaStoreClientFactory metaStoreClientFactory = commonBeans.metaStoreClientFactory(); HiveConf hiveConf = Mockito.mock(HiveConf.class); - Supplier metaStoreClientSupplier = - commonBeans.metaStoreClientSupplier(metaStoreClientFactory, hiveConf); + Supplier metaStoreClientSupplier = commonBeans + .metaStoreClientSupplier(metaStoreClientFactory, hiveConf); assertThat(metaStoreClientSupplier).isInstanceOf(HiveMetaStoreClientSupplier.class); } } diff --git a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java index 4af7f3a3..a64f832e 100644 --- a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java +++ b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java @@ -36,11 +36,8 @@ @Configuration @EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) -@EnableJpaRepositories( - basePackages = { - "com.expediagroup.beekeeper.core.repository", - "com.expediagroup.beekeeper.vacuum.repository" - }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository", + "com.expediagroup.beekeeper.vacuum.repository" }) public class CommonBeans { @Bean @@ -57,12 +54,9 @@ public HiveConf hiveConf(@Value("${metastore-uri}") String metastoreUri) { conf.set("fs.s3.impl", S3AFileSystem.class.getName()); conf.set("fs.s3a.impl", S3AFileSystem.class.getName()); conf.set("fs.s3n.impl", S3AFileSystem.class.getName()); - conf.set( - "fs.s3.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); - conf.set( - "fs.s3a.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); - conf.set( - "fs.s3n.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); + conf.set("fs.s3.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); + conf.set("fs.s3a.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); + conf.set("fs.s3n.aws.credentials.provider", EC2ContainerCredentialsProviderWrapper.class.getName()); // Disable FileSystem cache to avoid UserGroupInformation.getCurrentUser() which is // incompatible with Java 17+ when the security manager is not installed @@ -81,21 +75,19 @@ public CloseableMetaStoreClientFactory metaStoreClientFactory() { @Bean Supplier metaStoreClientSupplier( - CloseableMetaStoreClientFactory metaStoreClientFactory, HiveConf hiveConf) { + CloseableMetaStoreClientFactory metaStoreClientFactory, + HiveConf hiveConf) { String name = "beekeeper-vacuum-tool"; return new HiveMetaStoreClientSupplier(metaStoreClientFactory, hiveConf, name); } @Bean - public BeekeeperHistoryService beekeeperHistoryService( - BeekeeperEventsHistoryRepository repository) { + public BeekeeperHistoryService beekeeperHistoryService(BeekeeperEventsHistoryRepository repository) { return new BeekeeperHistoryService(repository); } @Bean - public SchedulerService schedulerService( - BeekeeperRepository beekeeperRepository, BeekeeperHistoryService beekeeperHistoryService) { - return new UnreferencedHousekeepingPathSchedulerService( - beekeeperRepository, beekeeperHistoryService); + public SchedulerService schedulerService(BeekeeperRepository beekeeperRepository, BeekeeperHistoryService beekeeperHistoryService) { + return new UnreferencedHousekeepingPathSchedulerService(beekeeperRepository, beekeeperHistoryService); } } diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java index 70a05977..2d56ae2e 100644 --- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java +++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/ConsistencyCheckTest.java @@ -28,8 +28,7 @@ class ConsistencyCheckTest { @Test void metastorePathsCorrect() { - ConsistencyCheck.checkMetastorePaths( - Collections.singleton(new Path("/db/table/snapshot/partition")), 4); + ConsistencyCheck.checkMetastorePaths(Collections.singleton(new Path("/db/table/snapshot/partition")), 4); } @Test @@ -49,14 +48,13 @@ void metastorePathCorrect() { @Test void metastorePathDepthIncorrect() { assertThatExceptionOfType(IllegalStateException.class) - .isThrownBy( - () -> ConsistencyCheck.checkMetastorePath(new Path("/db/snapshot/partition"), 4)); + .isThrownBy(() -> ConsistencyCheck.checkMetastorePath( + new Path("/db/snapshot/partition"), 4)); } @Test void unvisitedPath() throws IOException { - Path nonExistent = - new Path("/db/table/snapshot/" + RandomStringUtils.randomAlphanumeric(8) + "/partition"); + Path nonExistent = new Path("/db/table/snapshot/" + RandomStringUtils.randomAlphanumeric(8) + "/partition"); Configuration conf = new Configuration(false); conf.setBoolean("fs.file.impl.disable.cache", true); FileSystem fs = nonExistent.getFileSystem(conf); From e3337d5e0287169da9a2b5a3b9e443290dee0172 Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Fri, 27 Mar 2026 09:23:31 -0500 Subject: [PATCH 08/12] Fix integration tests error --- .../BeekeeperDryRunMetadataCleanupIntegrationTest.java | 2 +- .../BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java index ef609ef2..f1a3179a 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java @@ -121,7 +121,7 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte private final ExecutorService executorService = Executors.newFixedThreadPool(1); private final TestAppender appender = new TestAppender(); - private static Map metastoreProperties = ImmutableMap + private Map metastoreProperties = ImmutableMap .builder() .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) .put(ACCESS_KEY, S3_ACCESS_KEY) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index f78e973b..98e3ddb3 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -122,7 +122,7 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek protected static AmazonS3 amazonS3; - private static Map metastoreProperties = ImmutableMap + private Map metastoreProperties = ImmutableMap .builder() .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) .put(ACCESS_KEY, S3_ACCESS_KEY) From 6da37a9c7d591773b16789f1b32b530e3d8f0b11 Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Fri, 27 Mar 2026 14:31:54 -0500 Subject: [PATCH 09/12] fix: upgrade spotbugs-maven-plugin to 4.8.6.6 for Java 21 compatibility SpotBugs 4.0.4 (inherited from eg-oss-parent) uses an outdated ASM library that cannot parse Java 21 class files (major version 65), causing the CI build to fail with NoClassesFoundToAnalyzeException. Overriding maven.spotbugs.plugin.version to 4.8.6.6 resolves the issue. Co-Authored-By: Claude Sonnet 4.6 --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index d30f1e65..c0b480da 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ 1.4.2 3.4.4 2.43.0 + 4.8.6.6 From 414896ff8ad8595e9f8c794bfe24652f7b2eb1f0 Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Tue, 7 Apr 2026 14:03:21 -0500 Subject: [PATCH 10/12] Clean up --- ...rDryRunMetadataCleanupIntegrationTest.java | 6 +- ...ekeeperMetadataCleanupIntegrationTest.java | 169 ++++++------------ .../BeekeeperPathCleanupIntegrationTest.java | 11 +- ...cedPathSchedulerApiaryIntegrationTest.java | 10 +- .../api/BeekeeperApiIntegrationTest.java | 7 +- .../PagingMetadataCleanupServiceTest.java | 12 +- .../scheduler/apiary/context/CommonBeans.java | 6 +- 7 files changed, 83 insertions(+), 138 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java index f1a3179a..55ecf156 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java @@ -270,10 +270,8 @@ private void assertMetrics() { meterRegistry.forEach(registry -> { List meters = registry.getMeters(); assertThat(meters).extracting("id", Meter.Id.class).extracting("name") - .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + -DeletedMetadataReporter.DRY_RUN_METRIC_NAME, - "hive-partition-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, "s3-paths-deleted", "s3-" + -BytesDeletedReporter.DRY_RUN_METRIC_NAME); + .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, + "hive-partition-" + DeletedMetadataReporter.DRY_RUN_METRIC_NAME, "s3-paths-deleted", "s3-" + BytesDeletedReporter.DRY_RUN_METRIC_NAME); }); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index 2944a72f..127213b1 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -89,10 +89,8 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio protected static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; protected static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; protected static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; - protected static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = - "com.amazonaws.services.s3.disableGetObjectMD5Validation"; - protected static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = - "com.amazonaws.services.s3.disablePutObjectMD5Validation"; + protected static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disableGetObjectMD5Validation"; + protected static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disablePutObjectMD5Validation"; protected static final String S3_ACCESS_KEY = "access"; protected static final String S3_SECRET_KEY = "secret"; @@ -107,21 +105,24 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio protected static final String ROOT_PATH = "s3a://" + BUCKET + "/" + DATABASE_NAME_VALUE + "/"; - protected static final String PARTITIONED_TABLE_PATH = - ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; + protected static final String PARTITIONED_TABLE_PATH = ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; protected static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1"; - protected static final String PARTITION_PATH = - PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; - protected static final String PARTITIONED_TABLE_OBJECT_KEY = - DATABASE_NAME_VALUE + "/" + PARTITIONED_TABLE_NAME + "/id1"; - - protected static final String PARTITIONED_OBJECT_KEY = - DATABASE_NAME_VALUE + "/some_location/id1/" + PARTITION_NAME + "/file1"; - - protected static final String UNPARTITIONED_TABLE_PATH = - ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; - protected static final String UNPARTITIONED_OBJECT_KEY = - DATABASE_NAME_VALUE + "/" + UNPARTITIONED_TABLE_NAME + "/id1/file1"; + protected static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; + protected static final String PARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE + + "/" + + PARTITIONED_TABLE_NAME + + "/id1"; + + protected static final String PARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE + + "/some_location/id1/" + + PARTITION_NAME + + "/file1"; + + protected static final String UNPARTITIONED_TABLE_PATH = ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; + protected static final String UNPARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE + + "/" + + UNPARTITIONED_TABLE_NAME + + "/id1/file1"; @Container protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); @@ -129,16 +130,16 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio protected static AmazonS3 amazonS3; protected final ExecutorService executorService = Executors.newFixedThreadPool(1); - private Map metastoreProperties = - ImmutableMap.builder() - .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) - .put(ACCESS_KEY, S3_ACCESS_KEY) - .put(SECRET_KEY, S3_SECRET_KEY) - .build(); + private Map metastoreProperties = ImmutableMap + .builder() + .put(ENDPOINT, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)) + .put(ACCESS_KEY, S3_ACCESS_KEY) + .put(SECRET_KEY, S3_SECRET_KEY) + .build(); @RegisterExtension - public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = - new ThriftHiveMetaStoreJUnitExtension(DATABASE_NAME_VALUE, metastoreProperties); + public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( + DATABASE_NAME_VALUE, metastoreProperties); protected HiveTestUtils hiveTestUtils; protected HiveMetaStoreClient metastoreClient; @@ -148,8 +149,7 @@ public static void init() { System.setProperty(SPRING_PROFILES_ACTIVE_PROPERTY, "test"); System.setProperty(SCHEDULER_DELAY_MS_PROPERTY, SCHEDULER_DELAY_MS); System.setProperty(DRY_RUN_ENABLED_PROPERTY, "false"); - System.setProperty( - AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); + System.setProperty(AWS_S3_ENDPOINT_PROPERTY, ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3)); System.setProperty(AWS_DISABLE_GET_VALIDATION_PROPERTY, "true"); System.setProperty(AWS_DISABLE_PUT_VALIDATION_PROPERTY, "true"); @@ -192,12 +192,7 @@ public void stop() throws InterruptedException { @Test public void cleanupUnpartitionedTable() throws TException, SQLException { - hiveTestUtils.createTableWithProperties( - UNPARTITIONED_TABLE_PATH, - TABLE_NAME_VALUE, - false, - createBeeKeeperDeletionProperties(), - true); + hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, createBeeKeeperDeletionProperties(), true); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); @@ -212,12 +207,7 @@ public void cleanupUnpartitionedTable() throws TException, SQLException { @Test public void cleanupPartitionedTable() throws Exception { Table table = - hiveTestUtils.createTableWithProperties( - PARTITIONED_TABLE_PATH, - TABLE_NAME_VALUE, - true, - createBeeKeeperDeletionProperties(), - true); + hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE,true, createBeeKeeperDeletionProperties(),true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); @@ -258,12 +248,12 @@ public void cleanupPartitionButNotTable() throws Exception { Table table = hiveTestUtils.createTable(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); - hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); + hiveTestUtils + .addPartitionsToTable(PARTITION_ROOT_PATH, table, List.of("2020-01-01", "1", "B")); String partition2Name = "event_date=2020-01-01/event_hour=1/event_type=B"; String partition2Path = PARTITION_ROOT_PATH + "/" + partition2Name + "/file1"; - String partition2ObjectKey = - DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; + String partition2ObjectKey = DATABASE_NAME_VALUE + "/some_location/id1/" + partition2Name + "/file1"; amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -271,16 +261,14 @@ public void cleanupPartitionButNotTable() throws Exception { insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); insertExpiredMetadata(PARTITION_PATH, PARTITION_NAME); - insertExpiredMetadata( - TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); + insertExpiredMetadata(TABLE_NAME_VALUE, partition2Path, partition2Name, LONG_CLEANUP_DELAY_VALUE); await() .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getExpiredMetadata().get(0).getHousekeepingStatus() == DELETED); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); - List partitions = - metastoreClient.listPartitions(DATABASE_NAME_VALUE, TABLE_NAME_VALUE, (short) 1); + List partitions = metastoreClient.listPartitions(DATABASE_NAME_VALUE, TABLE_NAME_VALUE, (short) 1); assertEquals(partitions.size(), 1); assertEquals(partitions.get(0).getValues(), List.of("2020-01-01", "1", "B")); assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isTrue(); @@ -290,8 +278,7 @@ public void cleanupPartitionButNotTable() throws Exception { @Test public void cleanupPartitionedTableWithNoPartitions() throws TException, SQLException { - hiveTestUtils.createTableWithProperties( - PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); + hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(PARTITIONED_TABLE_PATH, null); @@ -349,38 +336,25 @@ public void disablePartitionedTable() throws Exception { @Test public void cleanupMultipleTablesOfMixedType() throws Exception { - hiveTestUtils.createTableWithProperties( - UNPARTITIONED_TABLE_PATH, - UNPARTITIONED_TABLE_NAME, - false, - createBeeKeeperDeletionProperties(), - true); - - Table partitionedTable = - hiveTestUtils.createTableWithProperties( - PARTITIONED_TABLE_PATH, - PARTITIONED_TABLE_NAME, - true, - createBeeKeeperDeletionProperties(), - true); - hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, partitionedTable, PARTITION_VALUES); + hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, UNPARTITIONED_TABLE_NAME,false, createBeeKeeperDeletionProperties(),true); + + Table partitionedTable = hiveTestUtils + .createTableWithProperties(PARTITIONED_TABLE_PATH, PARTITIONED_TABLE_NAME,true, createBeeKeeperDeletionProperties(),true); + hiveTestUtils + .addPartitionsToTable(PARTITION_ROOT_PATH, partitionedTable, PARTITION_VALUES); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, TABLE_DATA); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); - insertExpiredMetadata( - UNPARTITIONED_TABLE_NAME, UNPARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); - insertExpiredMetadata( - PARTITIONED_TABLE_NAME, PARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); - insertExpiredMetadata( - PARTITIONED_TABLE_NAME, PARTITION_PATH, PARTITION_NAME, SHORT_CLEANUP_DELAY_VALUE); + insertExpiredMetadata(UNPARTITIONED_TABLE_NAME, UNPARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); + insertExpiredMetadata(PARTITIONED_TABLE_NAME, PARTITIONED_TABLE_PATH, null, SHORT_CLEANUP_DELAY_VALUE); + insertExpiredMetadata(PARTITIONED_TABLE_NAME, PARTITION_PATH, PARTITION_NAME, SHORT_CLEANUP_DELAY_VALUE); await() .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getExpiredMetadata().get(1).getHousekeepingStatus() == DELETED); - assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, UNPARTITIONED_TABLE_NAME)) - .isFalse(); + assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, UNPARTITIONED_TABLE_NAME)).isFalse(); assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, PARTITIONED_TABLE_NAME)).isFalse(); assertThat(amazonS3.doesObjectExist(BUCKET, UNPARTITIONED_OBJECT_KEY)).isFalse(); assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isFalse(); @@ -396,8 +370,7 @@ public void onlyCleanupLocationWhenTableExists() throws SQLException { @Test public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLException { - hiveTestUtils.createTableWithProperties( - PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); + hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); amazonS3.putObject(BUCKET, PARTITIONED_OBJECT_KEY, TABLE_DATA); @@ -415,12 +388,7 @@ public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLExcep @Test public void testEventAddedToHistoryTable() throws TException, SQLException { - hiveTestUtils.createTableWithProperties( - UNPARTITIONED_TABLE_PATH, - TABLE_NAME_VALUE, - false, - createBeeKeeperDeletionProperties(), - true); + hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE,false, createBeeKeeperDeletionProperties(),true); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); @@ -444,8 +412,7 @@ public void tableNotDeletedWhenDeletionPropertyIsFalse() throws TException, SQLE Map tableProperties = new HashMap<>(); tableProperties.put("beekeeper.expired.data.table.deletion.enabled", "false"); - hiveTestUtils.createTableWithProperties( - UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, tableProperties, true); + hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, tableProperties, true); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); @@ -473,13 +440,7 @@ public void tableNotDeletedWhenDeletionPropertyNotSet() throws TException, SQLEx @Test public void metrics() throws Exception { - Table table = - hiveTestUtils.createTableWithProperties( - PARTITIONED_TABLE_PATH, - TABLE_NAME_VALUE, - true, - createBeeKeeperDeletionProperties(), - true); + Table table = hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE,true, createBeeKeeperDeletionProperties(),true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); @@ -498,24 +459,14 @@ public void metrics() throws Exception { } protected void assertMetrics() { - Set meterRegistry = - ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); + Set meterRegistry = ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); - meterRegistry.forEach( - registry -> { - List meters = registry.getMeters(); - assertThat(meters) - .extracting("id", Meter.Id.class) - .extracting("name") - .contains( - "metadata-cleanup-job", - "hive-table-deleted", - "hive-partition-deleted", - "hive-table-" + METRIC_NAME, - "hive-partition-" + METRIC_NAME, - "s3-paths-deleted", - "s3-" + BytesDeletedReporter.METRIC_NAME); - }); + meterRegistry.forEach(registry -> { + List meters = registry.getMeters(); + assertThat(meters).extracting("id", Meter.Id.class).extracting("name") + .contains("metadata-cleanup-job", "hive-table-deleted", "hive-partition-deleted", "hive-table-" + METRIC_NAME, + "hive-partition-" + METRIC_NAME, "s3-paths-deleted", "s3-" + BytesDeletedReporter.METRIC_NAME); + }); } private Map createBeeKeeperDeletionProperties() { @@ -528,8 +479,7 @@ private Map createBeeKeeperDeletionProperties() { public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(HEALTHCHECK_URI); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } @@ -537,8 +487,7 @@ public void healthCheck() { public void prometheus() { CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(PROMETHEUS_URI); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index 1a503114..88fa4b1c 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -115,7 +115,8 @@ public void setup() { .getObjectSummaries() .forEach(object -> amazonS3.deleteObject(BUCKET, object.getKey())); executorService.execute(() -> BeekeeperPathCleanup.main(new String[] {})); - await().atMost(Duration.ofMinutes(1)).until(BeekeeperPathCleanup::isRunning); + await().atMost(Duration.ofMinutes(1)) + .until(BeekeeperPathCleanup::isRunning); } @AfterEach @@ -172,8 +173,7 @@ public void cleanupPathsForDirectoryWithSpace() throws SQLException { amazonS3.putObject(BUCKET, objectKeySentinel, ""); insertUnreferencedPath(absolutePath); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, objectKey1)).isFalse(); @@ -234,8 +234,7 @@ public void cleanupSentinelForNonEmptyParent() throws SQLException { amazonS3.putObject(BUCKET, tableSentinel, ""); insertUnreferencedPath(ABSOLUTE_PATH); - await() - .atMost(TIMEOUT, TimeUnit.SECONDS) + await().atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> getUnreferencedPaths().get(0).getHousekeepingStatus() == DELETED); assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); @@ -290,7 +289,7 @@ private void assertMetrics() { List meters = registry.getMeters(); assertThat(meters).extracting("id", Meter.Id.class).extracting("name") .contains("path-cleanup-job", "s3-paths-deleted", "s3-" + METRIC_NAME); - }); + }); } @Test diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 2f4576ed..a7dc307e 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -158,7 +158,7 @@ public void unreferencedMultipleAlterTableEvents() throws SQLException, IOExcept public void unreferencedAlterPartitionEvent() throws SQLException, IOException, URISyntaxException { AlterPartitionSqsMessage alterPartitionSqsMessage = new AlterPartitionSqsMessage( "s3://bucket/table/expiredTableLocation", "s3://bucket/table/partitionLocation", - "s3://bucket/table/unreferencedPartitionLocation", true, true); + "s3://bucket/table/unreferencedPartitionLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterPartitionSqsMessage.getFormattedString())); alterPartitionSqsMessage.setTableLocation("s3://bucket/table/expiredTableLocation2"); alterPartitionSqsMessage.setPartitionLocation("s3://bucket/table/partitionLocation2"); @@ -176,9 +176,9 @@ public void unreferencedMultipleAlterPartitionEvent() throws IOException, SQLException, URISyntaxException { List .of(new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation", - "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), - new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation2", - "s3://bucket/table/partitionLocation2", "s3://bucket/table/partitionLocation", true, true)) + "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), + new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation2", + "s3://bucket/table/partitionLocation2", "s3://bucket/table/partitionLocation", true, true)) .forEach(msg -> amazonSQS.sendMessage(sendMessageRequest(msg.getFormattedString()))); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getUnreferencedPathsRowCount() == 2); @@ -218,7 +218,7 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy @Test public void testEventAddedToHistoryTable() throws IOException, URISyntaxException, SQLException { AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", - "s3://bucket/oldTableLocation", true, true); + "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 74183730..9c59c733 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -123,9 +123,8 @@ public void testGetMetadataWhenTableNotFoundReturnsEmptyList() HttpResponse response = testClient.getMetadata("wrong_database", "wrong_table"); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); - Page responsePage = - mapper.readValue( - body, new TypeReference>() {}); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); assertThat(responsePage.getTotalElements()).isEqualTo(0); } @@ -149,7 +148,7 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); Page responsePage = mapper - .readValue(body, new TypeReference>() {}); + .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThatMetadataEqualsResponse(testMetadataA, result.get(0)); diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java index 3a181b7e..6d3b4e94 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java @@ -71,9 +71,7 @@ @ExtendWith(SpringExtension.class) @ExtendWith(MockitoExtension.class) -@ContextConfiguration( - classes = {TestApplication.class}, - loader = AnnotationConfigContextLoader.class) +@ContextConfiguration(classes = {TestApplication.class}, loader = AnnotationConfigContextLoader.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) public class PagingMetadataCleanupServiceTest { @@ -220,7 +218,7 @@ public void mixOfScheduledAndFailedPaths() { @Test public void mixOfAllPaths() { - List tables =List + List tables = List .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, SCHEDULED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED), createHousekeepingMetadata("table3", "s3://bucket/some_foobar", null, DELETED), @@ -294,7 +292,6 @@ public void invalidPaths() { void doNotInfiniteLoopOnRepeatedFailures() { List tables = List .of(createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, FAILED), - createHousekeepingMetadata("table1", "s3://bucket/some_foo", null, FAILED), createHousekeepingMetadata("table2", "s3://bucket/some_bar", null, FAILED), createHousekeepingMetadata("table3", "s3://bucket/some_foobar", null, FAILED)); @@ -333,7 +330,10 @@ void doNotInfiniteLoopOnDryRunCleanup() { } private HousekeepingMetadata createHousekeepingMetadata( - String tableName, String path, String partitionName, HousekeepingStatus housekeepingStatus) { + String tableName, + String path, + String partitionName, + HousekeepingStatus housekeepingStatus) { HousekeepingMetadata metadata = HousekeepingMetadata .builder() .path(path) diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java index 1f93e896..01824679 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java @@ -66,9 +66,9 @@ import com.hotels.hcommon.hive.metastore.client.supplier.HiveMetaStoreClientSupplier; @Configuration -@ComponentScan(basePackages = {"com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.scheduler"}) -@EntityScan(basePackages = {"com.expediagroup.beekeeper.core"}) -@EnableJpaRepositories(basePackages = {"com.expediagroup.beekeeper.core.repository"}) +@ComponentScan(basePackages = { "com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.scheduler" }) +@EntityScan(basePackages = { "com.expediagroup.beekeeper.core" }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository" }) @EnableRetry(proxyTargetClass = true) public class CommonBeans { From df58fae27a161463c110a991c0a527d616adf893 Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Tue, 7 Apr 2026 14:33:25 -0500 Subject: [PATCH 11/12] Clean up --- .../BeekeeperMetadataCleanupIntegrationTest.java | 11 +++++------ ...nreferencedPathSchedulerApiaryIntegrationTest.java | 3 +-- .../beekeeper/vacuum/TestApplication.java | 7 ++----- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index 127213b1..98413aac 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -206,8 +206,7 @@ public void cleanupUnpartitionedTable() throws TException, SQLException { @Test public void cleanupPartitionedTable() throws Exception { - Table table = - hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE,true, createBeeKeeperDeletionProperties(),true); + Table table = hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE,true, createBeeKeeperDeletionProperties(),true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); @@ -336,10 +335,10 @@ public void disablePartitionedTable() throws Exception { @Test public void cleanupMultipleTablesOfMixedType() throws Exception { - hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, UNPARTITIONED_TABLE_NAME,false, createBeeKeeperDeletionProperties(),true); + hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, UNPARTITIONED_TABLE_NAME, false, createBeeKeeperDeletionProperties(), true); Table partitionedTable = hiveTestUtils - .createTableWithProperties(PARTITIONED_TABLE_PATH, PARTITIONED_TABLE_NAME,true, createBeeKeeperDeletionProperties(),true); + .createTableWithProperties(PARTITIONED_TABLE_PATH, PARTITIONED_TABLE_NAME, true, createBeeKeeperDeletionProperties(), true); hiveTestUtils .addPartitionsToTable(PARTITION_ROOT_PATH, partitionedTable, PARTITION_VALUES); @@ -388,7 +387,7 @@ public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLExcep @Test public void testEventAddedToHistoryTable() throws TException, SQLException { - hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE,false, createBeeKeeperDeletionProperties(),true); + hiveTestUtils.createTableWithProperties(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false, createBeeKeeperDeletionProperties(), true); amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); @@ -440,7 +439,7 @@ public void tableNotDeletedWhenDeletionPropertyNotSet() throws TException, SQLEx @Test public void metrics() throws Exception { - Table table = hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE,true, createBeeKeeperDeletionProperties(),true); + Table table = hiveTestUtils.createTableWithProperties(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, createBeeKeeperDeletionProperties(), true); hiveTestUtils.addPartitionsToTable(PARTITION_ROOT_PATH, table, PARTITION_VALUES); amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, ""); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index a7dc307e..eda37c91 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -172,8 +172,7 @@ public void unreferencedAlterPartitionEvent() throws SQLException, IOException, } @Test - public void unreferencedMultipleAlterPartitionEvent() - throws IOException, SQLException, URISyntaxException { + public void unreferencedMultipleAlterPartitionEvent() throws IOException, SQLException, URISyntaxException { List .of(new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation", "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java index 712f6d8e..3a5c9415 100644 --- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java +++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/TestApplication.java @@ -25,11 +25,8 @@ @SpringBootApplication @EnableConfigurationProperties @EntityScan(basePackages = {"com.expediagroup.beekeeper.core.model"}) -@EnableJpaRepositories( - basePackages = { - "com.expediagroup.beekeeper.core.repository", - "com.expediagroup.beekeeper.vacuum.repository" - }) +@EnableJpaRepositories(basePackages = { "com.expediagroup.beekeeper.core.repository", + "com.expediagroup.beekeeper.vacuum.repository" }) public class TestApplication { @PostConstruct From 5eaeed483433f8ee3d3a1ea34c10e706aad54e02 Mon Sep 17 00:00:00 2001 From: ninhomilton Date: Tue, 14 Apr 2026 12:59:26 -0500 Subject: [PATCH 12/12] refactor: centralise Java version via jdk.version property in root pom Remove hardcoded '21' from maven.compiler.source/target/release, docker.from.tag, and both maven-compiler-plugin configurations, replacing them with ${jdk.version}. Also remove unused springframework.version property now that direct Spring dependencies are managed by the spring-boot-dependencies BOM. Co-Authored-By: Claude Sonnet 4.6 --- pom.xml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index c0b480da..1e5971a3 100644 --- a/pom.xml +++ b/pom.xml @@ -44,18 +44,17 @@ 4.2.0 expediagroup 21 - 21 - 21 - 21 + ${jdk.version} + ${jdk.version} + ${jdk.version} 1.4.14 2.22.1 1.18.36 3.1.0 2.2 3.2.12 - 6.1.15 1.19.3 - 21-slim + ${jdk.version}-slim 2.16.1 2.3.7 1.4.2 @@ -187,9 +186,9 @@ maven-compiler-plugin 3.13.0 - 21 - 21 - 21 + ${jdk.version} + ${jdk.version} + ${jdk.version} UTF-8 true