From c7baca112ef1148203e97f607cc9a1b33a583c69 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Wed, 21 May 2025 23:33:47 +0100 Subject: [PATCH 01/10] update --- .../scheduler/apiary/context/CommonBeans.java | 5 +- .../ExpiredHousekeepingMetadataGenerator.java | 54 +++++++++++++++++-- ...iredHousekeepingMetadataGeneratorTest.java | 12 ++++- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java index bd9d6853..53893e35 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java @@ -116,8 +116,9 @@ public MessageEventHandler unreferencedHousekeepingPathMessageEventHandler( @Bean(name = "expiredHousekeepingMetadataGenerator") public HousekeepingEntityGenerator expiredHousekeepingMetadataGenerator( - @Value("${properties.beekeeper.default-expiration-delay}") String cleanupDelay) { - return new ExpiredHousekeepingMetadataGenerator(cleanupDelay); + @Value("${properties.beekeeper.default-expiration-delay}") String cleanupDelay, + @Qualifier("hiveClientFactory") HiveClientFactory hiveClientFactory) { + return new ExpiredHousekeepingMetadataGenerator(cleanupDelay, hiveClientFactory); } @Bean(name = "expiredHousekeepingMetadataMessageEventHandler") diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java index 159dc0d8..5d3c1661 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java @@ -26,6 +26,7 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -47,6 +48,9 @@ import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.scheduler.apiary.generator.utils.CleanupDelayExtractor; +import com.expediagroup.beekeeper.scheduler.hive.HiveClient; +import com.expediagroup.beekeeper.scheduler.hive.HiveClientFactory; +import com.expediagroup.beekeeper.scheduler.hive.PartitionInfo; public class ExpiredHousekeepingMetadataGenerator implements HousekeepingEntityGenerator { @@ -57,16 +61,19 @@ public class ExpiredHousekeepingMetadataGenerator implements HousekeepingEntityG private final CleanupDelayExtractor cleanupDelayExtractor; private final Clock clock; + private final HiveClientFactory hiveClientFactory; - public ExpiredHousekeepingMetadataGenerator(String cleanupDelay) { + public ExpiredHousekeepingMetadataGenerator(String cleanupDelay, HiveClientFactory hiveClientFactory) { this(new CleanupDelayExtractor(EXPIRED_DATA_RETENTION_PERIOD_PROPERTY_KEY, cleanupDelay), - Clock.systemDefaultZone()); + Clock.systemDefaultZone(), hiveClientFactory); } @VisibleForTesting - ExpiredHousekeepingMetadataGenerator(CleanupDelayExtractor cleanupDelayExtractor, Clock clock) { + ExpiredHousekeepingMetadataGenerator(CleanupDelayExtractor cleanupDelayExtractor, Clock clock, + HiveClientFactory hiveClientFactory) { this.cleanupDelayExtractor = cleanupDelayExtractor; this.clock = clock; + this.hiveClientFactory = hiveClientFactory; } @Override @@ -136,10 +143,23 @@ private HousekeepingEntity generateHousekeepingEntity( String path, String partitionName) { PeriodDuration cleanupDelay = cleanupDelayExtractor.extractCleanupDelay(listenerEvent); + + // Get the creation time from Hive if this is a partition + LocalDateTime creationTime; + if (partitionName != null) { + creationTime = getPartitionCreationTime(listenerEvent.getDbName(), listenerEvent.getTableName(), partitionName); + } else { + // For tables, use current time as there's no reliable way to get table creation time + creationTime = LocalDateTime.now(clock); + } + + log.info("TIMESTAMP_INFO: Creating housekeeping entity with creationTimestamp={} for partition={}", + creationTime, partitionName); + return HousekeepingMetadata .builder() .housekeepingStatus(SCHEDULED) - .creationTimestamp(LocalDateTime.now(clock)) + .creationTimestamp(creationTime) .cleanupDelay(cleanupDelay) .lifecycleType(LIFECYCLE_EVENT_TYPE.toString()) .clientId(clientId) @@ -163,4 +183,30 @@ private String generatePartitionName(List keys, List values) { .mapToObj(i -> keys.get(i) + "=" + values.get(i)) .collect(Collectors.joining("/")); } + + /** + * Gets the creation time of a partition from Hive. + * + * @param databaseName The database name + * @param tableName The table name + * @param partitionName The partition name + * @return The partition creation time from Hive, or current time if not available + */ + private LocalDateTime getPartitionCreationTime(String databaseName, String tableName, String partitionName) { + try (HiveClient hiveClient = hiveClientFactory.newInstance()) { + Map partitionInfo = hiveClient.getTablePartitionsInfo(databaseName, tableName); + + if (partitionInfo.containsKey(partitionName)) { + return partitionInfo.get(partitionName).getCreateTime(); + } + + log.warn("Partition {} not found in Hive for table {}.{}, using current time", + partitionName, databaseName, tableName); + return LocalDateTime.now(clock); + } catch (Exception e) { + log.warn("Failed to get partition creation time from Hive for {}.{}.{}, using current time", + databaseName, tableName, partitionName, e); + return LocalDateTime.now(clock); + } + } } diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java index 5041c910..4f33abc3 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.anyString; import static com.expedia.apiary.extensions.receiver.common.event.EventType.ADD_PARTITION; import static com.expedia.apiary.extensions.receiver.common.event.EventType.ALTER_PARTITION; @@ -32,6 +33,7 @@ import java.util.List; import java.util.Map; +import java.util.Collections; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -50,6 +52,9 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.scheduler.hive.HiveClient; +import com.expediagroup.beekeeper.scheduler.hive.HiveClientFactory; +import com.expediagroup.beekeeper.scheduler.hive.PartitionInfo; @ExtendWith(MockitoExtension.class) public class ExpiredHousekeepingMetadataGeneratorTest extends HousekeepingEntityGeneratorTestBase { @@ -65,11 +70,16 @@ public class ExpiredHousekeepingMetadataGeneratorTest extends HousekeepingEntity @Mock private AlterTableEvent alterTableEvent; @Mock private AddPartitionEvent addPartitionEvent; @Mock private AlterPartitionEvent alterPartitionEvent; + @Mock private HiveClientFactory hiveClientFactory; + @Mock private HiveClient hiveClient; private ExpiredHousekeepingMetadataGenerator generator; @BeforeEach public void setup() { - generator = new ExpiredHousekeepingMetadataGenerator(cleanupDelayExtractor, clock); + generator = new ExpiredHousekeepingMetadataGenerator(cleanupDelayExtractor, clock, hiveClientFactory); + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + Map emptyPartitionInfo = Collections.emptyMap(); + when(hiveClient.getTablePartitionsInfo(anyString(), anyString())).thenReturn(emptyPartitionInfo); } @Test From e048c10a77262e9d5fe48c2dccc8ace82f22d3d9 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Tue, 27 May 2025 12:23:58 +0100 Subject: [PATCH 02/10] Fix event-driven partition scheduling to use Hive creation time --- .../ExpiredHousekeepingMetadataGenerator.java | 6 +-- .../apiary/context/CommonBeansTest.java | 5 +- ...iredHousekeepingMetadataGeneratorTest.java | 30 ++++++++++-- .../beekeeper/scheduler/hive/HiveClient.java | 22 +++++++++ .../scheduler/hive/hive/HiveClientTest.java | 49 +++++++++++++++++++ 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java index 5d3c1661..d52fe112 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java @@ -194,10 +194,10 @@ private String generatePartitionName(List keys, List values) { */ private LocalDateTime getPartitionCreationTime(String databaseName, String tableName, String partitionName) { try (HiveClient hiveClient = hiveClientFactory.newInstance()) { - Map partitionInfo = hiveClient.getTablePartitionsInfo(databaseName, tableName); + PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(databaseName, tableName, partitionName); - if (partitionInfo.containsKey(partitionName)) { - return partitionInfo.get(partitionName).getCreateTime(); + if (partitionInfo != null) { + return partitionInfo.getCreateTime(); } log.warn("Partition {} not found in Hive for table {}.{}, using current time", diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java index ff1d5443..6211b5b5 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java @@ -46,6 +46,8 @@ import com.expediagroup.beekeeper.scheduler.apiary.handler.MessageEventHandler; import com.expediagroup.beekeeper.scheduler.apiary.messaging.BeekeeperEventReader; import com.expediagroup.beekeeper.scheduler.apiary.messaging.RetryingMessageReader; +import com.expediagroup.beekeeper.scheduler.hive.PartitionIteratorFactory; +import com.expediagroup.beekeeper.scheduler.hive.HiveClientFactory; import com.expediagroup.beekeeper.scheduler.service.SchedulerService; import com.hotels.hcommon.hive.metastore.client.api.CloseableMetaStoreClient; @@ -110,7 +112,8 @@ public void validateUnreferencedHousekeepingPathMessageEventHandler() { @Test public void validateExpiredHousekeepingMetadataGenerator() { - HousekeepingEntityGenerator generator = commonBeans.expiredHousekeepingMetadataGenerator("P30D"); + HiveClientFactory mockHiveClientFactory = mock(HiveClientFactory.class); + HousekeepingEntityGenerator generator = commonBeans.expiredHousekeepingMetadataGenerator("P30D", mockHiveClientFactory); assertThat(generator).isInstanceOf(ExpiredHousekeepingMetadataGenerator.class); } diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java index 4f33abc3..bb987442 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java @@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.anyString; import static com.expedia.apiary.extensions.receiver.common.event.EventType.ADD_PARTITION; import static com.expedia.apiary.extensions.receiver.common.event.EventType.ALTER_PARTITION; @@ -31,9 +30,9 @@ import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; +import java.time.LocalDateTime; import java.util.List; import java.util.Map; -import java.util.Collections; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -77,9 +76,6 @@ public class ExpiredHousekeepingMetadataGeneratorTest extends HousekeepingEntity @BeforeEach public void setup() { generator = new ExpiredHousekeepingMetadataGenerator(cleanupDelayExtractor, clock, hiveClientFactory); - when(hiveClientFactory.newInstance()).thenReturn(hiveClient); - Map emptyPartitionInfo = Collections.emptyMap(); - when(hiveClient.getTablePartitionsInfo(anyString(), anyString())).thenReturn(emptyPartitionInfo); } @Test @@ -111,6 +107,8 @@ public void typicalHandleAddPartitionEvent() { when(addPartitionEvent.getPartitionLocation()).thenReturn(PARTITION_PATH); when(addPartitionEvent.getPartitionKeys()).thenReturn(PARTITION_KEYS); when(addPartitionEvent.getPartitionValues()).thenReturn(PARTITION_VALUES); + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(null); List housekeepingEntities = generator.generate(addPartitionEvent, CLIENT_ID); assertThat(housekeepingEntities.size()).isEqualTo(1); @@ -124,6 +122,8 @@ public void typicalHandleAlterPartitionEvent() { when(alterPartitionEvent.getPartitionLocation()).thenReturn(PARTITION_PATH); when(alterPartitionEvent.getPartitionKeys()).thenReturn(PARTITION_KEYS); when(alterPartitionEvent.getPartitionValues()).thenReturn(PARTITION_VALUES); + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(null); List housekeepingEntities = generator.generate(alterPartitionEvent, CLIENT_ID); assertThat(housekeepingEntities.size()).isEqualTo(1); @@ -152,6 +152,26 @@ public void exceptionThrownOnUnhandledEvent() { } } + @Test + public void usesPartitionCreationTimeFromHive() { + setupListenerEvent(addPartitionEvent, ADD_PARTITION); + when(cleanupDelayExtractor.extractCleanupDelay(addPartitionEvent)).thenReturn(CLEANUP_DELAY); + when(addPartitionEvent.getPartitionLocation()).thenReturn(PARTITION_PATH); + when(addPartitionEvent.getPartitionKeys()).thenReturn(PARTITION_KEYS); + when(addPartitionEvent.getPartitionValues()).thenReturn(PARTITION_VALUES); + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + LocalDateTime hiveCreationTime = LocalDateTime.of(2023, 6, 15, 10, 30); + PartitionInfo partitionInfo = new PartitionInfo(PARTITION_PATH, hiveCreationTime); + when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(partitionInfo); + + List housekeepingEntities = generator.generate(addPartitionEvent, CLIENT_ID); + + assertThat(housekeepingEntities.size()).isEqualTo(1); + HousekeepingMetadata metadata = (HousekeepingMetadata) housekeepingEntities.get(0); + assertThat(metadata.getCreationTimestamp()).isEqualTo(hiveCreationTime); + assertThat(metadata.getPartitionName()).isEqualTo(PARTITION_NAME); + } + private void assertExpiredHousekeepingMetadataEntity(HousekeepingEntity housekeepingEntity, String partitionName) { HousekeepingMetadata housekeepingMetadata = (HousekeepingMetadata) housekeepingEntity; assertHousekeepingEntity(housekeepingMetadata, EXPIRED); diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java index b8541909..af8b3010 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java @@ -75,6 +75,28 @@ public Map getTablePartitionsInfo(String databaseName, St } } + public PartitionInfo getSinglePartitionInfo(String databaseName, String tableName, String partitionName) { + try { + Table table = metaStoreClient.getTable(databaseName, tableName); + List partitionKeys = table.getPartitionKeys(); + List partitionValues = Warehouse.getPartValuesFromPartName(partitionName); + + Partition partition = metaStoreClient.getPartition(databaseName, tableName, partitionValues); + + String path = partition.getSd().getLocation(); + LocalDateTime createTime = extractCreateTime(partition); + + log.debug("Retrieved partition '{}' with path '{}' for table {}.{}", + partitionName, path, databaseName, tableName); + + return new PartitionInfo(path, createTime); + } catch (TException e) { + log.warn("Failed to get partition info for {}.{}.{}: {}", + databaseName, tableName, partitionName, e.getMessage()); + return null; + } + } + private LocalDateTime extractCreateTime(Partition partition) { if (partition.getCreateTime() > 0) { return LocalDateTime.ofInstant( diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java index 13a49a00..4abd6f82 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java @@ -213,4 +213,53 @@ public void partitionInfoWithNegativeCreateTime() throws TException { assertThat(createTime).isAfterOrEqualTo(beforeTest); assertThat(createTime).isBeforeOrEqualTo(afterTest); } + + @Test + public void getSinglePartitionInfoSuccess() throws TException { + when(metaStoreClient.getTable(DATABASE_NAME, TABLE_NAME)).thenReturn(table); + when(table.getPartitionKeys()).thenReturn(List.of(eventDatePartitionKey, eventHourPartitionKey)); + when(metaStoreClient.getPartition(DATABASE_NAME, TABLE_NAME, List.of("2024-01-01", "1"))).thenReturn(partition); + when(partition.getSd()).thenReturn(storageDescriptor); + when(storageDescriptor.getLocation()).thenReturn(PARTITION_PATH); + when(partition.getCreateTime()).thenReturn(1234567890); + + PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); + + assertThat(partitionInfo).isNotNull(); + assertThat(partitionInfo.getPath()).isEqualTo(PARTITION_PATH); + assertThat(partitionInfo.getCreateTime()) + .isEqualTo(LocalDateTime.ofInstant(Instant.ofEpochSecond(1234567890), ZoneId.systemDefault())); + } + + @Test + public void getSinglePartitionInfoWithMissingCreateTime() throws TException { + when(metaStoreClient.getTable(DATABASE_NAME, TABLE_NAME)).thenReturn(table); + when(table.getPartitionKeys()).thenReturn(List.of(eventDatePartitionKey, eventHourPartitionKey)); + when(metaStoreClient.getPartition(DATABASE_NAME, TABLE_NAME, List.of("2024-01-01", "1"))).thenReturn(partition); + when(partition.getSd()).thenReturn(storageDescriptor); + when(storageDescriptor.getLocation()).thenReturn(PARTITION_PATH); + when(partition.getCreateTime()).thenReturn(0); + + LocalDateTime beforeTest = LocalDateTime.now(); + + PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); + + LocalDateTime afterTest = LocalDateTime.now(); + + assertThat(partitionInfo).isNotNull(); + assertThat(partitionInfo.getPath()).isEqualTo(PARTITION_PATH); + LocalDateTime createTime = partitionInfo.getCreateTime(); + assertThat(createTime).isNotNull(); + assertThat(createTime).isAfterOrEqualTo(beforeTest); + assertThat(createTime).isBeforeOrEqualTo(afterTest); + } + + @Test + public void getSinglePartitionInfoNotFound() throws TException { + when(metaStoreClient.getTable(DATABASE_NAME, TABLE_NAME)).thenThrow(TException.class); + + PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); + + assertThat(partitionInfo).isNull(); + } } From ad12f069161d5288630984bab36c40b6ef5b0550 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Tue, 27 May 2025 13:53:13 +0100 Subject: [PATCH 03/10] Update ExpiredHousekeepingMetadataGenerator.java --- .../ExpiredHousekeepingMetadataGenerator.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java index d52fe112..c2c7ae80 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java @@ -144,12 +144,10 @@ private HousekeepingEntity generateHousekeepingEntity( String partitionName) { PeriodDuration cleanupDelay = cleanupDelayExtractor.extractCleanupDelay(listenerEvent); - // Get the creation time from Hive if this is a partition LocalDateTime creationTime; if (partitionName != null) { creationTime = getPartitionCreationTime(listenerEvent.getDbName(), listenerEvent.getTableName(), partitionName); } else { - // For tables, use current time as there's no reliable way to get table creation time creationTime = LocalDateTime.now(clock); } @@ -184,14 +182,6 @@ private String generatePartitionName(List keys, List values) { .collect(Collectors.joining("/")); } - /** - * Gets the creation time of a partition from Hive. - * - * @param databaseName The database name - * @param tableName The table name - * @param partitionName The partition name - * @return The partition creation time from Hive, or current time if not available - */ private LocalDateTime getPartitionCreationTime(String databaseName, String tableName, String partitionName) { try (HiveClient hiveClient = hiveClientFactory.newInstance()) { PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(databaseName, tableName, partitionName); From 666a483b8dfcd69fa079296976d42ef6e8625f3c Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Tue, 10 Jun 2025 16:28:03 +0100 Subject: [PATCH 04/10] Update CHANGELOG.md --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 676751d2..33a0d297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.6.1] - 2025-05-14 +## [3.6.2] - 2025-06-10 +## Fixed +- Extended the use of Hive's partition `createTime` to event-driven scheduling. Cleanup events are now scheduled based on the partition's actual creation time in Hive, not the event processing time. + +- ## [3.6.1] - 2025-05-14 ## Fixed - Use Hive partition creation time (`createTime`) for scheduling partition cleanup, ensuring accurate expiry timing for both new and existing partitions. - Updated workflows to use `Ubuntu 22.04` instead of `Ubuntu 20.04`. From 3b23994501fb3bccee89104684252d58063ee832 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Mon, 16 Jun 2025 11:06:17 +0100 Subject: [PATCH 05/10] Update for comments --- .../ExpiredHousekeepingMetadataGenerator.java | 10 ++++----- ...iredHousekeepingMetadataGeneratorTest.java | 7 ++++--- .../beekeeper/scheduler/hive/HiveClient.java | 13 ++++++------ .../scheduler/hive/hive/HiveClientTest.java | 21 ++++++++++--------- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java index c2c7ae80..1d1c5ef8 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -150,9 +151,6 @@ private HousekeepingEntity generateHousekeepingEntity( } else { creationTime = LocalDateTime.now(clock); } - - log.info("TIMESTAMP_INFO: Creating housekeeping entity with creationTimestamp={} for partition={}", - creationTime, partitionName); return HousekeepingMetadata .builder() @@ -184,10 +182,10 @@ private String generatePartitionName(List keys, List values) { private LocalDateTime getPartitionCreationTime(String databaseName, String tableName, String partitionName) { try (HiveClient hiveClient = hiveClientFactory.newInstance()) { - PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(databaseName, tableName, partitionName); + Optional partitionInfo = hiveClient.getSinglePartitionInfo(databaseName, tableName, partitionName); - if (partitionInfo != null) { - return partitionInfo.getCreateTime(); + if (partitionInfo.isPresent()) { + return partitionInfo.get().getCreateTime(); } log.warn("Partition {} not found in Hive for table {}.{}, using current time", diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java index bb987442..34cfdc16 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGeneratorTest.java @@ -33,6 +33,7 @@ import java.time.LocalDateTime; import java.util.List; import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -108,7 +109,7 @@ public void typicalHandleAddPartitionEvent() { when(addPartitionEvent.getPartitionKeys()).thenReturn(PARTITION_KEYS); when(addPartitionEvent.getPartitionValues()).thenReturn(PARTITION_VALUES); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); - when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(null); + when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(Optional.empty()); List housekeepingEntities = generator.generate(addPartitionEvent, CLIENT_ID); assertThat(housekeepingEntities.size()).isEqualTo(1); @@ -123,7 +124,7 @@ public void typicalHandleAlterPartitionEvent() { when(alterPartitionEvent.getPartitionKeys()).thenReturn(PARTITION_KEYS); when(alterPartitionEvent.getPartitionValues()).thenReturn(PARTITION_VALUES); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); - when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(null); + when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(Optional.empty()); List housekeepingEntities = generator.generate(alterPartitionEvent, CLIENT_ID); assertThat(housekeepingEntities.size()).isEqualTo(1); @@ -162,7 +163,7 @@ public void usesPartitionCreationTimeFromHive() { when(hiveClientFactory.newInstance()).thenReturn(hiveClient); LocalDateTime hiveCreationTime = LocalDateTime.of(2023, 6, 15, 10, 30); PartitionInfo partitionInfo = new PartitionInfo(PARTITION_PATH, hiveCreationTime); - when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(partitionInfo); + when(hiveClient.getSinglePartitionInfo(DATABASE, TABLE, PARTITION_NAME)).thenReturn(Optional.of(partitionInfo)); List housekeepingEntities = generator.generate(addPartitionEvent, CLIENT_ID); diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java index af8b3010..6bfdc0f4 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -75,10 +76,8 @@ public Map getTablePartitionsInfo(String databaseName, St } } - public PartitionInfo getSinglePartitionInfo(String databaseName, String tableName, String partitionName) { + public Optional getSinglePartitionInfo(String databaseName, String tableName, String partitionName) { try { - Table table = metaStoreClient.getTable(databaseName, tableName); - List partitionKeys = table.getPartitionKeys(); List partitionValues = Warehouse.getPartValuesFromPartName(partitionName); Partition partition = metaStoreClient.getPartition(databaseName, tableName, partitionValues); @@ -89,11 +88,11 @@ public PartitionInfo getSinglePartitionInfo(String databaseName, String tableNam log.debug("Retrieved partition '{}' with path '{}' for table {}.{}", partitionName, path, databaseName, tableName); - return new PartitionInfo(path, createTime); + return Optional.of(new PartitionInfo(path, createTime)); } catch (TException e) { - log.warn("Failed to get partition info for {}.{}.{}: {}", - databaseName, tableName, partitionName, e.getMessage()); - return null; + log.warn("Failed to get partition info for {}.{}.{}", + databaseName, tableName, partitionName, e); + return Optional.empty(); } } diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java index 4abd6f82..d9e8738e 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java @@ -25,6 +25,7 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Partition; @@ -223,11 +224,11 @@ public void getSinglePartitionInfoSuccess() throws TException { when(storageDescriptor.getLocation()).thenReturn(PARTITION_PATH); when(partition.getCreateTime()).thenReturn(1234567890); - PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); + Optional partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); - assertThat(partitionInfo).isNotNull(); - assertThat(partitionInfo.getPath()).isEqualTo(PARTITION_PATH); - assertThat(partitionInfo.getCreateTime()) + assertThat(partitionInfo).isPresent(); + assertThat(partitionInfo.get().getPath()).isEqualTo(PARTITION_PATH); + assertThat(partitionInfo.get().getCreateTime()) .isEqualTo(LocalDateTime.ofInstant(Instant.ofEpochSecond(1234567890), ZoneId.systemDefault())); } @@ -242,13 +243,13 @@ public void getSinglePartitionInfoWithMissingCreateTime() throws TException { LocalDateTime beforeTest = LocalDateTime.now(); - PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); + Optional partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); LocalDateTime afterTest = LocalDateTime.now(); - assertThat(partitionInfo).isNotNull(); - assertThat(partitionInfo.getPath()).isEqualTo(PARTITION_PATH); - LocalDateTime createTime = partitionInfo.getCreateTime(); + assertThat(partitionInfo).isPresent(); + assertThat(partitionInfo.get().getPath()).isEqualTo(PARTITION_PATH); + LocalDateTime createTime = partitionInfo.get().getCreateTime(); assertThat(createTime).isNotNull(); assertThat(createTime).isAfterOrEqualTo(beforeTest); assertThat(createTime).isBeforeOrEqualTo(afterTest); @@ -258,8 +259,8 @@ public void getSinglePartitionInfoWithMissingCreateTime() throws TException { public void getSinglePartitionInfoNotFound() throws TException { when(metaStoreClient.getTable(DATABASE_NAME, TABLE_NAME)).thenThrow(TException.class); - PartitionInfo partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); + Optional partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); - assertThat(partitionInfo).isNull(); + assertThat(partitionInfo).isEmpty(); } } From bdd7ec7c35026701428c8605f3d4e4973c9059f9 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Mon, 16 Jun 2025 11:18:12 +0100 Subject: [PATCH 06/10] removing duplicate checks --- .../ExpiredHousekeepingMetadataGenerator.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java index 1d1c5ef8..1625ec95 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java @@ -145,13 +145,8 @@ private HousekeepingEntity generateHousekeepingEntity( String partitionName) { PeriodDuration cleanupDelay = cleanupDelayExtractor.extractCleanupDelay(listenerEvent); - LocalDateTime creationTime; - if (partitionName != null) { - creationTime = getPartitionCreationTime(listenerEvent.getDbName(), listenerEvent.getTableName(), partitionName); - } else { - creationTime = LocalDateTime.now(clock); - } - + LocalDateTime creationTime = getPartitionCreationTime(listenerEvent.getDbName(), listenerEvent.getTableName(), partitionName); + return HousekeepingMetadata .builder() .housekeepingStatus(SCHEDULED) @@ -181,6 +176,10 @@ private String generatePartitionName(List keys, List values) { } private LocalDateTime getPartitionCreationTime(String databaseName, String tableName, String partitionName) { + if (partitionName == null) { + return LocalDateTime.now(clock); + } + try (HiveClient hiveClient = hiveClientFactory.newInstance()) { Optional partitionInfo = hiveClient.getSinglePartitionInfo(databaseName, tableName, partitionName); From 5fa0ae42a04c2ec07537d75dbeda23967c5204c0 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Mon, 16 Jun 2025 11:43:25 +0100 Subject: [PATCH 07/10] Test fix --- .../beekeeper/scheduler/hive/hive/HiveClientTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java index d9e8738e..85239665 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java @@ -217,8 +217,6 @@ public void partitionInfoWithNegativeCreateTime() throws TException { @Test public void getSinglePartitionInfoSuccess() throws TException { - when(metaStoreClient.getTable(DATABASE_NAME, TABLE_NAME)).thenReturn(table); - when(table.getPartitionKeys()).thenReturn(List.of(eventDatePartitionKey, eventHourPartitionKey)); when(metaStoreClient.getPartition(DATABASE_NAME, TABLE_NAME, List.of("2024-01-01", "1"))).thenReturn(partition); when(partition.getSd()).thenReturn(storageDescriptor); when(storageDescriptor.getLocation()).thenReturn(PARTITION_PATH); @@ -234,8 +232,6 @@ public void getSinglePartitionInfoSuccess() throws TException { @Test public void getSinglePartitionInfoWithMissingCreateTime() throws TException { - when(metaStoreClient.getTable(DATABASE_NAME, TABLE_NAME)).thenReturn(table); - when(table.getPartitionKeys()).thenReturn(List.of(eventDatePartitionKey, eventHourPartitionKey)); when(metaStoreClient.getPartition(DATABASE_NAME, TABLE_NAME, List.of("2024-01-01", "1"))).thenReturn(partition); when(partition.getSd()).thenReturn(storageDescriptor); when(storageDescriptor.getLocation()).thenReturn(PARTITION_PATH); @@ -257,7 +253,7 @@ public void getSinglePartitionInfoWithMissingCreateTime() throws TException { @Test public void getSinglePartitionInfoNotFound() throws TException { - when(metaStoreClient.getTable(DATABASE_NAME, TABLE_NAME)).thenThrow(TException.class); + when(metaStoreClient.getPartition(DATABASE_NAME, TABLE_NAME, List.of("2024-01-01", "1"))).thenThrow(TException.class); Optional partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, PARTITION_NAME); From d995643824d2f13ace90b3b04e6b580f680b69cd Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Mon, 16 Jun 2025 23:13:27 +0100 Subject: [PATCH 08/10] Add JavaDoc for new method --- .../beekeeper/scheduler/hive/HiveClient.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java index 6bfdc0f4..7a7f52a0 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java @@ -76,6 +76,15 @@ public Map getTablePartitionsInfo(String databaseName, St } } + /** + * Retrieves information for a single partition from Hive metastore. + * + * @param databaseName the name of the Hive database + * @param tableName the name of the Hive table + * @param partitionName the partition identifier in Hive's standard format: "key1=value1/key2=value2" + * (e.g., "event_date=2024-01-01/event_hour=1") + * @return Optional containing PartitionInfo if found, empty if partition doesn't exist or on error + */ public Optional getSinglePartitionInfo(String databaseName, String tableName, String partitionName) { try { List partitionValues = Warehouse.getPartValuesFromPartName(partitionName); From a313e5f283d57e6bc74b561bb1e79064e4be8a21 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Tue, 17 Jun 2025 11:02:43 +0100 Subject: [PATCH 09/10] Moved null check --- .../generator/ExpiredHousekeepingMetadataGenerator.java | 4 ---- .../beekeeper/scheduler/hive/HiveClient.java | 9 +++++---- .../beekeeper/scheduler/hive/hive/HiveClientTest.java | 7 +++++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java index 1625ec95..65e24879 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/generator/ExpiredHousekeepingMetadataGenerator.java @@ -176,10 +176,6 @@ private String generatePartitionName(List keys, List values) { } private LocalDateTime getPartitionCreationTime(String databaseName, String tableName, String partitionName) { - if (partitionName == null) { - return LocalDateTime.now(clock); - } - try (HiveClient hiveClient = hiveClientFactory.newInstance()) { Optional partitionInfo = hiveClient.getSinglePartitionInfo(databaseName, tableName, partitionName); diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java index 7a7f52a0..25234381 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/hive/HiveClient.java @@ -82,15 +82,16 @@ public Map getTablePartitionsInfo(String databaseName, St * @param databaseName the name of the Hive database * @param tableName the name of the Hive table * @param partitionName the partition identifier in Hive's standard format: "key1=value1/key2=value2" - * (e.g., "event_date=2024-01-01/event_hour=1") - * @return Optional containing PartitionInfo if found, empty if partition doesn't exist or on error + * (e.g., "event_date=2024-01-01/event_hour=1"), or null for table-level events + * @return Optional containing PartitionInfo if found, empty if partition doesn't exist, on error, or if partitionName is null */ public Optional getSinglePartitionInfo(String databaseName, String tableName, String partitionName) { + if (partitionName == null) { + return Optional.empty(); + } try { List partitionValues = Warehouse.getPartValuesFromPartName(partitionName); - Partition partition = metaStoreClient.getPartition(databaseName, tableName, partitionValues); - String path = partition.getSd().getLocation(); LocalDateTime createTime = extractCreateTime(partition); diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java index 85239665..8aff9458 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/hive/hive/HiveClientTest.java @@ -259,4 +259,11 @@ public void getSinglePartitionInfoNotFound() throws TException { assertThat(partitionInfo).isEmpty(); } + + @Test + public void getSinglePartitionInfoWithNullPartitionName() { + Optional partitionInfo = hiveClient.getSinglePartitionInfo(DATABASE_NAME, TABLE_NAME, null); + + assertThat(partitionInfo).isEmpty(); + } } From 1c76301f08078669579feec2e4bc96329c949429 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Tue, 17 Jun 2025 13:53:29 +0100 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33a0d297..ccb314b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.6.2] - 2025-06-10 +## [3.6.2] - 2025-06-17 ## Fixed - Extended the use of Hive's partition `createTime` to event-driven scheduling. Cleanup events are now scheduled based on the partition's actual creation time in Hive, not the event processing time.