diff --git a/Src/AzureTablePurger/AzureTablePurger.App/AzureTablePurger.App.csproj b/Src/AzureTablePurger/AzureTablePurger.App/AzureTablePurger.App.csproj index dadd70d..c66268e 100644 --- a/Src/AzureTablePurger/AzureTablePurger.App/AzureTablePurger.App.csproj +++ b/Src/AzureTablePurger/AzureTablePurger.App/AzureTablePurger.App.csproj @@ -2,19 +2,18 @@ Exe - net5.0 + net6.0;net7.0 f21c130e-d067-4ec4-a6a5-1b7b81af9ef1 - - - - - - - - + + + + + + + diff --git a/Src/AzureTablePurger/AzureTablePurger.App/Program.cs b/Src/AzureTablePurger/AzureTablePurger.App/Program.cs index c54ed60..f3cbbc4 100644 --- a/Src/AzureTablePurger/AzureTablePurger.App/Program.cs +++ b/Src/AzureTablePurger/AzureTablePurger.App/Program.cs @@ -26,6 +26,7 @@ static async Task Main(string[] args) BuildConfig(args); var serviceCollection = RegisterServices(); ConfigureLogging(serviceCollection); + _serviceProvider = serviceCollection.BuildServiceProvider(); await using (_serviceProvider) @@ -77,7 +78,7 @@ private static ServiceCollection RegisterServices() // Core logic serviceCollection.AddScoped(); serviceCollection.AddScoped(); - serviceCollection.AddScoped(); + serviceCollection.AddScoped(); return serviceCollection; } diff --git a/Src/AzureTablePurger/AzureTablePurger.Services.Tests/AzureTablePurger.Services.Tests.csproj b/Src/AzureTablePurger/AzureTablePurger.Services.Tests/AzureTablePurger.Services.Tests.csproj index 4466c85..5e0082e 100644 --- a/Src/AzureTablePurger/AzureTablePurger.Services.Tests/AzureTablePurger.Services.Tests.csproj +++ b/Src/AzureTablePurger/AzureTablePurger.Services.Tests/AzureTablePurger.Services.Tests.csproj @@ -1,17 +1,17 @@  - netcoreapp3.1 + net7.0 false - - - - - + + + + + diff --git a/Src/AzureTablePurger/AzureTablePurger.Services.Tests/TicksAscendingWithLeadingZeroPartitionKeyHandlerTest.cs b/Src/AzureTablePurger/AzureTablePurger.Services.Tests/TicksAscendingWithLeadingZeroPartitionKeyHandlerTest.cs index a80eef2..ac93573 100644 --- a/Src/AzureTablePurger/AzureTablePurger.Services.Tests/TicksAscendingWithLeadingZeroPartitionKeyHandlerTest.cs +++ b/Src/AzureTablePurger/AzureTablePurger.Services.Tests/TicksAscendingWithLeadingZeroPartitionKeyHandlerTest.cs @@ -24,7 +24,7 @@ public void ConvertPartitionKeyToDateTime_FailsWithGuid() // Arrange // Act - _target.ConvertPartitionKeyToDateTime(Guid.NewGuid().ToString()); + _target.ConvertKeyToDateTime(Guid.NewGuid().ToString()); // Assert - handled by method attribute } diff --git a/Src/AzureTablePurger/AzureTablePurger.Services/AzureTablePurger.Services.csproj b/Src/AzureTablePurger/AzureTablePurger.Services/AzureTablePurger.Services.csproj index a70185a..7e58621 100644 --- a/Src/AzureTablePurger/AzureTablePurger.Services/AzureTablePurger.Services.csproj +++ b/Src/AzureTablePurger/AzureTablePurger.Services/AzureTablePurger.Services.csproj @@ -6,7 +6,7 @@ - + diff --git a/Src/AzureTablePurger/AzureTablePurger.Services/IEntityQueryHandler.cs b/Src/AzureTablePurger/AzureTablePurger.Services/IEntityQueryHandler.cs new file mode 100644 index 0000000..71535a7 --- /dev/null +++ b/Src/AzureTablePurger/AzureTablePurger.Services/IEntityQueryHandler.cs @@ -0,0 +1,17 @@ +using System; + +using Microsoft.Azure.Cosmos.Table; + +namespace AzureTablePurger.Services +{ + public interface IEntityQueryHandler + { + TableQuery GetTableQuery(int purgeEntitiesOlderThanDays); + + TableQuery GetTableQuery(string lowerBoundKey, string upperBoundKey); + + DateTime ConvertKeyToDateTime(DynamicTableEntity entry); + + + } +} diff --git a/Src/AzureTablePurger/AzureTablePurger.Services/IPartitionKeyHandler.cs b/Src/AzureTablePurger/AzureTablePurger.Services/IPartitionKeyHandler.cs deleted file mode 100644 index 177d2d8..0000000 --- a/Src/AzureTablePurger/AzureTablePurger.Services/IPartitionKeyHandler.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; - -using Microsoft.Azure.Cosmos.Table; - -namespace AzureTablePurger.Services -{ - public interface IPartitionKeyHandler - { - TableQuery GetTableQuery(int purgeEntitiesOlderThanDays); - - DateTime ConvertPartitionKeyToDateTime(string partitionKey); - - string GetPartitionKeyForDate(DateTime date); - - TableQuery GetTableQuery(string lowerBoundPartitionKey, string upperBoundPartitionKey); - } -} diff --git a/Src/AzureTablePurger/AzureTablePurger.Services/SimpleTablePurger.cs b/Src/AzureTablePurger/AzureTablePurger.Services/SimpleTablePurger.cs index 1132c92..dc8574d 100644 --- a/Src/AzureTablePurger/AzureTablePurger.Services/SimpleTablePurger.cs +++ b/Src/AzureTablePurger/AzureTablePurger.Services/SimpleTablePurger.cs @@ -20,12 +20,12 @@ public class SimpleTablePurger : ITablePurger private readonly IAzureStorageClientFactory _storageClientFactory; private readonly ILogger _logger; - private readonly IPartitionKeyHandler _partitionKeyHandler; + private readonly IEntityQueryHandler _rowHandler; - public SimpleTablePurger(IAzureStorageClientFactory storageClientFactory, IPartitionKeyHandler partitionKeyHandler, ILogger logger) + public SimpleTablePurger(IAzureStorageClientFactory storageClientFactory, IEntityQueryHandler rowHandler, ILogger logger) { _storageClientFactory = storageClientFactory; - _partitionKeyHandler = partitionKeyHandler; + _rowHandler = rowHandler; _logger = logger; ServicePointManager.DefaultConnectionLimit = ConnectionLimit; @@ -43,7 +43,7 @@ public async Task> PurgeEntitiesAsync(PurgeEntitiesOptions optio _logger.LogInformation($"TargetAccount={tableClient.StorageUri.PrimaryUri}, Table={table.Name}, PurgeRecordsOlderThanDays={options.PurgeRecordsOlderThanDays}"); - var query = _partitionKeyHandler.GetTableQuery(options.PurgeRecordsOlderThanDays); + var query = _rowHandler.GetTableQuery(options.PurgeRecordsOlderThanDays); var continuationToken = new TableContinuationToken(); int numPagesProcessed = 0; @@ -64,7 +64,7 @@ public async Task> PurgeEntitiesAsync(PurgeEntitiesOptions optio break; } - var firstResultTimestamp = _partitionKeyHandler.ConvertPartitionKeyToDateTime(page.Results.First().PartitionKey); + var firstResultTimestamp = _rowHandler.ConvertKeyToDateTime(page.Results.First()); _logger.LogInformation($"Page {pageNumber}: processing {page.Count()} results starting at timestamp {firstResultTimestamp}"); var partitionsFromPage = GetPartitionsFromPage(page.Results); @@ -106,7 +106,7 @@ public async Task> PurgeEntitiesAsync(PurgeEntitiesOptions optio var entitiesPerSecond = numEntitiesDeleted > 0 ? (int)(numEntitiesDeleted / sw.Elapsed.TotalSeconds) : 0; var msPerEntity = numEntitiesDeleted > 0 ? (int)(sw.Elapsed.TotalMilliseconds / numEntitiesDeleted) : 0; - + _logger.LogInformation($"Finished PurgeEntitiesAsync, processed {numPagesProcessed} pages and deleted {numEntitiesDeleted} entities in {sw.Elapsed} ({entitiesPerSecond} entities per second, or {msPerEntity} ms per entity)"); return new Tuple(numPagesProcessed, numEntitiesDeleted); diff --git a/Src/AzureTablePurger/AzureTablePurger.Services/StringDateRowKeyHandler.cs b/Src/AzureTablePurger/AzureTablePurger.Services/StringDateRowKeyHandler.cs new file mode 100644 index 0000000..d88a0b3 --- /dev/null +++ b/Src/AzureTablePurger/AzureTablePurger.Services/StringDateRowKeyHandler.cs @@ -0,0 +1,61 @@ +using System; +using System.Globalization; +using Microsoft.Azure.Cosmos.Table; +using Microsoft.Extensions.Logging; + +namespace AzureTablePurger.Services +{ + public class StringDateRowKeyHandler : IEntityQueryHandler + { + private readonly ILogger _logger; + + public StringDateRowKeyHandler(ILogger logger) + { + _logger = logger; + } + + public TableQuery GetTableQuery(int purgeEntitiesOlderThanDays) + { + + var maximumPartitionKeyToDelete = GetMaximumToDelete(purgeEntitiesOlderThanDays); + + _logger.LogDebug($"{nameof(DynamicTableEntity.RowKey)}: {purgeEntitiesOlderThanDays}"); + + return GetTableQuery(null, maximumPartitionKeyToDelete); + } + + public TableQuery GetTableQuery(string lowerBoundPartitionKey, string upperBoundRowKey) + { + + var upperBound = TableQuery.GenerateFilterCondition(nameof(DynamicTableEntity.RowKey), QueryComparisons.LessThan, upperBoundRowKey); + + var query = new TableQuery() + .Where(upperBound) + .Select(new[] { nameof(DynamicTableEntity.PartitionKey), nameof(DynamicTableEntity.RowKey) }); + + + _logger.LogInformation($"Query : {query.FilterString}"); + + return query; + } + + public string GetKeyForDate(DateTime date) + { + return date.ToString("yyyy_MM_dd_HH_mm"); + } + + public DateTime ConvertKeyToDateTime(DynamicTableEntity entry) + { + + var result = DateTime.ParseExact(entry.RowKey, "yyyy_MM_dd_HH_mm", CultureInfo.InvariantCulture); + + return result; + + } + + private string GetMaximumToDelete(int purgeRecordsOlderThanDays) + { + return GetKeyForDate(DateTime.UtcNow.AddDays(-1 * purgeRecordsOlderThanDays)); + } + } +} \ No newline at end of file diff --git a/Src/AzureTablePurger/AzureTablePurger.Services/TicksAscendingWithLeadingZeroPartitionKeyHandler.cs b/Src/AzureTablePurger/AzureTablePurger.Services/TicksAscendingWithLeadingZeroPartitionKeyHandler.cs index 4bbc8e1..498a306 100644 --- a/Src/AzureTablePurger/AzureTablePurger.Services/TicksAscendingWithLeadingZeroPartitionKeyHandler.cs +++ b/Src/AzureTablePurger/AzureTablePurger.Services/TicksAscendingWithLeadingZeroPartitionKeyHandler.cs @@ -5,7 +5,7 @@ namespace AzureTablePurger.Services { - public class TicksAscendingWithLeadingZeroPartitionKeyHandler : IPartitionKeyHandler + public class TicksAscendingWithLeadingZeroPartitionKeyHandler : IEntityQueryHandler { private readonly ILogger _logger; @@ -18,6 +18,8 @@ public TableQuery GetTableQuery(int purgeEntitiesOlderThanDays) { var maximumPartitionKeyToDelete = GetMaximumPartitionKeyToDelete(purgeEntitiesOlderThanDays); + _logger.LogDebug($"{nameof(DynamicTableEntity.PartitionKey)}: {purgeEntitiesOlderThanDays}"); + return GetTableQuery(null, maximumPartitionKeyToDelete); } @@ -28,22 +30,30 @@ public TableQuery GetTableQuery(string lowerBoundPartitionKey, string upperBound lowerBoundPartitionKey = "0"; } - var lowerBoundDateTime = ConvertPartitionKeyToDateTime(lowerBoundPartitionKey); - var upperBoundDateTime = ConvertPartitionKeyToDateTime(upperBoundPartitionKey); - _logger.LogDebug($"Generating table query: lowerBound partitionKey={lowerBoundPartitionKey} ({lowerBoundDateTime}), upperBound partitionKey={upperBoundPartitionKey} ({upperBoundDateTime})"); + var lowerBoundDateTime = ConvertKeyToDateTime(lowerBoundPartitionKey); + var upperBoundDateTime = ConvertKeyToDateTime(upperBoundPartitionKey); + _logger.LogDebug($"Generating table query: lowerBound {nameof(DynamicTableEntity.PartitionKey)}={lowerBoundPartitionKey} ({lowerBoundDateTime}), upperBound {nameof(DynamicTableEntity.PartitionKey)}={upperBoundPartitionKey} ({upperBoundDateTime})"); - var lowerBound = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, lowerBoundPartitionKey); - var upperBound = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThan, upperBoundPartitionKey); + var lowerBound = TableQuery.GenerateFilterCondition(nameof(DynamicTableEntity.PartitionKey), QueryComparisons.GreaterThanOrEqual, lowerBoundPartitionKey); + var upperBound = TableQuery.GenerateFilterCondition(nameof(DynamicTableEntity.PartitionKey), QueryComparisons.LessThan, upperBoundPartitionKey); var combinedFilter = TableQuery.CombineFilters(lowerBound, TableOperators.And, upperBound); var query = new TableQuery() .Where(combinedFilter) - .Select(new[] { "PartitionKey", "RowKey" }); + .Select(new[] { nameof(DynamicTableEntity.PartitionKey), nameof(DynamicTableEntity.RowKey) }); + + _logger.LogInformation($"Query : {query}"); return query; } - public DateTime ConvertPartitionKeyToDateTime(string partitionKey) + + public DateTime ConvertKeyToDateTime(DynamicTableEntity entry) + { + return ConvertKeyToDateTime(entry.PartitionKey); + } + + public DateTime ConvertKeyToDateTime(string partitionKey) { var result = long.TryParse(partitionKey, out long ticks);