Skip to content

Commit 94ca592

Browse files
dei79dei79
committed
Add structured updatable task metadata
1 parent 094ecbb commit 94ca592

19 files changed

Lines changed: 563 additions & 91 deletions

File tree

‎README.md‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,70 @@ builder.Services.AddTaskLoggerForAzureStorageTable(
3838
Use task scopes through `ILogger`:
3939

4040
```csharp
41-
using (_logger.BeginNewTaskScope("ImportJob", "Queue", "Worker-1"))
41+
var metadata = new JsonObject
42+
{
43+
["tenant"] = "north",
44+
["file"] = "customers.csv",
45+
["pageSize"] = 100000,
46+
["cleanUpData"] = true
47+
};
48+
49+
using (_logger.BeginNewTaskScope("ImportJob", "Queue", "Worker-1", metadata))
4250
{
4351
_logger.LogInformation("Processing started");
4452
}
4553
```
4654

55+
Metadata is exposed as `JsonObject` from `System.Text.Json.Nodes` and is stored as JSON in the Azure Table `TaskData` property. JSON value types, arrays, and nested objects are preserved. Use the overload without metadata when a task has none:
56+
57+
```csharp
58+
var taskId = await taskLoggerFactory.AnnounceTask(
59+
"ImportJob",
60+
"Queue",
61+
"Worker-1");
62+
```
63+
64+
### Updating Metadata
65+
66+
Metadata updates are buffered and persisted together with the next task status update. New values are merged into the stored JSON object; an existing key is overwritten:
67+
68+
```csharp
69+
await taskLoggerFactory.MergeTaskMetadata(taskId, new JsonObject
70+
{
71+
["progress"] = 50,
72+
["file"] = "customers-v2.csv"
73+
});
74+
75+
await taskLoggerFactory.UpdateTaskStatus(taskId, TaskStatus.Running);
76+
```
77+
78+
Task scopes provide the same deferred behavior. Disposal persists the metadata together with the completion status:
79+
80+
```csharp
81+
using var scope = _logger.BeginTaskScope(taskId);
82+
scope?.MergeTaskMetadata(new JsonObject
83+
{
84+
["records"] = 1200
85+
});
86+
```
87+
88+
Azure Table updates use optimistic ETag concurrency and partial entity updates. Concurrent metadata changes are re-read and merged without replacing unrelated task fields.
89+
90+
### Migrating from 0.x Metadata APIs
91+
92+
String and dictionary overloads were removed in 0.5.0. Parse existing JSON strings into a `JsonObject`:
93+
94+
```csharp
95+
// 0.x
96+
await taskLoggerFactory.AnnounceTask("ImportJob", "Queue", "Worker-1", "{\"tenant\":\"north\"}");
97+
98+
// 0.5
99+
var metadata = JsonNode.Parse("{\"tenant\":\"north\"}")!.AsObject();
100+
await taskLoggerFactory.AnnounceTask("ImportJob", "Queue", "Worker-1", metadata);
101+
```
102+
103+
`AnnounceTask` is available with or without metadata and with or without a `CancellationToken`. Structured metadata must be supplied as a `JsonObject`.
104+
47105
## Maintenance Cleanup
48106

49107
`CoreHelpers.TaskLogging.Maintenance` can delete old, month-based task logging tables that were created by `CoreHelpers.TaskLogging`.
@@ -118,4 +176,4 @@ git tag v0.0.0
118176
git push origin v0.0.0
119177
```
120178

121-
The release workflow builds, tests, packs all NuGet packages, creates the GitHub Release, uploads the `.nupkg` files as release assets, and then publishes the same package artifacts to NuGet.
179+
The release workflow builds, tests, packs all NuGet packages, creates the GitHub Release, uploads the `.nupkg` files as release assets, and then publishes the same package artifacts to NuGet.

‎dotNet/CoreHelpers.Extensions.Logging.DurableTask/CoreHelpers.Extensions.Logging.DurableTask.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
<PropertyGroup>
44
<TargetFramework>netstandard2.1</TargetFramework>
55
<Nullable>enable</Nullable>
6+
<Version>0.5.0</Version>
67
</PropertyGroup>
78

89
<ItemGroup>
@@ -13,4 +14,4 @@
1314
<ItemGroup>
1415
<ProjectReference Include="..\CoreHelpers.Extensions.Logging.Tasks\CoreHelpers.Extensions.Logging.Tasks.csproj" />
1516
</ItemGroup>
16-
</Project>
17+
</Project>

‎dotNet/CoreHelpers.Extensions.Logging.Tasks.Tests/FakeTaskLoggerFactory.cs‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using CoreHelpers.TaskLogging;
2+
using System.Text.Json.Nodes;
23
using LoggingTaskStatus = CoreHelpers.TaskLogging.TaskStatus;
34

45
namespace CoreHelpers.Extensions.Logging.Tasks.Tests;
@@ -23,6 +24,12 @@ internal sealed class FakeTaskLoggerFactory : ITaskLoggerFactory
2324

2425
public Exception? StatusUpdateException { get; set; }
2526

27+
public List<(string TaskId, JsonObject Metadata)> MetadataMerges { get; } = new();
28+
29+
public JsonObject? AnnouncedMetadata { get; private set; }
30+
31+
public List<string> LifecycleEvents { get; } = new();
32+
2633
public Func<int, string[], Exception?>? GetMergeException { get; set; }
2734

2835
public int? PersistedMessageCountPerMerge { get; set; }
@@ -44,34 +51,40 @@ public Task<string[]> MergePendingMessagesIfNeeded(DateTimeOffset flushTime, boo
4451
}
4552

4653
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker)
47-
=> AnnounceException == null ? Task.FromResult("announced-task") : Task.FromException<string>(AnnounceException);
54+
=> AnnounceTask(taskType, taskSource, taskWorker, new JsonObject());
4855

4956
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker, CancellationToken cancellationToken)
50-
=> Task.FromResult("announced-task");
57+
=> AnnounceTask(taskType, taskSource, taskWorker, new JsonObject(), cancellationToken);
5158

52-
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker, string metaData)
53-
=> Task.FromResult("announced-task");
54-
55-
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker, string metaData, CancellationToken cancellationToken)
56-
=> Task.FromResult("announced-task");
59+
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker, JsonObject metadata)
60+
{
61+
AnnouncedMetadata = (JsonObject)metadata.DeepClone();
62+
return AnnounceException == null ? Task.FromResult("announced-task") : Task.FromException<string>(AnnounceException);
63+
}
5764

58-
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker, IDictionary<string, string> metaDataTyped)
59-
=> Task.FromResult("announced-task");
65+
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker, JsonObject metadata, CancellationToken cancellationToken)
66+
=> AnnounceTask(taskType, taskSource, taskWorker, metadata);
6067

61-
public Task<string> AnnounceTask(string taskType, string taskSource, string taskWorker, IDictionary<string, string> metaDataTyped, CancellationToken cancellationToken)
62-
=> Task.FromResult("announced-task");
68+
public Task MergeTaskMetadata(string taskId, JsonObject metadata)
69+
{
70+
MetadataMerges.Add((taskId, (JsonObject)metadata.DeepClone()));
71+
LifecycleEvents.Add($"metadata:{taskId}");
72+
return Task.CompletedTask;
73+
}
6374

6475
public Task UpdateTaskStatus(string taskId, LoggingTaskStatus taskStatus)
6576
{
6677
StatusUpdates.Add(taskStatus);
6778
TaskStatusUpdates.Add((taskId, taskStatus));
79+
LifecycleEvents.Add($"status:{taskId}:{taskStatus}");
6880
return StatusUpdateException == null ? Task.CompletedTask : Task.FromException(StatusUpdateException);
6981
}
7082

7183
public Task UpdateTaskStatus(string taskId, LoggingTaskStatus taskStatus, string taskWorker)
7284
{
7385
StatusUpdates.Add(taskStatus);
7486
TaskStatusUpdates.Add((taskId, taskStatus));
87+
LifecycleEvents.Add($"status:{taskId}:{taskStatus}");
7588
return Task.CompletedTask;
7689
}
7790

‎dotNet/CoreHelpers.Extensions.Logging.Tasks.Tests/TaskLoggerScopeTests.cs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using CoreHelpers.TaskLogging;
22
using Microsoft.Extensions.DependencyInjection;
33
using Microsoft.Extensions.Logging;
4+
using System.Text.Json.Nodes;
45
using Xunit;
56

67
namespace CoreHelpers.Extensions.Logging.Tasks.Tests;
@@ -191,6 +192,40 @@ public void ExplicitSucceedStatus_OverridesAutomaticFailureStatus()
191192
Assert.Equal(CoreHelpers.TaskLogging.TaskStatus.Succeed, factory.TaskStatusUpdates.Last().Status);
192193
}
193194

195+
[Fact]
196+
public void NewTaskScope_AnnouncesTypedMetadata()
197+
{
198+
var factory = CreatePersistingFactory();
199+
using var services = CreateServices(factory);
200+
var logger = services.GetRequiredService<ILoggerFactory>().CreateLogger("TypedMetadata");
201+
202+
using (logger.BeginNewTaskScope("type", "source", "worker", new JsonObject { ["tenant"] = "north" }, TimeSpan.FromHours(1)))
203+
{
204+
}
205+
206+
Assert.Equal("north", factory.AnnouncedMetadata!["tenant"]!.GetValue<string>());
207+
}
208+
209+
[Fact]
210+
public void ScopeMetadata_IsBufferedUntilDisposeAndFlushedBeforeStatus()
211+
{
212+
var factory = CreatePersistingFactory();
213+
using var services = CreateServices(factory);
214+
var logger = services.GetRequiredService<ILoggerFactory>().CreateLogger("BufferedMetadata");
215+
var scope = Assert.IsAssignableFrom<ITaskLoggerScope>(logger.BeginTaskScope("task", TimeSpan.FromHours(1)));
216+
217+
scope.MergeTaskMetadata(new JsonObject { ["progress"] = 50 });
218+
219+
Assert.Empty(factory.MetadataMerges);
220+
221+
scope.Dispose();
222+
223+
var merge = Assert.Single(factory.MetadataMerges);
224+
Assert.Equal("task", merge.TaskId);
225+
Assert.Equal(50, merge.Metadata["progress"]!.GetValue<int>());
226+
Assert.True(factory.LifecycleEvents.IndexOf("metadata:task") < factory.LifecycleEvents.IndexOf("status:task:Succeed"));
227+
}
228+
194229
[Theory]
195230
[InlineData(CoreHelpers.TaskLogging.TaskStatus.Pending)]
196231
[InlineData(CoreHelpers.TaskLogging.TaskStatus.Running)]

‎dotNet/CoreHelpers.Extensions.Logging.Tasks/CoreHelpers.Extensions.Logging.Tasks.csproj‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@
33
<PropertyGroup>
44
<TargetFramework>netstandard2.1</TargetFramework>
55
<Nullable>enable</Nullable>
6-
<Version>0.4.1</Version>
6+
<Version>0.5.0</Version>
77
</PropertyGroup>
88

99
<ItemGroup>
1010
<ProjectReference Include="..\CoreHelpers.TaskLogging.Abstractions\CoreHelpers.TaskLogging.Abstractions.csproj" />
1111
</ItemGroup>
1212
<ItemGroup>
1313
<PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
14-
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
15-
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
14+
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
15+
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
1616
</ItemGroup>
1717
</Project>

‎dotNet/CoreHelpers.Extensions.Logging.Tasks/TaskLogger.cs‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,9 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except
4747
if (taskLoggerState.IsTaskAnnounced && !string.IsNullOrEmpty(taskLoggerState.TaskId))
4848
return;
4949

50-
if (String.IsNullOrEmpty(taskLoggerState.MetaData))
51-
{
52-
taskLoggerState.TaskId = _taskLoggerFactory
53-
.AnnounceTask(taskLoggerState.TaskType, taskLoggerState.TaskSource, taskLoggerState.TaskWorker).GetAwaiter()
54-
.GetResult();
55-
}
56-
else
57-
{
58-
taskLoggerState.TaskId = _taskLoggerFactory
59-
.AnnounceTask(taskLoggerState.TaskType, taskLoggerState.TaskSource, taskLoggerState.TaskWorker, taskLoggerState.MetaData).GetAwaiter()
60-
.GetResult();
61-
}
50+
taskLoggerState.TaskId = _taskLoggerFactory
51+
.AnnounceTask(taskLoggerState.TaskType, taskLoggerState.TaskSource, taskLoggerState.TaskWorker, taskLoggerState.Metadata).GetAwaiter()
52+
.GetResult();
6253

6354
taskLoggerState.IsTaskAnnounced = true;
6455

@@ -80,6 +71,12 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except
8071
{
8172
// at this point we need to flush if needed
8273
MergePendingMessages(taskLoggerState, true, true);
74+
75+
if (taskLoggerState.PendingMetadata.Count > 0)
76+
{
77+
_taskLoggerFactory.MergeTaskMetadata(taskLoggerState.TaskId, taskLoggerState.PendingMetadata).GetAwaiter().GetResult();
78+
taskLoggerState.PendingMetadata.Clear();
79+
}
8380

8481
// ensure the task is finished now
8582
var completionStatus = taskLoggerState.CompletionStatus ?? (taskLoggerState.LastLogWasAnError ? TaskStatus.Failed : TaskStatus.Succeed);

‎dotNet/CoreHelpers.Extensions.Logging.Tasks/TaskLoggerExtension.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using CoreHelpers.TaskLogging;
33
using Microsoft.Extensions.Logging;
44
using Microsoft.Extensions.DependencyInjection;
5+
using System.Text.Json.Nodes;
56

67
namespace CoreHelpers.Extensions.Logging.Tasks
78
{
@@ -30,10 +31,10 @@ public static ILoggingBuilder AddTaskLogger(this ILoggingBuilder builder, Action
3031
=> BeginTypedTaskScope(logger, new TaskLoggerState() { TaskId = taskId, IsTaskAnnounced = true, TaskWorker = taskWorker, CacheTimeSpan = cacheTimeSpan ?? TimeSpan.FromSeconds(30) });
3132

3233
public static ITaskLoggerScope? BeginNewTaskScope(this ILogger logger, string taskType, string taskSource, string taskWorker, TimeSpan? cacheTimeSpan = null)
33-
=> BeginTypedTaskScope(logger, new TaskLoggerState() { TaskId = string.Empty, TaskType = taskType, TaskSource = taskSource, TaskWorker = taskWorker, IsTaskAnnounced = false, CacheTimeSpan = cacheTimeSpan ?? TimeSpan.FromSeconds(30) });
34+
=> BeginNewTaskScope(logger, taskType, taskSource, taskWorker, new JsonObject(), cacheTimeSpan);
3435

35-
public static ITaskLoggerScope? BeginNewTaskScope(this ILogger logger, string taskType, string taskSource, string taskWorker, string metaDataString, TimeSpan? cacheTimeSpan = null)
36-
=> BeginTypedTaskScope(logger, new TaskLoggerState() { TaskId = string.Empty, TaskType = taskType, TaskSource = taskSource, TaskWorker = taskWorker, IsTaskAnnounced = false, MetaData = metaDataString, CacheTimeSpan = cacheTimeSpan ?? TimeSpan.FromSeconds(30) });
36+
public static ITaskLoggerScope? BeginNewTaskScope(this ILogger logger, string taskType, string taskSource, string taskWorker, JsonObject metadata, TimeSpan? cacheTimeSpan = null)
37+
=> BeginTypedTaskScope(logger, new TaskLoggerState() { TaskId = string.Empty, TaskType = taskType, TaskSource = taskSource, TaskWorker = taskWorker, IsTaskAnnounced = false, Metadata = (JsonObject)metadata.DeepClone(), CacheTimeSpan = cacheTimeSpan ?? TimeSpan.FromSeconds(30) });
3738

3839
private static ITaskLoggerScope? BeginTypedTaskScope(ILogger logger, TaskLoggerState taskLoggerState)
3940
{

‎dotNet/CoreHelpers.Extensions.Logging.Tasks/TaskLoggerScope.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Threading;
33
using CoreHelpers.TaskLogging;
44
using Microsoft.Extensions.Logging;
5+
using System.Text.Json.Nodes;
56

67
namespace CoreHelpers.Extensions.Logging.Tasks
78
{
@@ -45,6 +46,18 @@ public void Dispose()
4546
public string TaskSource => _taskLoggerState.TaskSource;
4647
public string TaskWorker => _taskLoggerState.TaskWorker;
4748

49+
public void MergeTaskMetadata(JsonObject metadata)
50+
{
51+
if (metadata == null)
52+
throw new ArgumentNullException(nameof(metadata));
53+
54+
lock (_taskLoggerState.PendingMessagesSyncRoot)
55+
{
56+
foreach (var property in metadata)
57+
_taskLoggerState.PendingMetadata[property.Key] = property.Value?.DeepClone();
58+
}
59+
}
60+
4861
public void SetStatus(TaskStatus status)
4962
{
5063
if (status != TaskStatus.Succeed && status != TaskStatus.Failed)

‎dotNet/CoreHelpers.Extensions.Logging.Tasks/TaskLoggerState.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using CoreHelpers.TaskLogging;
4+
using System.Text.Json.Nodes;
45

56
namespace CoreHelpers.Extensions.Logging.Tasks
67
{
@@ -12,7 +13,8 @@ internal class TaskLoggerState
1213
public string TaskSource { get; set; } = string.Empty;
1314
public string TaskWorker { get; set; } = string.Empty;
1415

15-
public string MetaData { get; set; } = string.Empty;
16+
public JsonObject Metadata { get; set; } = new JsonObject();
17+
public JsonObject PendingMetadata { get; } = new JsonObject();
1618
public bool IsTaskAnnounced { get; set; } = false;
1719
public bool LastLogWasAnError { get; set; } = false;
1820
public TaskStatus? CompletionStatus { get; set; }

‎dotNet/CoreHelpers.TaskLogging.Abstractions/CoreHelpers.TaskLogging.Abstractions.csproj‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@
44
<TargetFramework>netstandard2.1</TargetFramework>
55
<Nullable>enable</Nullable>
66
<RootNamespace>CoreHelpers.TaskLogging</RootNamespace>
7-
<Version>0.4.1</Version>
8-
</PropertyGroup>
9-
7+
<Version>0.5.0</Version>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="System.Text.Json" Version="8.0.6" />
12+
</ItemGroup>
13+
1014
</Project>

0 commit comments

Comments
 (0)