From 28a38bcb4e83c8b7662fc884400d20c87b8e0ade Mon Sep 17 00:00:00 2001 From: Guilherme Infante Sales Date: Wed, 18 Nov 2026 23:58:41 +0000 Subject: [PATCH] feat: distributed tracing for list sync, SharePoint reads and SQL loads --- README.md | 4 +- src/SPO2SQL.Console/Program.cs | 45 +++++++++++++------ src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs | 6 +++ src/SPO2SQL.Core/SqlServer/SQLInteraction.cs | 12 ++++- src/SPO2SQL.Core/Telemetry/SyncTracing.cs | 18 ++++++++ tests/SPO2SQL.Core.Tests/SyncTracingTests.cs | 31 +++++++++++++ 6 files changed, 100 insertions(+), 16 deletions(-) create mode 100644 src/SPO2SQL.Core/Telemetry/SyncTracing.cs create mode 100644 tests/SPO2SQL.Core.Tests/SyncTracingTests.cs diff --git a/README.md b/README.md index 0bf8903..dd87db6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - **Bounded parallelism** — independent lists are synchronized concurrently with `Application:MaxParallelLists` (default 4); failures in one list do not stop the others. - **App-only authentication** — certificate-based client credentials work under MFA and Conditional Access; username/password remains a legacy fallback. - **Startup health checks** — authenticated SharePoint connectivity and a real SQL connection are verified before any sync starts. -- **Observability** — structured logging plus OpenTelemetry metrics (`spo2sql.lists.synced`, `spo2sql.items.*`, `spo2sql.sync.duration`) tagged by list and sync mode. +- **Observability** — structured logging, OpenTelemetry metrics (`spo2sql.lists.synced`, `spo2sql.items.*`, `spo2sql.sync.duration`) and spans for each list sync, SharePoint read and SQL bulk copy. --- @@ -148,7 +148,7 @@ Example: } ``` -`Sql:BatchSize` controls both the SharePoint page size and the number of rows sent per SQL bulk-copy batch. `Application:EnableMetrics` publishes the `SPO2SQL` OpenTelemetry meter; without `OTEL_EXPORTER_OTLP_ENDPOINT` metrics are written to the console. +`Sql:BatchSize` controls both the SharePoint page size and the number of rows sent per SQL bulk-copy batch. `Application:EnableMetrics` publishes the `SPO2SQL` OpenTelemetry meter and activity source; without `OTEL_EXPORTER_OTLP_ENDPOINT` both are written to the console. Supply secrets outside source control. For local development: diff --git a/src/SPO2SQL.Console/Program.cs b/src/SPO2SQL.Console/Program.cs index af5e8ac..eb589de 100644 --- a/src/SPO2SQL.Console/Program.cs +++ b/src/SPO2SQL.Console/Program.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Options; using OpenTelemetry; using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; using SPO2SQL.Configuration; using SPO2SQL.Telemetry; @@ -94,20 +95,38 @@ private static IHostBuilder CreateHostBuilder(string[] args) => if (context.Configuration.GetValue("Application:EnableMetrics", true)) { - services.AddOpenTelemetry() - .WithMetrics(builder => + bool useOtlp = !string.IsNullOrWhiteSpace( + Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT")); + + var telemetry = services.AddOpenTelemetry(); + + telemetry.WithMetrics(builder => + { + builder.AddMeter(SyncMetrics.MeterName); + + if (useOtlp) + { + builder.AddOtlpExporter(); + } + else + { + builder.AddConsoleExporter(); + } + }); + + telemetry.WithTracing(builder => + { + builder.AddSource(SyncTracing.SourceName); + + if (useOtlp) + { + builder.AddOtlpExporter(); + } + else { - builder.AddMeter(SyncMetrics.MeterName); - - if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT"))) - { - builder.AddOtlpExporter(); - } - else - { - builder.AddConsoleExporter(); - } - }); + builder.AddConsoleExporter(); + } + }); } services.AddSingleton(); diff --git a/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs b/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs index d88d99e..79b3d93 100644 --- a/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs +++ b/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using SPO2SQL.Configuration; using SPO2SQL.Logging; using SPO2SQL.SharePoint; @@ -87,6 +88,10 @@ public static async Task RefreshListsSQLAsync( bool daily, CancellationToken cancellationToken = default) { + using Activity? activity = SyncTracing.Source.StartActivity("spo2sql.sync.list", ActivityKind.Internal); + activity?.SetTag(SyncTracing.ListTag, listName); + activity?.SetTag(SyncTracing.ModeTag, daily ? "incremental" : "full"); + var statistics = new OperationStatistics { StartTime = DateTime.UtcNow }; try @@ -147,6 +152,7 @@ await retryPolicy.ExecuteWithRetryAsync( } catch (Exception ex) { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); Logger.LogError($"RefreshListsSQL: FATAL ERROR updating list '{listName}'.", ex); throw; } diff --git a/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs b/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs index 2b10787..f3c21db 100644 --- a/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs +++ b/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs @@ -1,10 +1,12 @@ using System.Data; +using System.Diagnostics; using System.Globalization; using System.Text; using Microsoft.Data.SqlClient; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; using SPO2SQL.SharePoint; +using SPO2SQL.Telemetry; using SPO2SQL.XmlConfig; namespace SPO2SQL.SqlServer; @@ -102,7 +104,11 @@ public async Task BuildAsync() } Logger.Log(1, "[Build] Initializing SharePoint list structure..."); - await List.BuildAsync(CancellationToken).ConfigureAwait(false); + using (Activity? readActivity = SyncTracing.Source.StartActivity("spo2sql.sharepoint.read", ActivityKind.Client)) + { + readActivity?.SetTag(SyncTracing.ListTag, List.Name); + await List.BuildAsync(CancellationToken).ConfigureAwait(false); + } FNDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); BuildDictionary(); @@ -334,6 +340,10 @@ private async Task TransferDataAsync(DateTime snapDate) string safeTable = $"[{TableName.Replace("]", "]]")}]"; + using Activity? writeActivity = SyncTracing.Source.StartActivity("spo2sql.sql.bulk_copy", ActivityKind.Client); + writeActivity?.SetTag(SyncTracing.ListTag, TableName); + writeActivity?.SetTag(SyncTracing.RowsTag, batch.Rows.Count); + using var bulkCopy = new SqlBulkCopy( Connection!, SqlBulkCopyOptions.TableLock | diff --git a/src/SPO2SQL.Core/Telemetry/SyncTracing.cs b/src/SPO2SQL.Core/Telemetry/SyncTracing.cs new file mode 100644 index 0000000..16b8eb5 --- /dev/null +++ b/src/SPO2SQL.Core/Telemetry/SyncTracing.cs @@ -0,0 +1,18 @@ +using System.Diagnostics; + +namespace SPO2SQL.Telemetry; + +/// +/// Activity source used to trace list synchronization, SharePoint reads and SQL writes. +/// The host subscribes to it when metrics/telemetry are enabled. +/// +public static class SyncTracing +{ + public const string SourceName = "SPO2SQL"; + + public static readonly ActivitySource Source = new(SourceName, "2.0.0"); + + public const string ListTag = "spo2sql.list"; + public const string ModeTag = "spo2sql.mode"; + public const string RowsTag = "spo2sql.rows"; +} diff --git a/tests/SPO2SQL.Core.Tests/SyncTracingTests.cs b/tests/SPO2SQL.Core.Tests/SyncTracingTests.cs new file mode 100644 index 0000000..01c09b5 --- /dev/null +++ b/tests/SPO2SQL.Core.Tests/SyncTracingTests.cs @@ -0,0 +1,31 @@ +using System.Diagnostics; +using FluentAssertions; +using SPO2SQL.Telemetry; +using Xunit; + +namespace SPO2SQL.Core.Tests; + +public class SyncTracingTests +{ + [Fact] + public void Source_EmitsActivitiesToListeners() + { + Activity? captured = null; + + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == SyncTracing.SourceName, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStarted = activity => captured = activity + }; + + ActivitySource.AddActivityListener(listener); + + using var activity = SyncTracing.Source.StartActivity("spo2sql.sync.list", ActivityKind.Internal); + activity?.SetTag(SyncTracing.ListTag, "Activities"); + + captured.Should().NotBeNull(); + captured!.OperationName.Should().Be("spo2sql.sync.list"); + captured.GetTagItem(SyncTracing.ListTag).Should().Be("Activities"); + } +}