Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
- **Bounded parallelism** — independent lists are synchronized concurrently with `Application:MaxParallelLists` (default 4); failures in one list do not stop the others.
- **App-only authentication** — certificate-based client credentials work under MFA and Conditional Access; username/password remains a legacy fallback.
- **Startup health checks** — authenticated SharePoint connectivity and a real SQL connection are verified before any sync starts.
- **Observability** — structured logging plus OpenTelemetry metrics (`spo2sql.lists.synced`, `spo2sql.items.*`, `spo2sql.sync.duration`) tagged by list and sync mode.
- **Observability** — structured logging, OpenTelemetry metrics (`spo2sql.lists.synced`, `spo2sql.items.*`, `spo2sql.sync.duration`) and spans for each list sync, SharePoint read and SQL bulk copy.

---

Expand Down Expand Up @@ -148,7 +148,7 @@ Example:
}
```

`Sql:BatchSize` controls both the SharePoint page size and the number of rows sent per SQL bulk-copy batch. `Application:EnableMetrics` publishes the `SPO2SQL` OpenTelemetry meter; without `OTEL_EXPORTER_OTLP_ENDPOINT` metrics are written to the console.
`Sql:BatchSize` controls both the SharePoint page size and the number of rows sent per SQL bulk-copy batch. `Application:EnableMetrics` publishes the `SPO2SQL` OpenTelemetry meter and activity source; without `OTEL_EXPORTER_OTLP_ENDPOINT` both are written to the console.

Supply secrets outside source control. For local development:

Expand Down
45 changes: 32 additions & 13 deletions src/SPO2SQL.Console/Program.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using Microsoft.Extensions.Options;
using OpenTelemetry;
using OpenTelemetry.Metrics;
using OpenTelemetry.Trace;
using SPO2SQL.Configuration;
using SPO2SQL.Telemetry;

Expand Down Expand Up @@ -94,20 +95,38 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>

if (context.Configuration.GetValue<bool>("Application:EnableMetrics", true))
{
services.AddOpenTelemetry()
.WithMetrics(builder =>
bool useOtlp = !string.IsNullOrWhiteSpace(
Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT"));

var telemetry = services.AddOpenTelemetry();

telemetry.WithMetrics(builder =>
{
builder.AddMeter(SyncMetrics.MeterName);

if (useOtlp)
{
builder.AddOtlpExporter();
}
else
{
builder.AddConsoleExporter();
}
});

telemetry.WithTracing(builder =>
{
builder.AddSource(SyncTracing.SourceName);

if (useOtlp)
{
builder.AddOtlpExporter();
}
else
{
builder.AddMeter(SyncMetrics.MeterName);

if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT")))
{
builder.AddOtlpExporter();
}
else
{
builder.AddConsoleExporter();
}
});
builder.AddConsoleExporter();
}
});
}

services.AddSingleton<SyncExitCode>();
Expand Down
6 changes: 6 additions & 0 deletions src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System.Diagnostics;
using SPO2SQL.Configuration;
using SPO2SQL.Logging;
using SPO2SQL.SharePoint;
Expand Down Expand Up @@ -87,6 +88,10 @@ public static async Task RefreshListsSQLAsync(
bool daily,
CancellationToken cancellationToken = default)
{
using Activity? activity = SyncTracing.Source.StartActivity("spo2sql.sync.list", ActivityKind.Internal);
activity?.SetTag(SyncTracing.ListTag, listName);
activity?.SetTag(SyncTracing.ModeTag, daily ? "incremental" : "full");

var statistics = new OperationStatistics { StartTime = DateTime.UtcNow };

try
Expand Down Expand Up @@ -147,6 +152,7 @@ await retryPolicy.ExecuteWithRetryAsync(
}
catch (Exception ex)
{
activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
Logger.LogError($"RefreshListsSQL: FATAL ERROR updating list '{listName}'.", ex);
throw;
}
Expand Down
12 changes: 11 additions & 1 deletion src/SPO2SQL.Core/SqlServer/SQLInteraction.cs
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
using System.Data;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using Microsoft.Data.SqlClient;
using Microsoft.SharePoint.Client;
using SPO2SQL.Logging;
using SPO2SQL.SharePoint;
using SPO2SQL.Telemetry;
using SPO2SQL.XmlConfig;

namespace SPO2SQL.SqlServer;
Expand Down Expand Up @@ -102,7 +104,11 @@ public async Task BuildAsync()
}

Logger.Log(1, "[Build] Initializing SharePoint list structure...");
await List.BuildAsync(CancellationToken).ConfigureAwait(false);
using (Activity? readActivity = SyncTracing.Source.StartActivity("spo2sql.sharepoint.read", ActivityKind.Client))
{
readActivity?.SetTag(SyncTracing.ListTag, List.Name);
await List.BuildAsync(CancellationToken).ConfigureAwait(false);
}

FNDictionary = new Dictionary<string, Field>(StringComparer.OrdinalIgnoreCase);
BuildDictionary();
Expand Down Expand Up @@ -334,6 +340,10 @@ private async Task TransferDataAsync(DateTime snapDate)

string safeTable = $"[{TableName.Replace("]", "]]")}]";

using Activity? writeActivity = SyncTracing.Source.StartActivity("spo2sql.sql.bulk_copy", ActivityKind.Client);
writeActivity?.SetTag(SyncTracing.ListTag, TableName);
writeActivity?.SetTag(SyncTracing.RowsTag, batch.Rows.Count);

using var bulkCopy = new SqlBulkCopy(
Connection!,
SqlBulkCopyOptions.TableLock |
Expand Down
18 changes: 18 additions & 0 deletions src/SPO2SQL.Core/Telemetry/SyncTracing.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
using System.Diagnostics;

namespace SPO2SQL.Telemetry;

/// <summary>
/// Activity source used to trace list synchronization, SharePoint reads and SQL writes.
/// The host subscribes to it when metrics/telemetry are enabled.
/// </summary>
public static class SyncTracing
{
public const string SourceName = "SPO2SQL";

public static readonly ActivitySource Source = new(SourceName, "2.0.0");

public const string ListTag = "spo2sql.list";
public const string ModeTag = "spo2sql.mode";
public const string RowsTag = "spo2sql.rows";
}
31 changes: 31 additions & 0 deletions tests/SPO2SQL.Core.Tests/SyncTracingTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
using System.Diagnostics;
using FluentAssertions;
using SPO2SQL.Telemetry;
using Xunit;

namespace SPO2SQL.Core.Tests;

public class SyncTracingTests
{
[Fact]
public void Source_EmitsActivitiesToListeners()
{
Activity? captured = null;

using var listener = new ActivityListener
{
ShouldListenTo = source => source.Name == SyncTracing.SourceName,
Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllDataAndRecorded,
ActivityStarted = activity => captured = activity
};

ActivitySource.AddActivityListener(listener);

using var activity = SyncTracing.Source.StartActivity("spo2sql.sync.list", ActivityKind.Internal);
activity?.SetTag(SyncTracing.ListTag, "Activities");

captured.Should().NotBeNull();
captured!.OperationName.Should().Be("spo2sql.sync.list");
captured.GetTagItem(SyncTracing.ListTag).Should().Be("Activities");
}
}
Loading