diff --git a/.editorconfig b/.editorconfig
index df666d6..febd864 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -35,7 +35,7 @@ trim_trailing_whitespace = false
[*.cs]
# New line preferences
-end_of_line = crlf
+end_of_line = lf
insert_final_newline = true
# Indentation preferences
diff --git a/.gitattributes b/.gitattributes
index 1ff0c42..f33c92b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3,6 +3,10 @@
###############################################################################
* text=auto
+# C# sources are normalized to LF so dotnet format behaves identically on
+# Windows and Linux agents.
+*.cs text eol=lf
+
###############################################################################
# Set default behavior for command prompt diff.
#
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cfe6005..d7df891 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,8 +12,12 @@ env:
jobs:
build:
- name: Build & Test
- runs-on: windows-latest
+ name: Build & Test (${{ matrix.os }})
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ ubuntu-latest, windows-latest ]
timeout-minutes: 15
steps:
@@ -54,4 +58,4 @@ jobs:
retention-days: 14
- name: Audit NuGet packages
- run: dotnet nuget audit
+ run: dotnet restore --force /p:NuGetAudit=true /p:NuGetAuditMode=all /p:NuGetAuditLevel=moderate
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a8dee01..b45d301 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -56,5 +56,5 @@ format:
nuget_audit:
stage: security
script:
- - dotnet nuget audit
+ - dotnet restore --force /p:NuGetAudit=true /p:NuGetAuditMode=all /p:NuGetAuditLevel=moderate
allow_failure: true
diff --git a/Directory.Build.props b/Directory.Build.props
index 48a8fbf..a7f00b1 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -11,6 +11,11 @@
2.0.0.0
2.0.0
spo2sql-modernization-secrets
- $(NoWarn);CS8618;CS8600;CS8601;CS8602;CS8603;CS8604;CS8625;CS8605;CS8605;CA1416;NU1603;CS0618;CS0649;CA1305;CA1310;CA1311;CA1304;CA1846;CA1862;CA1866;CA5350;CA2201;CA1805
+ $(NoWarn);NU1603
+
+
+
+
+
diff --git a/README.md b/README.md
index dcab547..5a7167f 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@
- Per-list column mapping, destination names, data types, and ignore rules.
- Retry handling for transient SharePoint and network failures.
- Startup health checks for SharePoint and SQL connectivity.
-- Operation statistics, structured logging, and cancellation support.
+- End-to-end async I/O with cancellation, per-list operation statistics and structured logging.
- Credential providers for environment variables, Windows DPAPI, and application configuration.
- Automated build, test, formatting, dependency-audit, and CodeQL workflows.
@@ -53,7 +53,9 @@ docs/ Configuration, security, modernization, and operational
- SQL Server access with permission to read source metadata and create/update destination tables
- A SharePoint account or supported application identity configured for the target tenant
-The current authentication path uses username/password token acquisition. Environments requiring interactive MFA or Conditional Access should use an app-only/service-principal authentication design instead of relying on password-based authentication.
+Two authentication modes are supported. For unattended jobs, configure app-only authentication by setting `SharePoint:TenantId`, `SharePoint:ClientId` and `SharePoint:CertificatePath` (plus `CertificatePassword` when the PFX is protected); this uses client credentials with a certificate and is not blocked by MFA or Conditional Access. When those values are absent, the legacy username/password flow is used as a fallback, which will fail in tenants that enforce MFA.
+
+Run `dotnet run -- --setup-secrets` to store credentials in the current user's DPAPI-protected store (Windows only).
## Build and test
@@ -98,7 +100,11 @@ The console application binds these sections:
"Password": "",
"TimeoutSeconds": 120,
"MaxRetries": 3,
- "InitialRetryDelayMs": 1000
+ "InitialRetryDelayMs": 1000,
+ "TenantId": "",
+ "ClientId": "",
+ "CertificatePath": "",
+ "CertificatePassword": ""
},
"Sql": {
"ConnectionString": "Server=.;Database=SPO2SQL;Integrated Security=true;Encrypt=true;TrustServerCertificate=false;",
@@ -160,7 +166,7 @@ The detailed security guidance is available in [`docs/SECURITY.md`](docs/SECURIT
## CI/CD
-GitHub Actions runs on pushes and pull requests:
+GitHub Actions runs on pushes and pull requests on both Linux and Windows:
1. Restore NuGet dependencies.
2. Build with warnings treated as errors.
diff --git a/UPGRADE_SUMMARY.md b/UPGRADE_SUMMARY.md
index 246d371..3358eb7 100644
--- a/UPGRADE_SUMMARY.md
+++ b/UPGRADE_SUMMARY.md
@@ -181,13 +181,17 @@ Logger (Enhanced)
---
-## Future Improvements
-
-- [ ] Add async/await support for SharePoint operations
-- [ ] Implement connection pooling for SQL Server
-- [ ] Add retry logic with exponential backoff
-- [ ] Create configuration service layer
-- [ ] Add metrics/performance monitoring
-- [ ] Implement proper dependency injection
-- [ ] Add unit test suite
-- [ ] Add telemetry for debugging
+## Status of Previously Planned Improvements
+
+This summary describes an earlier modernization pass. The items below were
+subsequently delivered in the current codebase (see `docs/PROGRESS.md`):
+
+- [x] Async/await for SharePoint and SQL operations, with cancellation
+- [x] Retry logic with exponential backoff and jitter (`RetryPolicy`)
+- [x] Configuration service layer (`SyncSettings` plus validated options)
+- [x] Per-list metrics logging via `OperationStatistics`
+- [x] Dependency injection in the .NET host
+- [x] Unit test suite across both projects
+- [x] App-only authentication, real health checks and warning-free nullable build
+
+Still open: OpenTelemetry export and live integration tests.
diff --git a/docs/LoggerMigrationGuide.md b/docs/LoggerMigrationGuide.md
deleted file mode 100644
index f0f3051..0000000
--- a/docs/LoggerMigrationGuide.md
+++ /dev/null
@@ -1,492 +0,0 @@
-# Logger Migration Guide: From Legacy Logger to ILogger
-
-## Overview
-
-This guide helps you migrate from the legacy static `Logger` class to Microsoft's modern `ILogger` interface, which is the standard logging abstraction in .NET.
-
----
-
-## Key Differences
-
-| Aspect | Legacy Logger | Modern ILogger |
-|--------|--------------|------------------|
-| **Access Pattern** | Static class (`Logger.LogError(...)`) | Dependency injection (`_logger.LogError(...)`) |
-| **Verbosity Control** | Custom integer levels (0-3) | Standard `LogLevel` enum |
-| **Testability** | Difficult (static dependencies) | Easy (inject mock logger) |
-| **Configuration** | Manual property setting | Configured via `appsettings.json` |
-| **Providers** | Console only | Console, File, Event Log, Application Insights, etc. |
-| **Type Safety** | None | Generic type parameter identifies log source |
-| **Structured Logging** | No | Yes (with parameters) |
-| **Scoping** | No | Yes (with `BeginScope()`) |
-| **Performance** | Basic | Optimized with source generators |
-
----
-
-## Verbosity Level Migration
-
-### Legacy Verbosity Levels
-```csharp
-// Old: 0=silent, 1=errors, 2=warnings, 3=debug
-Logger.VerboseLevel = 2; // Show errors and warnings only
-```
-
-### Modern LogLevel Mapping
-```csharp
-// New: Configure in appsettings.json
-{
- "Logging": {
- "LogLevel": {
- "Default": "Warning",
- "Microsoft": "Warning",
- "Bring": "Information"
- }
- }
-}
-```
-
-**Mapping Table:**
-
-| Legacy VerboseLevel | Modern LogLevel | What Gets Logged |
-|---------------------|-----------------|------------------|
-| 0 (silent) | `None` | Nothing |
-| 1 (errors only) | `Error` | Errors only |
-| 2 (warnings+) | `Warning` | Errors and warnings |
-| 3 (debug+) | `Debug` or `Trace` | Everything including debug info |
-
----
-
-## Log Level Migration Examples
-
-### 1. Error Logging
-
-**Before (Legacy):**
-```csharp
-using Bring.SPODataQuality;
-
-try
-{
- // ... some operation
-}
-catch (Exception ex)
-{
- Logger.LogError("Failed to connect to SharePoint", ex);
-}
-```
-
-**After (Modern):**
-```csharp
-using Microsoft.Extensions.Logging;
-
-public class MyService
-{
- private readonly ILogger _logger;
-
- public MyService(ILogger logger)
- {
- _logger = logger;
- }
-
- public void DoWork()
- {
- try
- {
- // ... some operation
- }
- catch (Exception ex)
- {
- _logger.LogError(ex, "Failed to connect to SharePoint");
- }
- }
-}
-```
-
-**Key Improvements:**
-- Exception is the first parameter (better IntelliSense)
-- Automatic stack trace inclusion
-- Type-safe: `ILogger` identifies log source
-- Dependency injection enables testing
-
----
-
-### 2. Warning Logging
-
-**Before (Legacy):**
-```csharp
-Logger.LogWarning("No items found in list 'Documents'");
-```
-
-**After (Modern):**
-```csharp
-_logger.LogWarning("No items found in list '{ListName}'", "Documents");
-```
-
-**Key Improvements:**
-- Structured logging with named parameters
-- Parameters extracted for log aggregation
-- Better searchability in log management systems
-
----
-
-### 3. Debug Logging
-
-**Before (Legacy):**
-```csharp
-Logger.LogDebug($"Processing item {itemId} with status {status}");
-```
-
-**After (Modern):**
-```csharp
-_logger.LogDebug("Processing item {ItemId} with status {Status}", itemId, status);
-```
-
-**Key Improvements:**
-- No string interpolation needed
-- Parameters logged separately (structured logging)
-- Better performance (no allocation if debug logging disabled)
-- Searchable by ItemId or Status in log analytics
-
----
-
-### 4. Information Logging (New!)
-
-The legacy Logger didn't have an info level. Use this for important operational messages:
-
-```csharp
-_logger.LogInformation("SharePoint sync started at {StartTime}", DateTime.Now);
-_logger.LogInformation("Processed {ItemCount} items in {Duration}ms", count, elapsed);
-```
-
----
-
-## Common Migration Patterns
-
-### Pattern 1: Static Logger Usage Throughout Class
-
-**Before:**
-```csharp
-public class SharePointService
-{
- public void DownloadFiles()
- {
- Logger.LogDebug("Starting file download");
-
- try
- {
- // ... download logic
- Logger.LogDebug("Downloaded 5 files successfully");
- }
- catch (Exception ex)
- {
- Logger.LogError("Download failed", ex);
- }
- }
-}
-```
-
-**After:**
-```csharp
-public class SharePointService
-{
- private readonly ILogger _logger;
-
- public SharePointService(ILogger logger)
- {
- _logger = logger;
- }
-
- public void DownloadFiles()
- {
- _logger.LogDebug("Starting file download");
-
- try
- {
- // ... download logic
- _logger.LogInformation("Downloaded {FileCount} files successfully", 5);
- }
- catch (Exception ex)
- {
- _logger.LogError(ex, "Download failed");
- }
- }
-}
-```
-
----
-
-### Pattern 2: Verbosity Level Checking
-
-**Before:**
-```csharp
-if (Logger.VerboseLevel >= 3)
-{
- Logger.LogDebug($"Details: {expensiveOperation()}");
-}
-```
-
-**After:**
-```csharp
-// Option 1: Let ILogger handle it (recommended)
-_logger.LogDebug("Details: {Details}", expensiveOperation());
-
-// Option 2: Explicit check for expensive operations
-if (_logger.IsEnabled(LogLevel.Debug))
-{
- _logger.LogDebug("Details: {Details}", expensiveOperation());
-}
-```
-
-**Note:** The second approach only necessary if `expensiveOperation()` is truly expensive, as ILogger optimizes away disabled log levels.
-
----
-
-### Pattern 3: Custom Log Level Method
-
-**Before:**
-```csharp
-Logger.Log(2, "Custom warning message");
-```
-
-**After:**
-```csharp
-_logger.Log(LogLevel.Warning, "Custom warning message");
-
-// Or use the specific helper:
-_logger.LogWarning("Custom warning message");
-```
-
----
-
-## Using the LoggerAdapter (Transition Strategy)
-
-If you have a large codebase and can't migrate everything at once, use the `LoggerAdapter`:
-
-```csharp
-using Bring.Adapters;
-using Microsoft.Extensions.Logging;
-
-public class LegacyService
-{
- public LegacyService(ILogger modernLogger)
- {
- // Wrap modern logger to support legacy Logger API
- var adapter = new LoggerAdapter(modernLogger);
-
- // Configure legacy verbosity level
- adapter.VerboseLevel = 3;
-
- // Now legacy code can work unchanged
- adapter.LogError("Legacy error message");
- adapter.LogWarning("Legacy warning message");
- adapter.LogDebug("Legacy debug message");
- }
-}
-```
-
-**When to use the adapter:**
-- Large codebase with many Logger references
-- Gradual migration strategy
-- Testing migration incrementally
-- Supporting legacy code that can't be changed immediately
-
-**When NOT to use the adapter:**
-- New code (use ILogger directly)
-- Small codebases (just migrate directly)
-- When you need structured logging features
-
----
-
-## Configuration Migration
-
-### Legacy Configuration
-```csharp
-// Somewhere in Program.cs or startup
-Logger.VerboseLevel = args.Contains("--verbose") ? 3 : 1;
-```
-
-### Modern Configuration
-
-**appsettings.json:**
-```json
-{
- "Logging": {
- "LogLevel": {
- "Default": "Information",
- "Microsoft": "Warning",
- "Microsoft.Hosting.Lifetime": "Information",
- "Bring": "Debug"
- },
- "Console": {
- "FormatterName": "simple",
- "FormatterOptions": {
- "TimestampFormat": "yyyy-MM-dd HH:mm:ss.fff ",
- "SingleLine": true,
- "IncludeScopes": false
- }
- }
- }
-}
-```
-
-**appsettings.Development.json:**
-```json
-{
- "Logging": {
- "LogLevel": {
- "Default": "Debug",
- "Bring": "Trace"
- }
- }
-}
-```
-
-**Command-line override:**
-```bash
-dotnet run --Logging:LogLevel:Default=Debug
-```
-
----
-
-## Best Practices
-
-### ✅ DO
-
-1. **Use structured logging:**
- ```csharp
- _logger.LogInformation("User {UserId} logged in from {IpAddress}", userId, ip);
- ```
-
-2. **Pass exceptions as first parameter:**
- ```csharp
- _logger.LogError(ex, "Failed to process {FileName}", fileName);
- ```
-
-3. **Use appropriate log levels:**
- - `Trace`: Very detailed (e.g., entering/exiting methods)
- - `Debug`: Developer information during debugging
- - `Information`: Important business events
- - `Warning`: Recoverable errors or unexpected situations
- - `Error`: Failures that stop current operation
- - `Critical`: Catastrophic failures requiring immediate attention
-
-4. **Check log level for expensive operations:**
- ```csharp
- if (_logger.IsEnabled(LogLevel.Debug))
- {
- _logger.LogDebug("Complex data: {Data}", SerializeComplexObject());
- }
- ```
-
-### ❌ DON'T
-
-1. **Don't use string interpolation:**
- ```csharp
- // BAD - loses structured logging
- _logger.LogInformation($"Processing {itemId}");
-
- // GOOD - structured logging
- _logger.LogInformation("Processing {ItemId}", itemId);
- ```
-
-2. **Don't catch exceptions just to log them:**
- ```csharp
- // BAD
- try { DoWork(); }
- catch (Exception ex) { _logger.LogError(ex, "Error"); throw; }
-
- // GOOD - let exceptions bubble, log at appropriate handler level
- ```
-
-3. **Don't log sensitive data:**
- ```csharp
- // BAD
- _logger.LogDebug("Password: {Password}", password);
-
- // GOOD
- _logger.LogDebug("Authentication attempted for user {Username}", username);
- ```
-
----
-
-## Testing with ILogger
-
-### Legacy Logger (Hard to Test)
-```csharp
-public void TestMethod()
-{
- // No way to verify logging without capturing Console.Out
- var service = new SharePointService();
- service.DoWork();
- // Can't assert what was logged
-}
-```
-
-### Modern ILogger (Easy to Test)
-```csharp
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
-using Xunit;
-
-public class SharePointServiceTests
-{
- [Fact]
- public void DoWork_LogsInformation()
- {
- // Arrange
- var logger = new FakeLogger();
- var service = new SharePointService(logger);
-
- // Act
- service.DoWork();
-
- // Assert
- Assert.Contains(logger.LoggedMessages,
- m => m.Level == LogLevel.Information && m.Message.Contains("Downloaded"));
- }
-
- [Fact]
- public void DoWork_WithNullLogger_StillWorks()
- {
- // NullLogger for tests where logging doesn't matter
- var service = new SharePointService(NullLogger.Instance);
- service.DoWork(); // No exceptions
- }
-}
-```
-
----
-
-## Migration Checklist
-
-- [ ] Update all service constructors to accept `ILogger`
-- [ ] Replace `Logger.LogError()` with `_logger.LogError()`
-- [ ] Replace `Logger.LogWarning()` with `_logger.LogWarning()`
-- [ ] Replace `Logger.LogDebug()` with `_logger.LogDebug()`
-- [ ] Replace `Logger.Log()` with `_logger.Log()`
-- [ ] Convert string interpolation to structured logging parameters
-- [ ] Update exception logging to pass exception as first parameter
-- [ ] Configure logging in `appsettings.json` instead of `Logger.VerboseLevel`
-- [ ] Update unit tests to inject logger mocks
-- [ ] Remove `using Bring.SPODataQuality;` (legacy namespace)
-- [ ] Add `using Microsoft.Extensions.Logging;`
-- [ ] Consider using `LoggerAdapter` for gradual migration
-
----
-
-## Additional Resources
-
-- [Microsoft Logging Documentation](https://learn.microsoft.com/en-us/dotnet/core/extensions/logging)
-- [High-performance logging](https://learn.microsoft.com/en-us/dotnet/core/extensions/high-performance-logging)
-- [Logging Best Practices](https://learn.microsoft.com/en-us/dotnet/core/extensions/logging-best-practices)
-- Project: `Application.cs` - Example of modern ILogger usage
-
----
-
-## Questions?
-
-If you encounter issues during migration, check:
-1. Is the service registered in DI? (see `Program.cs`)
-2. Is logging configured in `appsettings.json`?
-3. Are you using `ILogger` where T is your class name?
-4. Have you removed static `Logger` references?
-
-For gradual migration, use the `LoggerAdapter` class in `ConsoleApp1Net8/Adapters/LoggerAdapter.cs`.
diff --git a/docs/MODERNIZATION-STATUS.md b/docs/MODERNIZATION-STATUS.md
index 462e399..f9fbf4a 100644
--- a/docs/MODERNIZATION-STATUS.md
+++ b/docs/MODERNIZATION-STATUS.md
@@ -1,211 +1,39 @@
-# .NET 8 Modernization Status Report
+# Modernization Status
-**Last Updated**: 2026-03-18 14:25 UTC
+**Last updated:** 2026-11-18
-## Overall Progress
+## Summary
-**Completed**: 9 of 27 tasks (33%) ✅
-**In Progress**: 3 tasks (11%) 🚧
-**Pending**: 15 tasks (56%) 📋
+The modernization plan from the original migration documents has been completed.
+The previous phase percentages in this document were snapshots of an older state
+and no longer describe the codebase.
----
+| Original phase | Outcome |
+| --- | --- |
+| Foundation & infrastructure | Completed: .NET 8 host, DI, options, multi-source configuration |
+| Core services modernization | Completed: async SharePoint/SQL/DQ pipeline, cancellation, paging |
+| Modern C# features | Completed: records, pattern matching, file-scoped namespaces, nullable enabled |
+| Architecture | Completed: `SyncSettings`, credential chain, health checks, retry policy |
+| Security | Completed: app-only certificate authentication, encrypted connection enforcement |
+| Testing | Unit suite in place; live integration tests remain environment-specific |
+| Documentation | README, configuration guide and security guide updated |
-## ✅ Completed Tasks (9)
-
-### Phase 1: Foundation & Infrastructure (100% Complete)
-1. ✅ **Add Modern NuGet Dependencies** - All Microsoft.Extensions.* packages added, Polly integrated
-2. ✅ **Create New Configuration System** - IOptions with strongly-typed records (SharePoint, SQL, Application)
-3. ✅ **Setup Dependency Injection** - IHostBuilder with full service registration
-4. ✅ **Implement Global Usings** - Common namespaces centralized in GlobalUsings.cs
-
-### Phase 2: Logging & Configuration (100% Complete)
-5. ✅ **Implement Structured Logging** - Production-ready ILogger with:
- - LoggerMessage source generators (zero-allocation logging)
- - Correlation IDs for operation tracking
- - Log scopes for contextual information
- - 12 semantic log methods with EventIds
- - Application.cs grew from 72 to 298 lines with comprehensive logging
-
-### Phase 3: Modern C# Features (67% Complete)
-6. ✅ **Use File-Scoped Namespaces** - All files already using modern syntax
-7. ✅ **Implement Pattern Matching** - Comprehensive OperationResult utility (416 lines):
- - Success/Failure discriminated union pattern
- - Switch expressions, property patterns, list patterns
- - 10+ extension methods demonstrating advanced patterns
- - ValidationResult and ResultSummary types
- - 12 usage examples in Application.cs
-
-### Phase 5: Security (50% Complete)
-8. ✅ **Modernize Configuration** - Enhanced Application.cs with best practices
-9. ✅ **Secure Credential Management** - Complete security infrastructure:
- - User Secrets for development
- - Environment variables for production
- - SECURITY.md (578 lines) with comprehensive best practices
- - setup-secrets.sh (bash) and setup-secrets.ps1 (PowerShell) scripts
- - Credential rotation procedures
- - Audit logging recommendations
-
----
-
-## 🚧 In Progress (3)
-
-### Phase 2: Core Services
-10. 🚧 **Modernize Logger Usage** - Creating migration guide and adapter pattern
-11. 🚧 **Modern LINQ Features** - Building LinqExtensions with .NET 6+ features
-12. 🚧 **Use Records for DTOs** - Creating record-based data models
-
----
-
-## 📋 Pending Tasks (15)
-
-### Phase 2: Core Services Modernization
-- Convert SharePoint Components to Async (SPOUser, SPOList, Context)
-- Convert SQL Components to Async (SQLInteraction, RefreshSQLLists)
-- Convert Data Quality Components to Async
-
-### Phase 3: Modern C# Features
-- Enable Nullable Reference Types fully
-
-### Phase 4: Architecture Improvements
-- Create Service Interfaces (ISharePointService, ISqlService, IDataQualityService)
-- Refactor to Service Pattern
-- Enhance Retry with Polly
-- Improve Health Checks
-
-### Phase 5: Security
-- Enhance Connection Security
-
-### Phase 6: Testing & Quality
-- Add Unit Tests Foundation
-- Add Integration Tests
-- Improve Code Quality
-
-### Phase 7: Documentation
-- Update Main Documentation
-- Create Config Migration Tools
-- Create Deployment Guide
-
----
-
-## 📊 Key Achievements
-
-### Code Metrics
-- **Application.cs**: 72 → 298 lines (314% growth with production features)
-- **OperationResult.cs**: 416 lines of reusable pattern matching utilities
-- **SECURITY.md**: 578 lines of comprehensive security guidance
-- **New Files Created**: 15+
-- **Scripts**: 2 (PowerShell + Bash for secret setup)
-
-### Architecture Improvements
-1. **Dependency Injection**: Full IHost integration with service registration
-2. **Configuration**: Multi-layer (JSON → Secrets → Env Vars → CLI)
-3. **Logging**: Zero-allocation LoggerMessage with correlation tracking
-4. **Security**: Never-commit-secrets architecture
-5. **Modern C#**: Records, pattern matching, file-scoped namespaces
-
-### Developer Experience
-- ✅ Clear separation of concerns
-- ✅ Type-safe configuration with validation
-- ✅ Comprehensive XML documentation
-- ✅ Security-first defaults
-- ✅ Production-ready error handling
-
----
-
-## 🎯 Next Steps
-
-### Immediate (Wave 3 - In Progress)
-1. Complete logger migration adapter
-2. Add modern LINQ utilities
-3. Create record-based DTOs
-
-### Wave 4 (Ready to Start)
-1. Async conversion of SharePoint components
-2. Async conversion of SQL components
-3. Nullable reference type enforcement
-
-### Wave 5 (Dependent on Wave 4)
-1. Service interfaces and patterns
-2. Polly integration for resilience
-3. Health check improvements
-
----
-
-## 📁 Project Structure
+## Architecture today
```
-SPOtoSQL-Net8/
-├── ConsoleApp1Net8/
-│ ├── Configuration/ ✅ Strongly-typed options
-│ │ ├── ApplicationOptions.cs
-│ │ ├── SharePointOptions.cs
-│ │ └── SqlOptions.cs
-│ ├── Utilities/ ✅ Modern C# utilities
-│ │ └── OperationResult.cs (Pattern matching, 416 lines)
-│ ├── Models/ 🚧 Record-based DTOs (in progress)
-│ ├── Adapters/ 🚧 Logger adapter (in progress)
-│ ├── Program.cs ✅ IHost with DI
-│ ├── Application.cs ✅ Structured logging (298 lines)
-│ ├── GlobalUsings.cs ✅ Common namespaces
-│ ├── appsettings.json ✅ Default configuration
-│ ├── appsettings.Development.json ✅ Dev overrides
-│ └── ConsoleApp1Net8.csproj ✅ Modern packages
-├── scripts/ ✅ Setup automation
-│ ├── setup-secrets.sh ✅ Bash script
-│ └── setup-secrets.ps1 ✅ PowerShell script
-├── docs/ 🚧 Documentation (in progress)
-├── SECURITY.md ✅ 578 lines of security best practices
-├── README-MODERNIZATION.md ✅ Setup and migration guide
-├── PROGRESS.md ✅ Detailed progress tracking
-└── MODERNIZATION-STATUS.md ✅ This file
-
-Legacy Code (to be modernized):
-SPOtoSQL-Snapshots/ConsoleApp1/ 📋 Original .NET Framework 4.8 code
+src/SPO2SQL.Console Host, options, validation, health checks, exit code, CLI setup
+src/SPO2SQL.Core SharePoint access, SQL transfer, credentials, XML list mappings
+tests/ xUnit unit tests for both projects
```
----
-
-## 🚀 Fleet Mode Performance
-
-**Active Parallel Agents**: 3 (modernize-logger-usage, modern-linq, use-records)
-**Completed Agent Runs**: 6
-**Total Agent Time**: ~15 minutes
-**Actual Elapsed Time**: ~6 minutes (2.5x speedup from parallelization)
-
----
-
-## 💡 Lessons Learned
-
-1. **Configuration First**: Having strong configuration foundation enables everything else
-2. **Logging Matters**: Investing in proper logging pays dividends in production
-3. **Records are Powerful**: Pattern matching with records dramatically improves code clarity
-4. **Security by Default**: User Secrets + Environment Variables = zero secrets in source
-5. **Parallel Execution**: Fleet mode accelerated development significantly
-
----
-
-## 📈 Quality Indicators
-
-- ✅ All configuration validated on startup
-- ✅ Comprehensive XML documentation
-- ✅ Security best practices documented
-- ✅ Zero secrets in source code
-- ✅ Production-ready logging patterns
-- ✅ Modern C# features consistently applied
-- ✅ Backward compatibility maintained (legacy code still works)
-
----
-
-## 🎓 Learning Resources Created
-
-1. **SECURITY.md** - Complete security guide for production deployments
-2. **README-MODERNIZATION.md** - Setup and configuration guide
-3. **Application.cs** - 12 logging examples showing best practices
-4. **OperationResult.cs** - 10+ pattern matching examples
-5. **Setup Scripts** - Automated secret configuration
+The host maps validated options into `SyncSettings` once and passes it through the
+sync pipeline. The legacy XML file is used only for list/column mappings; secrets
+come from the credential provider chain (environment, DPAPI on Windows, or the
+values supplied through configuration/user secrets).
----
+## Known limitations
-*For detailed task breakdown, see PROGRESS.md*
-*For security guidelines, see SECURITY.md*
-*For setup instructions, see README-MODERNIZATION.md*
+- Username/password authentication is retained as a fallback and will not work in
+ tenants that require interactive MFA. Use app-only authentication instead.
+- Metrics are logged per list; there is no metrics exporter yet.
+- `dotnet format` expects LF line endings, enforced for C# files in `.gitattributes`.
diff --git a/docs/MODERNIZATION.md b/docs/MODERNIZATION.md
index 37c9e14..93a529d 100644
--- a/docs/MODERNIZATION.md
+++ b/docs/MODERNIZATION.md
@@ -6,12 +6,12 @@ This document explains the modern .NET 8 architecture and how to configure and r
### Modern Architecture
- **Dependency Injection**: Full DI container with IHost pattern
-- **Async/Await**: All I/O operations use async patterns (work in progress)
-- **Structured Logging**: ILogger with semantic log properties
-- **Configuration**: appsettings.json with IOptions pattern
-- **Security**: User Secrets (dev) and Environment Variables (prod) for credentials
-- **Resilience**: Polly integration for retry policies (coming soon)
-- **Testing**: Unit and integration test support (coming soon)
+- **Async/Await**: All SharePoint, SQL and data-quality I/O uses async patterns with cancellation
+- **Structured Logging**: ILogger in the host, bridged to the Core library
+- **Configuration**: appsettings.json with validated IOptions, mapped to a single `SyncSettings` record
+- **Security**: User Secrets (dev), environment variables and app-only certificate authentication (prod)
+- **Resilience**: Cancellation-aware retry with exponential backoff and jitter
+- **Testing**: xUnit unit tests; integration tests require a live tenant/SQL Server
### Modern C# Features
- Records for immutable configuration
@@ -25,23 +25,19 @@ This document explains the modern .NET 8 architecture and how to configure and r
### Development Setup (User Secrets)
-**Quick Setup with Helper Script**:
+**Quick Setup**:
```bash
-# Interactive setup wizard
-./scripts/setup-secrets.sh
+# Stores credentials in the current user's DPAPI-protected store (Windows)
+dotnet run --project src/SPO2SQL.Console/SPO2SQL.Console.csproj -- --setup-secrets
```
-The setup script will:
-- ✅ Initialize User Secrets automatically
-- ✅ Guide you through configuration
-- ✅ Validate email addresses and connection strings
-- ✅ Ensure security best practices
+For development, user secrets remain the preferred option; see the manual setup below.
**Manual Setup**:
1. **Initialize User Secrets** (one-time setup):
```bash
-cd SPOtoSQL-Net8/ConsoleApp1Net8
+cd src/SPO2SQL.Console
dotnet user-secrets init
```
@@ -105,7 +101,7 @@ export SPO2SQL_Sql__ConnectionString="Server=myServer;Database=myDB;Integrated S
### Development
```bash
-cd SPOtoSQL-Net8/ConsoleApp1Net8
+cd src/SPO2SQL.Console
dotnet restore
dotnet build
dotnet run
@@ -115,7 +111,7 @@ dotnet run
```bash
dotnet publish -c Release -r win-x64 --self-contained false
cd bin/Release/net8.0/win-x64/publish
-./ConsoleApp1Net8.exe
+dotnet run --project src/SPO2SQL.Console/SPO2SQL.Console.csproj
```
## Logging
@@ -165,7 +161,7 @@ Available log levels:
Both versions can coexist:
- **.NET Framework 4.8**: `SPOtoSQL-Snapshots/ConsoleApp1/`
-- **.NET 8**: `SPOtoSQL-Net8/ConsoleApp1Net8/`
+- **.NET 8**: `src/SPO2SQL.Console/`
The .NET 8 version links to the same source files for now, so changes are shared.
@@ -189,25 +185,15 @@ Key security practices:
dotnet user-secrets set "SharePoint:Password" "new-password"
```
-### Helper Scripts
+### Credential Setup
-**Development Setup**:
```bash
-# Interactive setup wizard (Linux/Mac)
-./scripts/setup-secrets.sh
-
-# Interactive setup wizard (Windows)
-.\scripts\setup-secrets.ps1
-
-# Verify configuration
-./scripts/verify-config.sh
+# Windows: store credentials in the current user's DPAPI-protected store
+dotnet run --project src/SPO2SQL.Console/SPO2SQL.Console.csproj -- --setup-secrets
```
-These scripts help you:
-- ✅ Set up User Secrets securely
-- ✅ Validate input (email format, connection strings, etc.)
-- ✅ Verify configuration without exposing credentials
-- ✅ Check for common security issues
+Configuration is validated at startup by the host, so an invalid connection
+string or an incomplete authentication configuration fails fast with a clear message.
## Troubleshooting
diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md
index 86cdfb8..fd87283 100644
--- a/docs/PROGRESS.md
+++ b/docs/PROGRESS.md
@@ -1,188 +1,40 @@
-# .NET 8 Modernization Progress
-
-## Completed Tasks ✅
-
-### Phase 1: Foundation & Infrastructure (100% Complete)
-
-1. **Add Modern NuGet Dependencies** ✅
- - Added Microsoft.Extensions.* packages (DI, Configuration, Logging, Hosting, Options)
- - Updated SharePoint CSOM to 16.1.24816.12000
- - Updated SqlClient to 5.2.2
- - Added Polly 8.4.2 for resilience
- - Added Health Checks support
-
-2. **Create Configuration System** ✅
- - Created strongly-typed configuration records:
- - `SharePointOptions` - SP credentials and settings
- - `SqlOptions` - SQL connection and batch settings
- - `ApplicationOptions` - App behavior and logging
- - Implemented IOptions pattern with validation
- - Created appsettings.json with sensible defaults
- - Created appsettings.Development.json for dev overrides
- - Added User Secrets support (ID: spo2sql-modernization-secrets)
- - Environment variable support with SPO2SQL_ prefix
-
-3. **Setup Dependency Injection** ✅
- - Created modern Program.cs with IHostBuilder
- - Configured configuration sources (JSON, secrets, env vars, command line)
- - Registered options with validation
- - Created Application class as IHostedService
- - Implemented graceful startup and shutdown
-
-4. **Implement Global Usings** ✅
- - Created GlobalUsings.cs with common namespaces
- - Reduced repetitive using statements across project
-
-### Additional Improvements
-
-5. **Security Enhancements** ✅
- - User Secrets for development credentials
- - Environment variables for production
- - Created secrets.json.template for easy setup
- - Added .gitignore to prevent secret commits
- - Documented security best practices
-
-6. **Code Quality** ✅
- - Added .editorconfig with C# coding standards
- - Enabled nullable reference types
- - File-scoped namespace preference configured
- - Consistent formatting rules
-
-7. **Documentation** ✅
- - Created README-MODERNIZATION.md with:
- - Architecture overview
- - Configuration setup (dev & prod)
- - Migration guide from .NET Framework
- - Security best practices
- - Troubleshooting guide
- - Created PROGRESS.md (this file)
-
-## Next Tasks 🚧
-
-### Phase 2: Core Services Modernization
-
-1. **Modernize Logger** (Ready to start)
- - Replace Logger with ILogger
- - Update all classes to use injected logger
- - Add structured logging properties
-
-2. **Convert SharePoint Components to Async** (Ready to start)
- - Update SPOUser, SPOList, Context to async/await
- - Use ExecuteQueryAsync instead of ExecuteQuery
- - Add CancellationToken support
-
-3. **Convert SQL Components to Async** (Ready to start)
- - Update SQLInteraction to async patterns
- - Use async ADO.NET methods
- - Add CancellationToken support
-
-4. **Modernize Configuration Components** (Depends on logger)
- - Replace ConfigHelper usage with IOptions
- - Remove XML configuration dependencies
-
-5. **Convert Data Quality to Async** (Depends on SharePoint & SQL async)
- - Update DataQualityBase to async
- - Update all DQ classes (Activities, InvoiceRequest, Timesheet)
- - Update RefreshSPOLists orchestration
-
-### Phase 3: Modern C# Features
-
-- Use Records for DTOs
-- Implement Pattern Matching
-- Leverage Modern LINQ
-- Enable Nullable Reference Types fully
-- Use File-Scoped Namespaces throughout
-
-### Phase 4: Architecture
-
-- Create Service Interfaces
-- Refactor to Service Pattern
-- Enhance Retry with Polly
-- Improve Health Checks
-
-### Phase 5: Security
-
-- Implement secure credential management
-- Enhance connection security
-
-### Phase 6: Testing
-
-- Add unit tests
-- Add integration tests
-- Improve code quality metrics
-
-### Phase 7: Documentation
-
-- Update main README
-- Create config migration tools
-- Create deployment guide
-
-## Current Structure
-
-```
-SPOtoSQL-Net8/
-├── ConsoleApp1Net8/
-│ ├── Configuration/ # ✅ Modern config classes
-│ │ ├── ApplicationOptions.cs
-│ │ ├── SharePointOptions.cs
-│ │ └── SqlOptions.cs
-│ ├── GlobalUsings.cs # ✅ Global usings
-│ ├── Program.cs # ✅ Modern entry point with DI
-│ ├── Application.cs # ✅ Main app as hosted service
-│ ├── appsettings.json # ✅ Default configuration
-│ ├── appsettings.Development.json # ✅ Dev overrides
-│ ├── secrets.json.template # ✅ Template for User Secrets
-│ ├── .gitignore # ✅ Security
-│ └── ConsoleApp1Net8.csproj # ✅ Updated with modern packages
-├── README-MODERNIZATION.md # ✅ Modernization docs
-└── PROGRESS.md # ✅ This file
-
-SPOtoSQL-Snapshots/ # Legacy .NET Framework code
-└── ConsoleApp1/ # (linked for now, to be modernized)
-```
-
-## Statistics
-
-- **Completed Tasks**: 7 / 27 (26%)
-- **Phase 1 (Foundation)**: 4/4 (100%) ✅
-- **Phase 2 (Core Services)**: 0/5 (0%)
-- **Phase 3 (C# Features)**: 1/6 (17%)
-- **Phase 4 (Architecture)**: 0/4 (0%)
-- **Phase 5 (Security)**: 0/2 (0%)
-- **Phase 6 (Testing)**: 0/3 (0%)
-- **Phase 7 (Documentation)**: 0/3 (0%)
-- **Additional**: 3 extra improvements completed
-
-## Key Achievements
-
-1. **Modern Foundation**: The application now uses industry-standard .NET patterns
-2. **Security First**: No credentials in source code, proper secrets management
-3. **Configuration Flexibility**: JSON-based with multiple override mechanisms
-4. **Validation**: Configuration validates on startup with clear error messages
-5. **Logging Ready**: Structured logging infrastructure in place
-6. **Extensibility**: DI container makes testing and extending easier
-
-## How to Test Current Progress
+# Implementation Status
+
+## Current state (v2.0)
+
+The .NET 8 rewrite is complete and the legacy .NET Framework snapshot in
+`SPOtoSQL-Snapshots/` is kept only as a historical reference.
+
+| Area | Status |
+| --- | --- |
+| Host, dependency injection, options validation | Done |
+| Structured logging bridged from the Core library | Done |
+| SharePoint paging and cancellation | Done |
+| Transaction scoped to the write phase | Done |
+| UTC watermark for incremental syncs | Done |
+| Unified `SyncSettings` passed to the sync layer | Done |
+| App-only authentication with certificate | Done |
+| Real SharePoint/SQL startup health checks | Done |
+| `--setup-secrets` in the real console host | Done |
+| End-to-end async (CSOM, ADO.NET, retries, DQ) | Done |
+| Nullable reference types enforced without blanket suppressions | Done |
+| CI on Linux and Windows with format and NuGet audit | Done |
+
+## Validation
```bash
-cd SPOtoSQL-Net8/ConsoleApp1Net8
-
-# Setup secrets
-dotnet user-secrets set "SharePoint:Username" "test@example.com"
-dotnet user-secrets set "SharePoint:Password" "password"
-dotnet user-secrets set "SharePoint:SiteUrl" "https://test.sharepoint.com"
-dotnet user-secrets set "Sql:ConnectionString" "Server=test;Database=test;Integrated Security=true;"
-
-# Build and run
-dotnet restore
-dotnet build
-dotnet run
+dotnet restore SPO2SQL.sln
+dotnet build SPO2SQL.sln --no-restore -warnaserror
+dotnet test SPO2SQL.sln --no-build --no-restore
+dotnet format SPO2SQL.sln --verify-no-changes --no-restore
```
-Expected output:
-- Application starts
-- Logs show configuration loaded
-- Validates all settings
-- Application name and version displayed
-- Graceful shutdown
+The build is warning-free; the unit test suite covers CAML generation, retry
+behaviour (sync and async), configuration binding/validation, XML list mappings,
+logging routing, and exit-code propagation.
+
+## Remaining work
+- Integration tests against a real SharePoint tenant and SQL Server.
+- Observability export (OpenTelemetry) behind `Application:EnableMetrics`.
+- Replace the in-house `RetryPolicy` with a Polly resilience pipeline.
diff --git a/docs/SECURITY.md b/docs/SECURITY.md
index 26b2a49..3866f22 100644
--- a/docs/SECURITY.md
+++ b/docs/SECURITY.md
@@ -28,7 +28,7 @@ This document outlines security best practices for deploying and operating the S
```bash
# Navigate to project directory
-cd SPOtoSQL-Net8/ConsoleApp1Net8
+cd src/SPO2SQL.Console
# Set credentials (stored encrypted outside project directory)
dotnet user-secrets set "SharePoint:Username" "dev@company.com"
@@ -549,7 +549,7 @@ The application automatically redacts:
The `scripts/` directory contains tools to assist with secure configuration:
-- **setup-secrets.sh** / **setup-secrets.ps1**: Interactive script to configure User Secrets
+- **`dotnet run -- --setup-secrets`**: Interactive command to store credentials in the Windows DPAPI-protected user store; on other platforms use environment variables or user secrets.
- **verify-config.sh**: Validates configuration without exposing credentials
### Documentation
diff --git a/docs/Scripts-README.md b/docs/Scripts-README.md
deleted file mode 100644
index ff412b1..0000000
--- a/docs/Scripts-README.md
+++ /dev/null
@@ -1,258 +0,0 @@
-# Helper Scripts
-
-This directory contains utility scripts to help with secure configuration and deployment of the SharePoint to SQL Sync Tool.
-
-## Scripts Overview
-
-### setup-secrets.sh (Linux/Mac)
-
-Interactive script to configure User Secrets for development.
-
-**Usage**:
-```bash
-./scripts/setup-secrets.sh
-```
-
-**Features**:
-- ✅ Validates .NET SDK installation
-- ✅ Initializes User Secrets automatically
-- ✅ Validates email format for SharePoint username
-- ✅ Validates SharePoint URL format
-- ✅ Validates SQL connection string format
-- ✅ Checks for security best practices (encryption, password length)
-- ✅ Masks passwords in output
-- ✅ Provides guidance for next steps
-
-**When to use**:
-- First-time setup on a development machine
-- Updating credentials
-- Migrating from XML configuration
-
-### setup-secrets.ps1 (Windows PowerShell)
-
-Windows PowerShell version of the setup script with identical functionality.
-
-**Usage**:
-```powershell
-.\scripts\setup-secrets.ps1
-```
-
-**Features**:
-- Same as setup-secrets.sh but optimized for Windows
-- Uses `Read-Host -AsSecureString` for password input
-- PowerShell-native error handling
-
-### verify-config.sh
-
-Verifies that all required configuration is present and valid.
-
-**Usage**:
-```bash
-# Verify User Secrets (development)
-./scripts/verify-config.sh
-
-# Verify Environment Variables (production)
-./scripts/verify-config.sh --env-vars
-```
-
-**Checks performed**:
-- ✅ All required configuration keys are present
-- ✅ Email format validation
-- ✅ URL format validation
-- ✅ Connection string format validation
-- ✅ Security checks (encryption, TrustServerCertificate)
-- ✅ Password length validation
-- ✅ Scans appsettings.json for accidentally committed secrets
-
-**Exit codes**:
-- `0` - Configuration is valid
-- `1` - Configuration has errors
-
-**When to use**:
-- Before running the application
-- After changing configuration
-- In CI/CD pipelines to validate deployment
-- Troubleshooting configuration issues
-
-## Quick Start
-
-### First-Time Development Setup
-
-1. **Run the setup wizard**:
- ```bash
- cd SPOtoSQL-Net8
- ./scripts/setup-secrets.sh
- ```
-
-2. **Follow the prompts** to enter:
- - SharePoint username (email)
- - SharePoint password
- - SharePoint site URL
- - SQL Server connection details
-
-3. **Verify configuration**:
- ```bash
- ./scripts/verify-config.sh
- ```
-
-4. **Run the application**:
- ```bash
- cd ConsoleApp1Net8
- dotnet run
- ```
-
-### Production Deployment
-
-1. **Set environment variables** (see SECURITY.md for details):
- ```bash
- export SPO2SQL_SharePoint__Username="user@company.com"
- export SPO2SQL_SharePoint__Password="your-password"
- export SPO2SQL_SharePoint__SiteUrl="https://company.sharepoint.com/sites/prod"
- export SPO2SQL_Sql__ConnectionString="Server=...;Database=...;Encrypt=true;..."
- ```
-
-2. **Verify configuration**:
- ```bash
- ./scripts/verify-config.sh --env-vars
- ```
-
-3. **Deploy and run**:
- ```bash
- dotnet publish -c Release
- cd bin/Release/net8.0/publish
- ./ConsoleApp1Net8
- ```
-
-## Troubleshooting
-
-### "dotnet: command not found"
-
-**Problem**: .NET SDK is not installed or not in PATH.
-
-**Solution**: Install .NET 8 SDK from https://dotnet.microsoft.com/download
-
-### "Project file not found"
-
-**Problem**: Scripts are being run from the wrong directory.
-
-**Solution**: Always run scripts from the repository root:
-```bash
-cd /path/to/SPO2SQL
-./SPOtoSQL-Net8/scripts/setup-secrets.sh
-```
-
-Or from the SPOtoSQL-Net8 directory:
-```bash
-cd /path/to/SPO2SQL/SPOtoSQL-Net8
-./scripts/setup-secrets.sh
-```
-
-### "No secrets configured"
-
-**Problem**: User Secrets have not been initialized or set.
-
-**Solution**: Run the setup script:
-```bash
-./scripts/setup-secrets.sh
-```
-
-### Validation warnings
-
-**Warning**: "Connection string should contain 'Encrypt=true'"
-
-**Recommendation**: Always use encrypted connections in production. Update your connection string:
-```bash
-dotnet user-secrets set "Sql:ConnectionString" "Server=myServer;Database=myDB;User Id=user;Password=pwd;Encrypt=true;TrustServerCertificate=false;"
-```
-
-**Warning**: "Password is less than 8 characters"
-
-**Recommendation**: Use strong passwords (12+ characters, mixed case, numbers, symbols).
-
-## Security Notes
-
-### What Gets Stored Where
-
-**User Secrets** (Development):
-- Location: `~/.microsoft/usersecrets//secrets.json`
-- Format: JSON file with key-value pairs
-- Permissions: User-readable only
-- Automatically excluded from source control
-
-**Environment Variables** (Production):
-- Location: System or user environment variables
-- Prefix: `SPO2SQL_`
-- Separator: Double underscore `__` for nested sections
-- Set via: Shell, systemd service file, Docker, Kubernetes, etc.
-
-### Best Practices
-
-1. **Never commit secrets to source control**
- - User Secrets are stored outside the project
- - appsettings.json should NOT contain passwords
- - The verify-config script checks for this
-
-2. **Use encryption for all connections**
- - SQL: `Encrypt=true;TrustServerCertificate=false`
- - SharePoint: Always HTTPS
-
-3. **Rotate credentials regularly**
- - Update secrets with setup script or manually
- - Test changes before deploying to production
-
-4. **Use principle of least privilege**
- - SharePoint: Read-only access if possible
- - SQL: Minimal required permissions
-
-5. **Audit and monitor**
- - Enable application logging
- - Review logs for authentication failures
- - Monitor for unusual activity
-
-## Advanced Usage
-
-### Automated Setup (Non-Interactive)
-
-For CI/CD pipelines or automated deployments:
-
-```bash
-# Set secrets via dotnet CLI
-cd ConsoleApp1Net8
-dotnet user-secrets set "SharePoint:Username" "$SP_USERNAME"
-dotnet user-secrets set "SharePoint:Password" "$SP_PASSWORD"
-dotnet user-secrets set "SharePoint:SiteUrl" "$SP_SITEURL"
-dotnet user-secrets set "Sql:ConnectionString" "$SQL_CONNSTR"
-
-# Verify
-cd ..
-./scripts/verify-config.sh
-if [ $? -ne 0 ]; then
- echo "Configuration validation failed"
- exit 1
-fi
-```
-
-### Integration with CI/CD
-
-Example GitHub Actions workflow:
-
-```yaml
-- name: Configure Secrets
- run: |
- cd SPOtoSQL-Net8/ConsoleApp1Net8
- dotnet user-secrets set "SharePoint:Username" "${{ secrets.SP_USERNAME }}"
- dotnet user-secrets set "SharePoint:Password" "${{ secrets.SP_PASSWORD }}"
- dotnet user-secrets set "SharePoint:SiteUrl" "${{ secrets.SP_SITEURL }}"
- dotnet user-secrets set "Sql:ConnectionString" "${{ secrets.SQL_CONNSTR }}"
-
-- name: Verify Configuration
- run: |
- cd SPOtoSQL-Net8
- ./scripts/verify-config.sh
-```
-
-## See Also
-
-- **[SECURITY.md](../SECURITY.md)**: Comprehensive security documentation
-- **[README-MODERNIZATION.md](../README-MODERNIZATION.md)**: Configuration and deployment guide
-- **[.NET User Secrets Documentation](https://docs.microsoft.com/en-us/aspnet/core/security/app-secrets)**: Official Microsoft documentation
diff --git a/global.json b/global.json
new file mode 100644
index 0000000..391ba3c
--- /dev/null
+++ b/global.json
@@ -0,0 +1,6 @@
+{
+ "sdk": {
+ "version": "8.0.100",
+ "rollForward": "latestFeature"
+ }
+}
diff --git a/src/SPO2SQL.Console/Application.cs b/src/SPO2SQL.Console/Application.cs
index 0623805..0b83a19 100644
--- a/src/SPO2SQL.Console/Application.cs
+++ b/src/SPO2SQL.Console/Application.cs
@@ -1,15 +1,16 @@
+using Microsoft.Data.SqlClient;
using SPO2SQL.Configuration;
-using SPO2SQL.Models;
+using SPO2SQL.Logging;
using SPO2SQL.Security;
-using Microsoft.Data.SqlClient;
namespace SPO2SQL;
-public class Application : IHostedService
+public sealed class Application : BackgroundService
{
private readonly ILogger _logger;
+ private readonly ILoggerFactory _loggerFactory;
private readonly IHostApplicationLifetime _lifetime;
- private readonly IHttpClientFactory _httpClientFactory;
+ private readonly SyncExitCode _exitCode;
private readonly ApplicationOptions _appOptions;
private readonly SharePointOptions _sharePointOptions;
@@ -17,137 +18,87 @@ public class Application : IHostedService
public Application(
ILogger logger,
+ ILoggerFactory loggerFactory,
IHostApplicationLifetime lifetime,
- IHttpClientFactory httpClientFactory,
+ SyncExitCode exitCode,
IOptions appOptions,
IOptions sharePointOptions,
IOptions sqlOptions)
{
_logger = logger;
+ _loggerFactory = loggerFactory;
_lifetime = lifetime;
- _httpClientFactory = httpClientFactory;
+ _exitCode = exitCode;
_appOptions = appOptions.Value;
_sharePointOptions = sharePointOptions.Value;
_sqlOptions = sqlOptions.Value;
}
- public Task StartAsync(CancellationToken cancellationToken)
+ public override Task StartAsync(CancellationToken cancellationToken)
{
- _logger.LogInformation("═══════════════════════════════════════════════════════════");
- _logger.LogInformation(" {AppName} v{Version}", _appOptions.Name, _appOptions.Version);
- _logger.LogInformation("═══════════════════════════════════════════════════════════");
+ Logger.Configure(_loggerFactory.CreateLogger("SPO2SQL"));
- LogConfigurationSummary();
-
- if (cancellationToken.IsCancellationRequested)
+ if (_logger.IsEnabled(LogLevel.Information))
{
- _logger.LogWarning("Application cancelled before startup.");
- _lifetime.StopApplication();
- return Task.CompletedTask;
+ _logger.LogInformation("═══════════════════════════════════════════════════════════");
+ _logger.LogInformation(" {AppName} v{Version}", _appOptions.Name, _appOptions.Version);
+ _logger.LogInformation("═══════════════════════════════════════════════════════════");
}
- _ = Task.Run(async () => await ExecuteAsync(cancellationToken), cancellationToken)
- .ContinueWith(t =>
- {
- if (t.IsFaulted)
- {
- _logger.LogError(t.Exception, "Application task failed");
- }
- else if (t.IsCanceled)
- {
- _logger.LogWarning("Application task was cancelled");
- }
-
- _lifetime.StopApplication();
- }, TaskContinuationOptions.ExecuteSynchronously);
-
- return Task.CompletedTask;
- }
-
- public Task StopAsync(CancellationToken cancellationToken)
- {
- _logger.LogInformation("{AppName} stopping gracefully...", _appOptions.Name);
- return Task.CompletedTask;
- }
-
- private void LogConfigurationSummary()
- {
- _logger.LogInformation("Configuration Summary:");
- _logger.LogInformation(" Environment: {Environment}", _appOptions.Environment);
- _logger.LogInformation(" Metrics Enabled: {EnableMetrics}", _appOptions.EnableMetrics);
- _logger.LogInformation(" Health Checks Enabled: {EnableHealthChecks}", _appOptions.EnableHealthChecks);
-
- _logger.LogInformation("SharePoint Configuration:");
- _logger.LogInformation(" Site URL: {SiteUrl}", _sharePointOptions.SiteUrl);
- _logger.LogInformation(" Username: {Username}", MaskSensitiveData(_sharePointOptions.Username, 3));
- _logger.LogInformation(" Password: {Password}", "****** (hidden)");
- _logger.LogInformation(" Timeout: {TimeoutSeconds}s", _sharePointOptions.TimeoutSeconds);
- _logger.LogInformation(" Max Retries: {MaxRetries}", _sharePointOptions.MaxRetries);
- _logger.LogInformation(" Initial Retry Delay: {InitialRetryDelayMs}ms", _sharePointOptions.InitialRetryDelayMs);
-
- _logger.LogInformation("SQL Server Configuration:");
- _logger.LogInformation(" Connection String: {ConnectionString}", MaskConnectionString(_sqlOptions.ConnectionString));
- _logger.LogInformation(" Command Timeout: {CommandTimeoutSeconds}s", _sqlOptions.CommandTimeoutSeconds);
- _logger.LogInformation(" Batch Size: {BatchSize}", _sqlOptions.BatchSize);
- _logger.LogInformation(" Enforce Encryption: {EnforceEncryption}", _sqlOptions.EnforceEncryption);
-
- _logger.LogInformation("═══════════════════════════════════════════════════════════");
+ LogConfigurationSummary();
- ValidateConfigurationInvariantsExample();
+ return base.StartAsync(cancellationToken);
}
- private void ValidateConfigurationInvariantsExample()
+ public override async Task StopAsync(CancellationToken cancellationToken)
{
- if (_sqlOptions.BatchSize < 10 || _sqlOptions.BatchSize > 1000)
+ if (_logger.IsEnabled(LogLevel.Information))
{
- throw new InvalidOperationException(
- "This should never happen! BatchSize validation should have caught this at startup.");
+ _logger.LogInformation("{AppName} stopping gracefully...", _appOptions.Name);
}
-
- if (string.IsNullOrWhiteSpace(_sharePointOptions.SiteUrl))
- {
- throw new InvalidOperationException(
- "This should never happen! SiteUrl validation should have caught this at startup.");
- }
-
- _logger.LogDebug("Configuration validation checks passed (as expected)");
+ await base.StopAsync(cancellationToken).ConfigureAwait(false);
}
- private async Task ExecuteAsync(CancellationToken cancellationToken)
+ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
_logger.LogInformation("Starting SharePoint to SQL synchronization...");
- ConfigureLegacyBridge();
+ SyncSettings settings = BuildSyncSettings();
+ ConfigureLegacyBridge(settings);
if (_appOptions.EnableHealthChecks)
{
- await PerformHealthChecksAsync(cancellationToken);
+ await PerformHealthChecksAsync(settings, stoppingToken).ConfigureAwait(false);
}
- _logger.LogInformation("Sync configuration: DailyMode={DailyMode}, BatchSize={BatchSize}",
- _appOptions.DailyMode, _sqlOptions.BatchSize);
-
- await Task.Run(() =>
+ if (_logger.IsEnabled(LogLevel.Information))
{
- SqlServer.RefreshSQLLists.SPOtoSQLUpdate(
- daily: _appOptions.DailyMode,
- cancellationToken: cancellationToken);
- }, cancellationToken);
+ _logger.LogInformation("Sync configuration: DailyMode={DailyMode}, BatchSize={BatchSize}, Auth={AuthMode}, Metrics={EnableMetrics}",
+ _appOptions.DailyMode,
+ settings.BatchSize,
+ settings.HasAppOnlyAuth ? "app-only" : "username/password",
+ settings.EnableMetrics);
+ }
+
+ await RefreshSQLLists.SPOtoSQLUpdateAsync(
+ settings: settings,
+ daily: _appOptions.DailyMode,
+ cancellationToken: stoppingToken).ConfigureAwait(false);
_logger.LogInformation("SharePoint to SQL synchronization completed successfully");
}
catch (OperationCanceledException)
{
_logger.LogWarning("SharePoint to SQL synchronization was cancelled.");
- Environment.ExitCode = 1;
+ _exitCode.Fail();
}
catch (Exception ex)
{
_logger.LogError(ex, "Application failed with error: {Message}", ex.Message);
- Environment.ExitCode = 1;
+ _exitCode.Fail();
}
finally
{
@@ -155,16 +106,63 @@ await Task.Run(() =>
}
}
- private void ConfigureLegacyBridge()
+ private void LogConfigurationSummary()
+ {
+ if (!_logger.IsEnabled(LogLevel.Information))
+ {
+ return;
+ }
+
+ _logger.LogInformation("Configuration Summary:");
+ _logger.LogInformation(" Environment: {Environment}", _appOptions.Environment);
+ _logger.LogInformation(" Metrics Enabled: {EnableMetrics}", _appOptions.EnableMetrics);
+ _logger.LogInformation(" Health Checks Enabled: {EnableHealthChecks}", _appOptions.EnableHealthChecks);
+
+ _logger.LogInformation("SharePoint Configuration:");
+ _logger.LogInformation(" Site URL: {SiteUrl}", _sharePointOptions.SiteUrl);
+ _logger.LogInformation(" Username: {Username}", MaskSensitiveData(_sharePointOptions.Username, 3));
+ _logger.LogInformation(" Password: ****** (hidden)");
+ _logger.LogInformation(" Timeout: {TimeoutSeconds}s", _sharePointOptions.TimeoutSeconds);
+ _logger.LogInformation(" Max Retries: {MaxRetries}", _sharePointOptions.MaxRetries);
+ _logger.LogInformation(" Initial Retry Delay: {InitialRetryDelayMs}ms", _sharePointOptions.InitialRetryDelayMs);
+
+ _logger.LogInformation("SQL Server Configuration:");
+ _logger.LogInformation(" Connection String: {ConnectionString}", MaskConnectionString(_sqlOptions.ConnectionString));
+ _logger.LogInformation(" Command Timeout: {CommandTimeoutSeconds}s", _sqlOptions.CommandTimeoutSeconds);
+ _logger.LogInformation(" Batch Size: {BatchSize}", _sqlOptions.BatchSize);
+ _logger.LogInformation(" Enforce Encryption: {EnforceEncryption}", _sqlOptions.EnforceEncryption);
+
+ _logger.LogInformation("═══════════════════════════════════════════════════════════");
+ }
+
+ private SyncSettings BuildSyncSettings() => new()
+ {
+ Username = _sharePointOptions.Username,
+ Password = _sharePointOptions.Password,
+ SiteUrl = _sharePointOptions.SiteUrl,
+ SqlConnectionString = _sqlOptions.ConnectionString,
+ TenantId = _sharePointOptions.TenantId,
+ ClientId = _sharePointOptions.ClientId,
+ CertificatePath = _sharePointOptions.CertificatePath,
+ CertificatePassword = _sharePointOptions.CertificatePassword,
+ CommandTimeoutSeconds = _sqlOptions.CommandTimeoutSeconds,
+ BatchSize = _sqlOptions.BatchSize,
+ MaxRetries = _sharePointOptions.MaxRetries,
+ InitialRetryDelayMs = _sharePointOptions.InitialRetryDelayMs,
+ EnableMetrics = _appOptions.EnableMetrics,
+ LegacyConfigPath = _appOptions.LegacyConfigPath
+ };
+
+ private void ConfigureLegacyBridge(SyncSettings settings)
{
CredentialManager.RegisterFallbackProvider(
new StaticCredentialProvider(
- _sharePointOptions.Username,
- _sharePointOptions.Password,
- _sqlOptions.ConnectionString));
- XmlConfig.ConfigurationReader.SetSharePointBaseUrl(_sharePointOptions.SiteUrl);
+ settings.Username,
+ settings.Password,
+ settings.SqlConnectionString));
+ XmlConfig.ConfigurationReader.SetSharePointBaseUrl(settings.SiteUrl);
- string configPath = _appOptions.LegacyConfigPath ?? "";
+ string configPath = settings.LegacyConfigPath ?? "";
if (!Path.IsPathRooted(configPath))
{
configPath = Path.Combine(AppContext.BaseDirectory, configPath);
@@ -173,7 +171,10 @@ private void ConfigureLegacyBridge()
if (File.Exists(configPath))
{
XmlConfig.ConfigurationReader.SetConfigPath(configPath);
- _logger.LogInformation("Legacy config path set to: {ConfigPath}", configPath);
+ if (_logger.IsEnabled(LogLevel.Information))
+ {
+ _logger.LogInformation("Legacy config path set to: {ConfigPath}", configPath);
+ }
}
else
{
@@ -181,33 +182,43 @@ private void ConfigureLegacyBridge()
}
}
- private async Task PerformHealthChecksAsync(CancellationToken cancellationToken)
+ private async Task PerformHealthChecksAsync(SyncSettings settings, CancellationToken cancellationToken)
{
_logger.LogInformation("Performing startup health checks...");
+ ValidateSqlEncryption();
+
try
{
- using var httpClient = _httpClientFactory.CreateClient();
- httpClient.Timeout = TimeSpan.FromSeconds(_sharePointOptions.TimeoutSeconds);
+ using var spoUser = SPOUserFactory.Create(settings);
+ var checker = new HealthChecker();
+
+ var result = await checker.PerformHealthCheckAsync(
+ spoUser,
+ settings.SiteUrl,
+ settings.SqlConnectionString,
+ cancellationToken).ConfigureAwait(false);
- using var response = await httpClient.GetAsync(_sharePointOptions.SiteUrl, cancellationToken);
- _logger.LogInformation("SharePoint URL health check: {StatusCode}", response.StatusCode);
+ foreach (string warning in result.Warnings)
+ {
+ _logger.LogWarning("{HealthMessage}", warning);
+ }
+
+ foreach (string error in result.Errors)
+ {
+ _logger.LogError("{HealthMessage}", error);
+ }
+
+ if (!result.IsHealthy)
+ {
+ throw new InvalidOperationException(
+ "Startup health checks failed: " + string.Join("; ", result.Errors));
+ }
}
catch (OperationCanceledException)
{
throw;
}
- catch (Exception ex)
- {
- _logger.LogWarning(ex, "SharePoint URL health check failed (this may be expected if authentication is required)");
- }
-
- if (!HasServerKeyword(_sqlOptions.ConnectionString))
- {
- _logger.LogWarning("SQL connection string does not contain a server keyword (Server=/Data Source=) - verify configuration");
- }
-
- ValidateSqlEncryption();
_logger.LogInformation("Health checks completed");
}
@@ -232,119 +243,6 @@ private void ValidateSqlEncryption()
}
}
- private void DemonstrateRecordModels()
- {
- _logger.LogInformation("═══════════════════════════════════════════════════════════");
- _logger.LogInformation(" Record Models Demonstration");
- _logger.LogInformation("═══════════════════════════════════════════════════════════");
-
- var listItem = new SharePointListItem
- {
- Id = 1,
- Title = "Q1 Sales Report",
- Created = DateTime.UtcNow.AddDays(-30),
- Author = "john.doe@contoso.com"
- };
-
- var updatedItem = listItem with
- {
- Title = "Q1 Sales Report - Final",
- Modified = DateTime.UtcNow
- };
-
- _logger.LogInformation("SharePoint Item Original: {Title}", listItem.Title);
- _logger.LogInformation("SharePoint Item Updated: {Title} (original unchanged: {Unchanged})",
- updatedItem.Title, listItem.Title == "Q1 Sales Report");
-
- var operation = new SyncOperation(
- Guid.NewGuid(),
- SyncType.Daily,
- DateTime.UtcNow.AddHours(-2),
- null,
- 0,
- SyncStatus.Running
- );
-
- var (opId, syncType, start, _, _, status) = operation;
- _logger.LogInformation("Sync Operation: {Type} started at {Start}, Status: {Status}",
- syncType, start, status);
-
- var completedOp = operation with
- {
- EndTime = DateTime.UtcNow,
- ItemsProcessed = 1250,
- Status = SyncStatus.Completed
- };
-
- _logger.LogInformation("Operation completed: {Items} items in {Duration}",
- completedOp.ItemsProcessed, completedOp.Duration);
-
- var issues = new[]
- {
- new DataQualityIssue("CustomerList", 101, "Email", "InvalidFormat", "Missing @ symbol", Severity.High),
- new DataQualityIssue("OrderList", 202, "Total", "OutOfRange", "Negative total amount", Severity.Critical),
- new DataQualityIssue("ProductList", 303, "Description", "TooLong", "Exceeds 500 chars", Severity.Low)
- };
-
- foreach (var issue in issues)
- {
- var action = issue switch
- {
- { Severity: Severity.Critical } => "BLOCK SYNC",
- { Severity: Severity.High, IssueType: "InvalidFormat" } => "AUTO-FIX",
- { Severity: >= Severity.Medium } => "REVIEW",
- _ => "LOG ONLY"
- };
-
- _logger.LogInformation("DQ Issue: {ErrorMessage} \u2192 Action: {Action}",
- issue.ErrorMessage, action);
- }
-
- var stats = new SyncStatistics(
- TotalItems: 1500,
- Successful: 1450,
- Failed: 50,
- Duration: TimeSpan.FromMinutes(15)
- );
-
- _logger.LogInformation("Sync Statistics:");
- _logger.LogInformation(" {Summary}", stats.Summary);
- _logger.LogInformation(" Success Rate: {SuccessRate:F2}%", stats.SuccessRate);
- _logger.LogInformation(" Throughput: {Throughput:F2} items/sec", stats.Throughput);
- _logger.LogInformation(" Avg Time/Item: {AvgTime:F2}ms", stats.AverageTimePerItem.TotalMilliseconds);
- _logger.LogInformation(" Meets Quality Standards: {MeetsStandards}", stats.MeetsQualityStandards);
-
- var conn1 = new ConnectionInfo(
- "portal.contoso.com",
- "SharePointDB",
- "Production",
- DateTime.UtcNow.AddHours(-6)
- );
-
- var conn2 = new ConnectionInfo(
- "portal.contoso.com",
- "SharePointDB",
- "Production",
- conn1.LastSync
- );
-
- var conn3 = conn1 with { Environment = "Development" };
-
- _logger.LogInformation("Connection Info:");
- _logger.LogInformation(" conn1.Description: {Description}", conn1.Description);
- _logger.LogInformation(" conn1 == conn2 (value equality): {Equal}", conn1 == conn2);
- _logger.LogInformation(" ReferenceEquals(conn1, conn2): {RefEqual}", ReferenceEquals(conn1, conn2));
- _logger.LogInformation(" conn1 == conn3 (different env): {Equal}", conn1 == conn3);
- _logger.LogInformation(" Time Since Last Sync: {TimeSince:F1} hours",
- conn1.TimeSinceLastSync?.TotalHours ?? 0);
-
- var connections = new HashSet { conn1, conn2, conn3 };
- _logger.LogInformation(" HashSet with conn1, conn2, conn3: {Count} unique (conn1==conn2)",
- connections.Count);
-
- _logger.LogInformation("═══════════════════════════════════════════════════════════");
- }
-
private static string MaskSensitiveData(string value, int visibleChars = 3)
{
if (string.IsNullOrWhiteSpace(value))
@@ -360,20 +258,6 @@ private static string MaskSensitiveData(string value, int visibleChars = 3)
return value[..visibleChars] + new string('*', value.Length - visibleChars);
}
- private static bool HasServerKeyword(string connectionString)
- {
- if (string.IsNullOrWhiteSpace(connectionString))
- {
- return false;
- }
-
- var match = System.Text.RegularExpressions.Regex.Match(
- connectionString,
- @"(?:Server|Data\s*Source|Address|Addr|Network\s*Address)\s*=",
- System.Text.RegularExpressions.RegexOptions.IgnoreCase);
- return match.Success;
- }
-
private static string MaskConnectionString(string connectionString)
{
if (string.IsNullOrWhiteSpace(connectionString))
diff --git a/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs b/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs
index e46e4d8..f9664ba 100644
--- a/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs
+++ b/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs
@@ -24,5 +24,5 @@ public record ApplicationOptions
public bool DailyMode { get; init; } = true;
[Required]
- public string LegacyConfigPath { get; init; } = "XmlConfig\\UserConfig.xml";
+ public string LegacyConfigPath { get; init; } = "XmlConfig/UserConfig.xml";
}
diff --git a/src/SPO2SQL.Console/Configuration/SharePointOptions.cs b/src/SPO2SQL.Console/Configuration/SharePointOptions.cs
index 4dd5ed4..aeb9b67 100644
--- a/src/SPO2SQL.Console/Configuration/SharePointOptions.cs
+++ b/src/SPO2SQL.Console/Configuration/SharePointOptions.cs
@@ -2,16 +2,12 @@
namespace SPO2SQL.Configuration;
-public record SharePointOptions
+public record SharePointOptions : IValidatableObject
{
public const string SectionName = "SharePoint";
- [Required(ErrorMessage = "SharePoint username is required")]
- [EmailAddress(ErrorMessage = "SharePoint username must be a valid email address")]
public string Username { get; init; } = string.Empty;
- [Required(ErrorMessage = "SharePoint password is required")]
- [MinLength(1, ErrorMessage = "SharePoint password cannot be empty")]
public string Password { get; init; } = string.Empty;
[Required(ErrorMessage = "SharePoint site URL is required")]
@@ -21,9 +17,54 @@ public record SharePointOptions
[Range(10, 600, ErrorMessage = "Timeout must be between 10 and 600 seconds")]
public int TimeoutSeconds { get; init; } = 120;
- [Range(0, 10, ErrorMessage = "Max retries must be between 0 and 10")]
+ [Range(1, 10, ErrorMessage = "Max retries must be between 1 and 10")]
public int MaxRetries { get; init; } = 3;
[Range(100, 10000, ErrorMessage = "Initial retry delay must be between 100 and 10000 ms")]
public int InitialRetryDelayMs { get; init; } = 1000;
+
+ // App-only authentication. When configured, these take precedence over
+ // username/password and work with MFA and Conditional Access policies.
+ public string? TenantId { get; init; }
+
+ public string? ClientId { get; init; }
+
+ public string? CertificatePath { get; init; }
+
+ public string? CertificatePassword { get; init; }
+
+ public bool HasAppOnlyAuth =>
+ !string.IsNullOrWhiteSpace(TenantId) &&
+ !string.IsNullOrWhiteSpace(ClientId) &&
+ !string.IsNullOrWhiteSpace(CertificatePath);
+
+ public IEnumerable Validate(ValidationContext validationContext)
+ {
+ bool hasUserPassword =
+ !string.IsNullOrWhiteSpace(Username) &&
+ !string.IsNullOrWhiteSpace(Password);
+
+ if (!string.IsNullOrWhiteSpace(Username) &&
+ !new EmailAddressAttribute().IsValid(Username))
+ {
+ yield return new ValidationResult(
+ "SharePoint username must be a valid email address.",
+ [nameof(Username)]);
+ }
+
+ if (!hasUserPassword && !HasAppOnlyAuth)
+ {
+ yield return new ValidationResult(
+ "Configure either Username/Password or app-only authentication " +
+ "(TenantId, ClientId and CertificatePath).",
+ [nameof(Username), nameof(TenantId), nameof(ClientId), nameof(CertificatePath)]);
+ }
+
+ if (HasAppOnlyAuth && !File.Exists(CertificatePath))
+ {
+ yield return new ValidationResult(
+ $"The configured SharePoint certificate was not found: '{CertificatePath}'.",
+ [nameof(CertificatePath)]);
+ }
+ }
}
diff --git a/src/SPO2SQL.Console/CredentialSetup.cs b/src/SPO2SQL.Console/CredentialSetup.cs
new file mode 100644
index 0000000..b5588b9
--- /dev/null
+++ b/src/SPO2SQL.Console/CredentialSetup.cs
@@ -0,0 +1,57 @@
+using SPO2SQL.Security;
+
+namespace SPO2SQL;
+
+///
+/// Implements the --setup-secrets command in the real console host.
+///
+internal static class CredentialSetup
+{
+ public static int Run()
+ {
+ Console.WriteLine("=== Secure Credential Setup ===");
+ Console.WriteLine("Credentials will be encrypted and stored in the current user's profile.");
+ Console.WriteLine();
+
+ string? username = Read("SharePoint Username: ");
+ string? password = Read("SharePoint Password: ");
+ string? connectionString = Read("SQL Connection String: ");
+
+ if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password))
+ {
+ Console.WriteLine("ERROR: Username and password are required.");
+ return 1;
+ }
+
+ try
+ {
+ if (!OperatingSystem.IsWindows())
+ {
+ Console.WriteLine("ERROR: DPAPI encryption is not available on this platform.");
+ Console.WriteLine("Use environment variables or user secrets instead.");
+ return 1;
+ }
+
+ DpapiCredentialProvider.SaveCredentials(username, password, connectionString ?? string.Empty);
+ Console.WriteLine("Credentials saved securely.");
+ return 0;
+ }
+ catch (PlatformNotSupportedException)
+ {
+ Console.WriteLine("ERROR: DPAPI encryption is not available on this platform.");
+ Console.WriteLine("Use environment variables or user secrets instead.");
+ return 1;
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine($"ERROR: Failed to save credentials: {ex.Message}");
+ return 1;
+ }
+ }
+
+ private static string? Read(string prompt)
+ {
+ Console.Write(prompt);
+ return Console.ReadLine()?.Trim();
+ }
+}
diff --git a/src/SPO2SQL.Console/Program.cs b/src/SPO2SQL.Console/Program.cs
index a9eeea4..7ee0c03 100644
--- a/src/SPO2SQL.Console/Program.cs
+++ b/src/SPO2SQL.Console/Program.cs
@@ -7,14 +7,22 @@ internal sealed class Program
{
private static async Task Main(string[] args)
{
+ if (args.Any(a => string.Equals(a, "--setup-secrets", StringComparison.OrdinalIgnoreCase)))
+ {
+ return CredentialSetup.Run();
+ }
+
try
{
var builder = CreateHostBuilder(args);
- using var host = builder.Build();
+ var host = builder.Build();
+
+ // RunAsync disposes the host when it completes, so resolve the outcome first.
+ var exitCode = host.Services.GetRequiredService();
await host.RunAsync();
- return 0;
+ return exitCode.Value;
}
catch (OptionsValidationException ex)
{
@@ -81,7 +89,7 @@ private static IHostBuilder CreateHostBuilder(string[] args) =>
.ValidateDataAnnotations()
.ValidateOnStart();
- services.AddHttpClient();
+ services.AddSingleton();
services.AddHostedService();
});
}
diff --git a/src/SPO2SQL.Console/SPO2SQL.Console.csproj b/src/SPO2SQL.Console/SPO2SQL.Console.csproj
index fb394ae..b5efcb4 100644
--- a/src/SPO2SQL.Console/SPO2SQL.Console.csproj
+++ b/src/SPO2SQL.Console/SPO2SQL.Console.csproj
@@ -9,8 +9,6 @@
-
-
@@ -24,11 +22,7 @@
-
-
-
-
-
+
diff --git a/src/SPO2SQL.Console/SyncExitCode.cs b/src/SPO2SQL.Console/SyncExitCode.cs
new file mode 100644
index 0000000..cd934c8
--- /dev/null
+++ b/src/SPO2SQL.Console/SyncExitCode.cs
@@ -0,0 +1,15 @@
+namespace SPO2SQL;
+
+///
+/// Carries the synchronization outcome from the hosted service to the process
+/// entry point. cannot be used here because
+/// Main returns an explicit integer, which always wins.
+///
+public sealed class SyncExitCode
+{
+ private int _value;
+
+ public int Value => Volatile.Read(ref _value);
+
+ public void Fail() => Interlocked.CompareExchange(ref _value, 1, 0);
+}
diff --git a/src/SPO2SQL.Console/Utilities/OperationResult.cs b/src/SPO2SQL.Console/Utilities/OperationResult.cs
index 951be0e..b0ff6be 100644
--- a/src/SPO2SQL.Console/Utilities/OperationResult.cs
+++ b/src/SPO2SQL.Console/Utilities/OperationResult.cs
@@ -94,9 +94,9 @@ public static string CategorizeError(this OperationResult result) =>
{
OperationResult.SuccessResult => "Success",
OperationResult.FailureResult { ErrorCode: "NotFound" } => "Resource Not Found",
- OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Auth") == true =>
+ OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Auth", StringComparison.Ordinal) == true =>
"Authentication/Authorization Error",
- OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Validation") == true =>
+ OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Validation", StringComparison.Ordinal) == true =>
"Validation Error",
OperationResult.FailureResult { Exception: not null } => "Exception Occurred",
OperationResult.FailureResult => "General Error",
diff --git a/src/SPO2SQL.Console/appsettings.json b/src/SPO2SQL.Console/appsettings.json
index adbc511..693f602 100644
--- a/src/SPO2SQL.Console/appsettings.json
+++ b/src/SPO2SQL.Console/appsettings.json
@@ -6,7 +6,7 @@
"EnableMetrics": true,
"EnableHealthChecks": true,
"Environment": "Production",
- "LegacyConfigPath": "XmlConfig\\UserConfig.xml"
+ "LegacyConfigPath": "XmlConfig/UserConfig.xml"
},
"SharePoint": {
"Username": "",
@@ -14,7 +14,11 @@
"SiteUrl": "https://bringglobal.sharepoint.com",
"TimeoutSeconds": 120,
"MaxRetries": 3,
- "InitialRetryDelayMs": 1000
+ "InitialRetryDelayMs": 1000,
+ "TenantId": "",
+ "ClientId": "",
+ "CertificatePath": "",
+ "CertificatePassword": ""
},
"Sql": {
"ConnectionString": "",
diff --git a/src/SPO2SQL.Console/secrets.json.template b/src/SPO2SQL.Console/secrets.json.template
index 112606d..f841e26 100644
--- a/src/SPO2SQL.Console/secrets.json.template
+++ b/src/SPO2SQL.Console/secrets.json.template
@@ -6,7 +6,7 @@
"EnableMetrics": true,
"EnableHealthChecks": true,
"Environment": "Development",
- "LegacyConfigPath": "XmlConfig\\UserConfig.xml"
+ "LegacyConfigPath": "XmlConfig/UserConfig.xml"
},
"SharePoint": {
"Username": "your-email@company.com",
@@ -14,7 +14,11 @@
"SiteUrl": "https://yourtenant.sharepoint.com",
"TimeoutSeconds": 120,
"MaxRetries": 3,
- "InitialRetryDelayMs": 1000
+ "InitialRetryDelayMs": 1000,
+ "TenantId": "",
+ "ClientId": "",
+ "CertificatePath": "",
+ "CertificatePassword": ""
},
"Sql": {
"ConnectionString": "Server=localhost;Database=SPO2SQL;Integrated Security=true;TrustServerCertificate=true;",
diff --git a/src/SPO2SQL.Core/Configuration/SyncSettings.cs b/src/SPO2SQL.Core/Configuration/SyncSettings.cs
new file mode 100644
index 0000000..aa61a1a
--- /dev/null
+++ b/src/SPO2SQL.Core/Configuration/SyncSettings.cs
@@ -0,0 +1,42 @@
+namespace SPO2SQL.Configuration;
+
+///
+/// Immutable snapshot of the settings needed by the synchronization layer.
+/// The console host maps its validated options into this record once, at startup,
+/// so the core never has to reach back into global configuration state.
+///
+public sealed record SyncSettings
+{
+ public string Username { get; init; } = string.Empty;
+
+ public string Password { get; init; } = string.Empty;
+
+ public required string SiteUrl { get; init; }
+
+ public required string SqlConnectionString { get; init; }
+
+ public string? TenantId { get; init; }
+
+ public string? ClientId { get; init; }
+
+ public string? CertificatePath { get; init; }
+
+ public string? CertificatePassword { get; init; }
+
+ public int CommandTimeoutSeconds { get; init; } = 300;
+
+ public int BatchSize { get; init; } = 80;
+
+ public int MaxRetries { get; init; } = 3;
+
+ public int InitialRetryDelayMs { get; init; } = 1000;
+
+ public bool EnableMetrics { get; init; }
+
+ public string? LegacyConfigPath { get; init; }
+
+ public bool HasAppOnlyAuth =>
+ !string.IsNullOrWhiteSpace(TenantId) &&
+ !string.IsNullOrWhiteSpace(ClientId) &&
+ !string.IsNullOrWhiteSpace(CertificatePath);
+}
diff --git a/src/SPO2SQL.Core/Logging/Logger.cs b/src/SPO2SQL.Core/Logging/Logger.cs
index 30157ce..dabc3e8 100644
--- a/src/SPO2SQL.Core/Logging/Logger.cs
+++ b/src/SPO2SQL.Core/Logging/Logger.cs
@@ -1,57 +1,110 @@
-using System;
+using Microsoft.Extensions.Logging;
namespace SPO2SQL.Logging;
+///
+/// Bridge between the legacy static logging calls used across the core library and
+/// the host's . The host must call at
+/// startup; until then messages are written to the console so failures are never
+/// silently discarded.
+///
public static class Logger
{
- private const string TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.fff";
- private enum LogLevel { ERROR = 1, WARNING = 2, DEBUG = 3 }
+ private static readonly object Sync = new();
- public static int VerboseLevel { get; set; } = 0;
+ private static ILogger? _sink;
+ private static int _verboseLevel = 2;
- public static void Log(int level, string message)
+ ///
+ /// Legacy verbosity gate kept for source compatibility. 1 = errors only,
+ /// 2 = warnings and above, 3 = all messages.
+ ///
+ public static int VerboseLevel
{
- if (level < 1 || level > 3)
- {
- return;
- }
+ get => Volatile.Read(ref _verboseLevel);
+ set => Volatile.Write(ref _verboseLevel, value);
+ }
- if (VerboseLevel < level)
- {
- return;
- }
+ ///
+ /// Connects the static facade to the application's logging pipeline.
+ ///
+ public static void Configure(ILogger sink)
+ {
+ ArgumentNullException.ThrowIfNull(sink);
- string timestamp = DateTime.Now.ToString(TIMESTAMP_FORMAT);
- string levelName = GetLevelName(level);
- string formattedMessage = $"[{timestamp}] [{levelName}] {message}";
+ Volatile.Write(ref _sink, sink);
+ VerboseLevel = sink.IsEnabled(LogLevel.Debug) ? 3 : 2;
+ }
- Console.WriteLine(formattedMessage);
+ internal static void Reset()
+ {
+ Volatile.Write(ref _sink, null);
+ _verboseLevel = 2;
}
- public static void LogError(string message, Exception ex = null)
+ ///
+ /// Logs a message using the legacy numeric levels: 1 = information,
+ /// 2 = warning, 3 = debug.
+ ///
+ public static void Log(int level, string message)
{
- Log(1, message);
- if (ex != null && VerboseLevel >= 3)
+ switch (level)
{
- Log(3, $"Exception Details: {ex.GetType().Name}: {ex.Message}");
+ case 1:
+ Write(LogLevel.Information, 1, message, null);
+ break;
+ case 2:
+ Write(LogLevel.Warning, 2, message, null);
+ break;
+ case 3:
+ Write(LogLevel.Debug, 3, message, null);
+ break;
}
}
- public static void LogWarning(string message)
- {
- Log(2, message);
- }
+ public static void LogInformation(string message) => Write(LogLevel.Information, 1, message, null);
- public static void LogDebug(string message)
- {
- Log(3, message);
- }
+ public static void LogWarning(string message) => Write(LogLevel.Warning, 2, message, null);
- private static string GetLevelName(int level) => level switch
+ public static void LogDebug(string message) => Write(LogLevel.Debug, 3, message, null);
+
+ public static void LogError(string message, Exception? ex = null) => Write(LogLevel.Error, 1, message, ex);
+
+ private static void Write(LogLevel severity, int legacyLevel, string message, Exception? exception)
{
- 1 => "ERROR",
- 2 => "WARN",
- 3 => "DEBUG",
- _ => "INFO"
- };
+ if (legacyLevel > VerboseLevel)
+ {
+ return;
+ }
+
+ ILogger? sink = Volatile.Read(ref _sink);
+ if (sink != null)
+ {
+ if (!sink.IsEnabled(severity))
+ {
+ return;
+ }
+
+ sink.Log(severity, new EventId(0, "Legacy"), message, exception, static (state, _) => state);
+ return;
+ }
+
+ string levelName = severity switch
+ {
+ LogLevel.Error or LogLevel.Critical => "ERROR",
+ LogLevel.Warning => "WARN",
+ LogLevel.Debug or LogLevel.Trace => "DEBUG",
+ _ => "INFO"
+ };
+
+ string line = $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] [{levelName}] {message}";
+ lock (Sync)
+ {
+ Console.WriteLine(line);
+ if (exception != null)
+ {
+ Console.WriteLine(exception);
+ }
+ }
+ }
}
diff --git a/src/SPO2SQL.Core/SPO2SQL.Core.csproj b/src/SPO2SQL.Core/SPO2SQL.Core.csproj
index 66c8c0a..5bf826d 100644
--- a/src/SPO2SQL.Core/SPO2SQL.Core.csproj
+++ b/src/SPO2SQL.Core/SPO2SQL.Core.csproj
@@ -7,10 +7,10 @@
+
-
@@ -18,5 +18,8 @@
<_Parameter1>SPO2SQL.Console
+
+ <_Parameter1>SPO2SQL.Core.Tests
+
diff --git a/src/SPO2SQL.Core/SPODataQuality/RefreshSPOLists.cs b/src/SPO2SQL.Core/SPODataQuality/RefreshSPOLists.cs
deleted file mode 100644
index df4725e..0000000
--- a/src/SPO2SQL.Core/SPODataQuality/RefreshSPOLists.cs
+++ /dev/null
@@ -1,616 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Linq.Expressions;
-using System.Threading;
-using Microsoft.Data.SqlClient;
-using Microsoft.SharePoint.Client;
-using SPO2SQL.Logging;
-using SPO2SQL.Security;
-using SPO2SQL.SharePoint;
-using SPO2SQL.SqlServer;
-using SPO2SQL.XmlConfig;
-
-namespace SPO2SQL.SPODataQuality;
-
-internal sealed class RefreshSPOLists
-{
- private static void Main(string[] args)
- {
- try
- {
- InitializeApplication(args);
-
- if (args.Any(a => a.Equals("--setup-secrets", StringComparison.OrdinalIgnoreCase)))
- {
- RunSetupSecrets();
- return;
- }
-
- RunMainWorkflow();
- }
- catch (Exception ex)
- {
- HandleFatalError(ex);
- }
- }
-
- private static void RunSetupSecrets()
- {
- Console.WriteLine("=== Secure Credential Setup ===");
- Console.WriteLine("Credentials will be encrypted and stored at:");
- Console.WriteLine(" %APPDATA%\\SPO2SQL\\credentials.enc");
- Console.WriteLine();
-
- Console.Write("SharePoint Username: ");
- string username = Console.ReadLine()?.Trim();
- Console.Write("SharePoint Password: ");
- string password = Console.ReadLine()?.Trim();
- Console.Write("SQL Connection String: ");
- string connString = Console.ReadLine()?.Trim();
-
- if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password))
- {
- Console.WriteLine("ERROR: Username and password are required.");
- Environment.Exit(1);
- }
-
- try
- {
- DpapiCredentialProvider.SaveCredentials(username, password, connString ?? "");
- Console.WriteLine("Credentials saved securely.");
- }
- catch (PlatformNotSupportedException)
- {
- Console.WriteLine("ERROR: DPAPI encryption is not available on this platform.");
- Console.WriteLine("Use environment variables instead:");
- Console.WriteLine(" set SPO_USERNAME=" + username);
- Console.WriteLine(" set SPO_PASSWORD=...");
- Console.WriteLine(" set SQL_CONNECTION_STRING=...");
- Environment.Exit(1);
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to save credentials: " + ex.Message);
- Environment.Exit(1);
- }
- }
-
- private static void InitializeApplication(string[] args)
- {
- Logger.Log(1, "DEBUG: Using the Default config");
- string configPath = "XmlConfig\\UserConfig.xml";
-
- int verbose = 0;
- bool diagnostic = false;
-
- foreach (var arg in args)
- {
- if (arg.StartsWith("--config="))
- {
- configPath = arg.Substring("--config=".Length);
- }
- else if (arg.StartsWith("--verbose="))
- {
- if (int.TryParse(arg.Substring("--verbose=".Length), out int v) && v >= 0 && v <= 3)
- {
- verbose = v;
- }
- }
- else if (arg.ToLower() == "diagnostic")
- {
- diagnostic = true;
- }
- }
-
- if (diagnostic && verbose == 0)
- {
- verbose = 1;
- }
-
- Logger.VerboseLevel = verbose;
-
- ConfigurationReader.SetConfigPath(configPath);
-
- Logger.Log(1, "DEBUG: Application initialized");
- Logger.Log(2, "CURRENT TIME: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"));
- }
-
- private static void RunMainWorkflow()
- {
- try
- {
- TestSQLConnection();
- Logger.Log(1, "DEBUG: SQL connection test completed");
-
- var credentials = ConfigurationReader.GetSharePointCredentials();
- if (string.IsNullOrEmpty(credentials.Username) || string.IsNullOrEmpty(credentials.Password))
- {
- throw new InvalidOperationException("Failed to retrieve SharePoint credentials from configuration");
- }
-
- if (!TwoFactorAuth.PerformVerification())
- {
- Console.WriteLine("2FA verification failed. Exiting.");
- Environment.Exit(1);
- }
-
- using (var spoUser = new SPOUser(credentials.Username, credentials.Password))
- {
- Logger.Log(1, "DEBUG: SPOUser created");
-
- Logger.Log(3, "DEBUG: SPOList objects configured");
-
- ProcessCommandLineArguments();
-
- Logger.Log(2, "End of requests.");
- Logger.Log(2, "");
- }
- }
- catch (Exception ex)
- {
- throw new ApplicationException("Error in main workflow", ex);
- }
- }
-
- private static void ProcessCommandLineArguments()
- {
- Logger.Log(1, "DEBUG: Processing command-line arguments");
- Logger.Log(1, "DEBUG: Command-line arguments processed");
- }
-
- private static void HandleFatalError(Exception ex)
- {
- Console.WriteLine("FATAL ERROR: An unexpected error occurred.");
- Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}");
- Console.WriteLine("Stack Trace: " + ex.StackTrace);
- Environment.Exit(1);
- }
-
- private static void TestSQLConnection()
- {
- Logger.Log(2, "Testing SQL Server connection...");
- try
- {
- string connectionString = ConfigurationReader.GetSqlConnectionString();
- if (string.IsNullOrWhiteSpace(connectionString))
- {
- throw new InvalidOperationException("SQL connection string is not configured");
- }
-
- Logger.Log(2, "Attempting to connect to SQL server...");
-
- using (var connection = new SqlConnection(connectionString))
- {
- try
- {
- connection.Open();
- }
- catch (SqlException ex) when (IsNetworkRelatedSqlError(ex.Number))
- {
- Console.WriteLine("ERROR: Unable to connect to the SQL Server.");
- Console.WriteLine("Please check if your VPN connection is active and try again.");
- Console.WriteLine("Technical details: " + ex.Message);
- Console.WriteLine("Press any key to exit...");
- if (Console.KeyAvailable || !Console.IsInputRedirected)
- {
- try
- {
- Console.ReadKey(true);
- }
- catch (InvalidOperationException)
- {
- }
- }
- else
- {
- Thread.Sleep(2000);
- }
- Environment.Exit(1);
- }
- catch (SqlException ex)
- {
- Console.WriteLine($"SQL error: {ex.Number} - {ex.Message}");
- throw;
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Unexpected error while opening SQL connection.");
- Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}");
- throw;
- }
-
- Logger.Log(2, $"Server: {connection.DataSource}");
- Logger.Log(2, $"Database: {connection.Database}");
- Logger.Log(2, "SQL connection established successfully!");
-
- using (var command = connection.CreateCommand())
- {
- try
- {
- command.CommandText = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES";
- object result = command.ExecuteScalar();
- if (result != null)
- {
- int tableCount = Convert.ToInt32(result);
- Logger.Log(2, $"Number of tables in database: {tableCount}");
- }
- else
- {
- Logger.Log(2, "Number of tables in database: 0 (null result)");
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to execute SELECT COUNT(*) on INFORMATION_SCHEMA.TABLES.");
- Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}");
- }
-
- try
- {
- command.CommandText = "CREATE TABLE #TempTest (ID int); DROP TABLE #TempTest;";
- command.ExecuteNonQuery();
- Logger.Log(2, "CREATE TABLE permission: OK");
- }
- catch (SqlException ex)
- {
- Logger.Log(2, $"Warning: No CREATE TABLE permission: {ex.Number} - {ex.Message}");
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Unexpected error during CREATE TABLE permission test.");
- Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}");
- }
- }
- }
- }
- catch (SqlException ex)
- {
- Console.WriteLine($"SQL connection error: {ex.Number} - {ex.Message}");
- Console.WriteLine("Error details:");
- Console.WriteLine($" Number: {ex.Number}");
- Console.WriteLine($" State: {ex.State}");
- if (!string.IsNullOrEmpty(ex.Procedure))
- {
- Console.WriteLine($" Procedure: {ex.Procedure}");
- }
- Console.WriteLine("The execution is stopping.");
- throw;
- }
- catch (InvalidOperationException ex)
- {
- Console.WriteLine($"Configuration error: {ex.Message}");
- throw;
- }
- catch (Exception ex)
- {
- Console.WriteLine($"General error testing SQL connection: {ex.GetType().Name}: {ex.Message}");
- throw;
- }
- }
-
- private static bool IsNetworkRelatedSqlError(int errorNumber)
- {
- switch (errorNumber)
- {
- case -2:
- case -1:
- case 0:
- case 53:
- case 64:
- case 87:
- case 10060:
- case 10061:
- case 11001:
- return true;
- default:
- return false;
- }
- }
-
- public static void GetAllLists()
- {
- Logger.Log(1, "DEBUG: Entering GetAllLists");
- try
- {
- var (username, password) = ConfigurationReader.GetSharePointCredentials();
- using (SPOUser spoUser = new SPOUser(username, password))
- {
- Context context = new Context()
- {
- Site = "seed",
- SPOUser = spoUser
- };
- foreach (Microsoft.SharePoint.Client.List allList in context.GetAllLists())
- {
- try
- {
- Logger.Log(1, "DEBUG: Loading list - " + allList.Title);
- context.Ctx.Load(allList, l => l.IsSystemList);
- context.Ctx.ExecuteQuery();
- Logger.Log(2, "List Name: " + allList.Title + "; is: " + allList.IsSystemList.ToString());
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to load or display list '" + allList.Title + "'.");
- Console.WriteLine("Exception: " + ex.Message);
- }
- }
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Exception in GetAllLists.");
- Console.WriteLine("Exception: " + ex.Message);
- Console.WriteLine("Stack Trace: " + ex.StackTrace);
- }
- }
-
- private static void SPODebug(string listName, string ctxURL, SPOUser user)
- {
- Logger.Log(1, "DEBUG: Entering SPODebug");
- try
- {
- SPOList spoList = new SPOList
- {
- Name = listName,
- Site = ctxURL,
- SPOUser = user,
- CAMLQuery = "1"
- };
-
- Logger.Log(1, "DEBUG: Executing Build");
- try
- {
- spoList.Build();
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to build SPOList.");
- Console.WriteLine("Exception: " + ex.Message);
- return;
- }
-
- Logger.Log(1, "DEBUG: Executing PrintItemProperties");
- try
- {
- if (spoList.ItemCollection != null && spoList.ItemCollection.Count > 0)
- {
- spoList.PrintItemProperties(spoList.ItemCollection[0]);
- }
- else
- {
- Logger.Log(2, "No items found in the list.");
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to print properties of the first item.");
- Console.WriteLine("Exception: " + ex.Message);
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Exception in SPODebug.");
- Console.WriteLine("Exception: " + ex.Message);
- Console.WriteLine("Stack Trace: " + ex.StackTrace);
- }
- }
-
- private static void RefreshListsSPO(SPOList sourceList, SPOList destList)
- {
- try
- {
- Logger.Log(1, "DEBUG: Starting RefreshListsSPO");
-
- try
- {
- sourceList.Build();
- Logger.Log(1, "DEBUG: sourceList.Build completed");
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to build source SPOList.");
- Console.WriteLine("Exception: " + ex.Message);
- return;
- }
-
- try
- {
- destList.Build();
- Logger.Log(1, "DEBUG: destList.Build completed");
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to build destination SPOList.");
- Console.WriteLine("Exception: " + ex.Message);
- return;
- }
-
- int num1 = 0;
- int num2 = 0;
-
- string[,] actualFields;
- try
- {
- actualFields = GetActualFields(sourceList, destList);
- Logger.Log(1, "DEBUG: Fields obtained");
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to get actual fields mapping.");
- Console.WriteLine("Exception: " + ex.Message);
- return;
- }
-
- try
- {
- if ((uint)sourceList.ItemCollection.Count > 0U)
- {
- num1 = (int)sourceList.ItemCollection[sourceList.ItemCollection.Count - 1]["ID"];
- }
-
- if ((uint)destList.ItemCollection.Count > 0U)
- {
- num2 = (int)destList.ItemCollection[destList.ItemCollection.Count - 1]["ID"];
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to determine last IDs in source/destination lists.");
- Console.WriteLine("Exception: " + ex.Message);
- }
-
- if (num2 < num1)
- {
- Logger.Log(1, "DEBUG: Adding new items");
- try
- {
- do
- {
- destList.AddItem();
- ++num2;
- }
- while (num2 < num1);
-
- Logger.Log(2, "Adding new items...");
- destList.Update();
- Logger.Log(2, "Done adding items.");
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to add or update new items in destination list.");
- Console.WriteLine("Exception: " + ex.Message);
- }
- }
-
- try
- {
- for (int index1 = 0; index1 < sourceList.ItemCollection.Count; ++index1)
- {
- int id = (int)sourceList.ItemCollection[index1]["ID"];
- for (int index2 = 0; index2 < actualFields.Length / 2; ++index2)
- {
- try
- {
- destList.ItemCollection.GetById(id)[actualFields[index2, 0]] = sourceList.ItemCollection[index1][actualFields[index2, 1]];
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to copy field '" + actualFields[index2, 1] + "' to '" + actualFields[index2, 0] + "' for item ID " + id + ".");
- Console.WriteLine("Exception: " + ex.Message);
- }
- }
- try
- {
- destList.ItemCollection.GetById(id).Update();
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to update item ID " + id + " in destination list.");
- Console.WriteLine("Exception: " + ex.Message);
- }
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Exception during item synchronization.");
- Console.WriteLine("Exception: " + ex.Message);
- }
-
- try
- {
- destList.Ctx.ExecuteQuery();
- Logger.Log(2, sourceList.Site + " " + sourceList.Name + " -> " + destList.Site + " " + destList.Name + ": Done!");
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Failed to execute query on destination list context.");
- Console.WriteLine("Exception: " + ex.Message);
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR in RefreshListsSPO: " + ex.Message);
- Console.WriteLine("Stack Trace: " + ex.StackTrace);
- }
- }
-
- private static string[,] GetActualFields(SPOList listone, SPOList listtwo)
- {
- Logger.Log(1, "DEBUG: Entering GetActualFields");
- try
- {
- List fields1 = GetFields(listone);
- List fields2 = GetFields(listtwo);
-
- string[,] strArray = new string[fields1.Count, 2];
- for (int i = 0; i < fields1.Count; i++)
- {
- strArray[i, 0] = "";
- strArray[i, 1] = "";
- }
-
- int index1 = 0;
- int index2 = 0;
-
- foreach (Field field1 in fields1)
- {
- bool found = false;
- while (index2 < fields2.Count)
- {
- Field field2 = fields2[index2];
- if (field1.Title == field2.Title)
- {
- strArray[index1, 0] = field2.InternalName;
- strArray[index1, 1] = field1.InternalName;
- found = true;
- Logger.Log(1, "DEBUG: Match found - " + field1.Title);
- }
- ++index2;
- if (found)
- {
- break;
- }
- }
-
- ++index1;
- index2 = 0;
- }
-
- return strArray;
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Exception in GetActualFields.");
- Console.WriteLine("Exception: " + ex.Message);
- throw;
- }
- }
-
- private static List GetFields(SPOList list)
- {
- Logger.Log(1, "DEBUG: Entering GetFields");
- List fieldList = new List();
- try
- {
- foreach (Field field in (ClientObjectCollection)list.Fields)
- {
- if (!field.FromBaseType || field.InternalName == "Title")
- {
- fieldList.Add(field);
- Logger.Log(1, "DEBUG: Field added - " + field.Title);
- }
- }
- }
- catch (Exception ex)
- {
- Console.WriteLine("ERROR: Exception in GetFields.");
- Console.WriteLine("Exception: " + ex.Message);
- throw;
- }
-
- return fieldList;
- }
-}
diff --git a/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs b/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs
index 8832b67..54faadb 100644
--- a/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs
+++ b/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs
@@ -1,11 +1,14 @@
-using System;
-using System.IO;
+using System.Runtime.Versioning;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
namespace SPO2SQL.Security;
+///
+/// Windows DPAPI credential store. All entry points fail fast on other platforms so
+/// callers can fall back to environment variables or user secrets.
+///
public class DpapiCredentialProvider : ICredentialProvider
{
private sealed record StoredCredentials(string Username, string Password, string SqlConnectionString);
@@ -17,13 +20,13 @@ private sealed record StoredCredentials(string Username, string Password, string
public (string Username, string Password, string SqlConnectionString)? GetCredentials()
{
- try
+ if (!OperatingSystem.IsWindows() || !File.Exists(FilePath))
{
- if (!File.Exists(FilePath))
- {
- return null;
- }
+ return null;
+ }
+ try
+ {
byte[] encrypted = File.ReadAllBytes(FilePath);
byte[] decrypted = ProtectedData.Unprotect(encrypted, null, DataProtectionScope.CurrentUser);
string plaintext = Encoding.UTF8.GetString(decrypted);
@@ -39,15 +42,22 @@ private sealed record StoredCredentials(string Username, string Password, string
}
}
+ [SupportedOSPlatform("windows")]
public static void SaveCredentials(string username, string password, string connectionString)
{
- string dir = Path.GetDirectoryName(FilePath);
- Directory.CreateDirectory(dir);
+ if (!OperatingSystem.IsWindows())
+ {
+ throw new PlatformNotSupportedException("DPAPI credential storage is only available on Windows.");
+ }
ArgumentException.ThrowIfNullOrWhiteSpace(username);
ArgumentNullException.ThrowIfNull(password);
ArgumentNullException.ThrowIfNull(connectionString);
+ string dir = Path.GetDirectoryName(FilePath)
+ ?? throw new InvalidOperationException("Could not determine the credential storage directory.");
+ Directory.CreateDirectory(dir);
+
string plaintext = JsonSerializer.Serialize(
new StoredCredentials(username, password, connectionString));
byte[] data = Encoding.UTF8.GetBytes(plaintext);
diff --git a/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs b/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs
index d97179e..6cbe44d 100644
--- a/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs
+++ b/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs
@@ -1,20 +1,15 @@
-using System;
-
namespace SPO2SQL.Security;
public class EnvCredentialProvider : ICredentialProvider
{
public (string Username, string Password, string SqlConnectionString)? GetCredentials()
{
- string user = Environment.GetEnvironmentVariable("SPO_USERNAME");
- string pass = Environment.GetEnvironmentVariable("SPO_PASSWORD");
- string conn = Environment.GetEnvironmentVariable("SQL_CONNECTION_STRING");
-
- if (!string.IsNullOrEmpty(user) && !string.IsNullOrEmpty(pass))
- {
- return (user, pass, conn ?? string.Empty);
- }
+ string user = Environment.GetEnvironmentVariable("SPO_USERNAME") ?? string.Empty;
+ string pass = Environment.GetEnvironmentVariable("SPO_PASSWORD") ?? string.Empty;
+ string conn = Environment.GetEnvironmentVariable("SQL_CONNECTION_STRING") ?? string.Empty;
- return null;
+ return !string.IsNullOrEmpty(user) && !string.IsNullOrEmpty(pass)
+ ? (user, pass, conn)
+ : null;
}
}
diff --git a/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs b/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs
index 49c46d9..742fabf 100644
--- a/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs
+++ b/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs
@@ -1,5 +1,3 @@
-using System;
-using Microsoft.SharePoint.Client;
using SPO2SQL.Logging;
namespace SPO2SQL.SharePoint;
@@ -15,7 +13,7 @@ public ActivitiesDQ(SPOUser user) : base(user)
{
}
- public override bool Execute()
+ public override async Task ExecuteAsync(CancellationToken cancellationToken = default)
{
try
{
@@ -28,7 +26,8 @@ public override bool Execute()
"" +
"";
- var activitiesList = CreateAndBuildList(LIST_NAME, SITE, camlQuery);
+ using var activitiesList = await CreateAndBuildListAsync(LIST_NAME, SITE, camlQuery, cancellationToken)
+ .ConfigureAwait(false);
if (activitiesList.ItemCollection.Count == 0)
{
@@ -36,7 +35,7 @@ public override bool Execute()
return true;
}
- ProcessListItemsInBatches(activitiesList, item =>
+ await ProcessListItemsInBatchesAsync(activitiesList, item =>
{
var sourceValue = GetFieldValue