diff --git a/.editorconfig b/.editorconfig index df666d6..febd864 100644 --- a/.editorconfig +++ b/.editorconfig @@ -35,7 +35,7 @@ trim_trailing_whitespace = false [*.cs] # New line preferences -end_of_line = crlf +end_of_line = lf insert_final_newline = true # Indentation preferences diff --git a/.gitattributes b/.gitattributes index 1ff0c42..f33c92b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,10 @@ ############################################################################### * text=auto +# C# sources are normalized to LF so dotnet format behaves identically on +# Windows and Linux agents. +*.cs text eol=lf + ############################################################################### # Set default behavior for command prompt diff. # diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfe6005..d7df891 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,12 @@ env: jobs: build: - name: Build & Test - runs-on: windows-latest + name: Build & Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest ] timeout-minutes: 15 steps: @@ -54,4 +58,4 @@ jobs: retention-days: 14 - name: Audit NuGet packages - run: dotnet nuget audit + run: dotnet restore --force /p:NuGetAudit=true /p:NuGetAuditMode=all /p:NuGetAuditLevel=moderate diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8dee01..b45d301 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,5 +56,5 @@ format: nuget_audit: stage: security script: - - dotnet nuget audit + - dotnet restore --force /p:NuGetAudit=true /p:NuGetAuditMode=all /p:NuGetAuditLevel=moderate allow_failure: true diff --git a/Directory.Build.props b/Directory.Build.props index 48a8fbf..a7f00b1 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,6 +11,11 @@ 2.0.0.0 2.0.0 spo2sql-modernization-secrets - $(NoWarn);CS8618;CS8600;CS8601;CS8602;CS8603;CS8604;CS8625;CS8605;CS8605;CA1416;NU1603;CS0618;CS0649;CA1305;CA1310;CA1311;CA1304;CA1846;CA1862;CA1866;CA5350;CA2201;CA1805 + $(NoWarn);NU1603 + + + + + diff --git a/README.md b/README.md index dcab547..5a7167f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ - Per-list column mapping, destination names, data types, and ignore rules. - Retry handling for transient SharePoint and network failures. - Startup health checks for SharePoint and SQL connectivity. -- Operation statistics, structured logging, and cancellation support. +- End-to-end async I/O with cancellation, per-list operation statistics and structured logging. - Credential providers for environment variables, Windows DPAPI, and application configuration. - Automated build, test, formatting, dependency-audit, and CodeQL workflows. @@ -53,7 +53,9 @@ docs/ Configuration, security, modernization, and operational - SQL Server access with permission to read source metadata and create/update destination tables - A SharePoint account or supported application identity configured for the target tenant -The current authentication path uses username/password token acquisition. Environments requiring interactive MFA or Conditional Access should use an app-only/service-principal authentication design instead of relying on password-based authentication. +Two authentication modes are supported. For unattended jobs, configure app-only authentication by setting `SharePoint:TenantId`, `SharePoint:ClientId` and `SharePoint:CertificatePath` (plus `CertificatePassword` when the PFX is protected); this uses client credentials with a certificate and is not blocked by MFA or Conditional Access. When those values are absent, the legacy username/password flow is used as a fallback, which will fail in tenants that enforce MFA. + +Run `dotnet run -- --setup-secrets` to store credentials in the current user's DPAPI-protected store (Windows only). ## Build and test @@ -98,7 +100,11 @@ The console application binds these sections: "Password": "", "TimeoutSeconds": 120, "MaxRetries": 3, - "InitialRetryDelayMs": 1000 + "InitialRetryDelayMs": 1000, + "TenantId": "", + "ClientId": "", + "CertificatePath": "", + "CertificatePassword": "" }, "Sql": { "ConnectionString": "Server=.;Database=SPO2SQL;Integrated Security=true;Encrypt=true;TrustServerCertificate=false;", @@ -160,7 +166,7 @@ The detailed security guidance is available in [`docs/SECURITY.md`](docs/SECURIT ## CI/CD -GitHub Actions runs on pushes and pull requests: +GitHub Actions runs on pushes and pull requests on both Linux and Windows: 1. Restore NuGet dependencies. 2. Build with warnings treated as errors. diff --git a/UPGRADE_SUMMARY.md b/UPGRADE_SUMMARY.md index 246d371..3358eb7 100644 --- a/UPGRADE_SUMMARY.md +++ b/UPGRADE_SUMMARY.md @@ -181,13 +181,17 @@ Logger (Enhanced) --- -## Future Improvements - -- [ ] Add async/await support for SharePoint operations -- [ ] Implement connection pooling for SQL Server -- [ ] Add retry logic with exponential backoff -- [ ] Create configuration service layer -- [ ] Add metrics/performance monitoring -- [ ] Implement proper dependency injection -- [ ] Add unit test suite -- [ ] Add telemetry for debugging +## Status of Previously Planned Improvements + +This summary describes an earlier modernization pass. The items below were +subsequently delivered in the current codebase (see `docs/PROGRESS.md`): + +- [x] Async/await for SharePoint and SQL operations, with cancellation +- [x] Retry logic with exponential backoff and jitter (`RetryPolicy`) +- [x] Configuration service layer (`SyncSettings` plus validated options) +- [x] Per-list metrics logging via `OperationStatistics` +- [x] Dependency injection in the .NET host +- [x] Unit test suite across both projects +- [x] App-only authentication, real health checks and warning-free nullable build + +Still open: OpenTelemetry export and live integration tests. diff --git a/docs/LoggerMigrationGuide.md b/docs/LoggerMigrationGuide.md deleted file mode 100644 index f0f3051..0000000 --- a/docs/LoggerMigrationGuide.md +++ /dev/null @@ -1,492 +0,0 @@ -# Logger Migration Guide: From Legacy Logger to ILogger - -## Overview - -This guide helps you migrate from the legacy static `Logger` class to Microsoft's modern `ILogger` interface, which is the standard logging abstraction in .NET. - ---- - -## Key Differences - -| Aspect | Legacy Logger | Modern ILogger | -|--------|--------------|------------------| -| **Access Pattern** | Static class (`Logger.LogError(...)`) | Dependency injection (`_logger.LogError(...)`) | -| **Verbosity Control** | Custom integer levels (0-3) | Standard `LogLevel` enum | -| **Testability** | Difficult (static dependencies) | Easy (inject mock logger) | -| **Configuration** | Manual property setting | Configured via `appsettings.json` | -| **Providers** | Console only | Console, File, Event Log, Application Insights, etc. | -| **Type Safety** | None | Generic type parameter identifies log source | -| **Structured Logging** | No | Yes (with parameters) | -| **Scoping** | No | Yes (with `BeginScope()`) | -| **Performance** | Basic | Optimized with source generators | - ---- - -## Verbosity Level Migration - -### Legacy Verbosity Levels -```csharp -// Old: 0=silent, 1=errors, 2=warnings, 3=debug -Logger.VerboseLevel = 2; // Show errors and warnings only -``` - -### Modern LogLevel Mapping -```csharp -// New: Configure in appsettings.json -{ - "Logging": { - "LogLevel": { - "Default": "Warning", - "Microsoft": "Warning", - "Bring": "Information" - } - } -} -``` - -**Mapping Table:** - -| Legacy VerboseLevel | Modern LogLevel | What Gets Logged | -|---------------------|-----------------|------------------| -| 0 (silent) | `None` | Nothing | -| 1 (errors only) | `Error` | Errors only | -| 2 (warnings+) | `Warning` | Errors and warnings | -| 3 (debug+) | `Debug` or `Trace` | Everything including debug info | - ---- - -## Log Level Migration Examples - -### 1. Error Logging - -**Before (Legacy):** -```csharp -using Bring.SPODataQuality; - -try -{ - // ... some operation -} -catch (Exception ex) -{ - Logger.LogError("Failed to connect to SharePoint", ex); -} -``` - -**After (Modern):** -```csharp -using Microsoft.Extensions.Logging; - -public class MyService -{ - private readonly ILogger _logger; - - public MyService(ILogger logger) - { - _logger = logger; - } - - public void DoWork() - { - try - { - // ... some operation - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to connect to SharePoint"); - } - } -} -``` - -**Key Improvements:** -- Exception is the first parameter (better IntelliSense) -- Automatic stack trace inclusion -- Type-safe: `ILogger` identifies log source -- Dependency injection enables testing - ---- - -### 2. Warning Logging - -**Before (Legacy):** -```csharp -Logger.LogWarning("No items found in list 'Documents'"); -``` - -**After (Modern):** -```csharp -_logger.LogWarning("No items found in list '{ListName}'", "Documents"); -``` - -**Key Improvements:** -- Structured logging with named parameters -- Parameters extracted for log aggregation -- Better searchability in log management systems - ---- - -### 3. Debug Logging - -**Before (Legacy):** -```csharp -Logger.LogDebug($"Processing item {itemId} with status {status}"); -``` - -**After (Modern):** -```csharp -_logger.LogDebug("Processing item {ItemId} with status {Status}", itemId, status); -``` - -**Key Improvements:** -- No string interpolation needed -- Parameters logged separately (structured logging) -- Better performance (no allocation if debug logging disabled) -- Searchable by ItemId or Status in log analytics - ---- - -### 4. Information Logging (New!) - -The legacy Logger didn't have an info level. Use this for important operational messages: - -```csharp -_logger.LogInformation("SharePoint sync started at {StartTime}", DateTime.Now); -_logger.LogInformation("Processed {ItemCount} items in {Duration}ms", count, elapsed); -``` - ---- - -## Common Migration Patterns - -### Pattern 1: Static Logger Usage Throughout Class - -**Before:** -```csharp -public class SharePointService -{ - public void DownloadFiles() - { - Logger.LogDebug("Starting file download"); - - try - { - // ... download logic - Logger.LogDebug("Downloaded 5 files successfully"); - } - catch (Exception ex) - { - Logger.LogError("Download failed", ex); - } - } -} -``` - -**After:** -```csharp -public class SharePointService -{ - private readonly ILogger _logger; - - public SharePointService(ILogger logger) - { - _logger = logger; - } - - public void DownloadFiles() - { - _logger.LogDebug("Starting file download"); - - try - { - // ... download logic - _logger.LogInformation("Downloaded {FileCount} files successfully", 5); - } - catch (Exception ex) - { - _logger.LogError(ex, "Download failed"); - } - } -} -``` - ---- - -### Pattern 2: Verbosity Level Checking - -**Before:** -```csharp -if (Logger.VerboseLevel >= 3) -{ - Logger.LogDebug($"Details: {expensiveOperation()}"); -} -``` - -**After:** -```csharp -// Option 1: Let ILogger handle it (recommended) -_logger.LogDebug("Details: {Details}", expensiveOperation()); - -// Option 2: Explicit check for expensive operations -if (_logger.IsEnabled(LogLevel.Debug)) -{ - _logger.LogDebug("Details: {Details}", expensiveOperation()); -} -``` - -**Note:** The second approach only necessary if `expensiveOperation()` is truly expensive, as ILogger optimizes away disabled log levels. - ---- - -### Pattern 3: Custom Log Level Method - -**Before:** -```csharp -Logger.Log(2, "Custom warning message"); -``` - -**After:** -```csharp -_logger.Log(LogLevel.Warning, "Custom warning message"); - -// Or use the specific helper: -_logger.LogWarning("Custom warning message"); -``` - ---- - -## Using the LoggerAdapter (Transition Strategy) - -If you have a large codebase and can't migrate everything at once, use the `LoggerAdapter`: - -```csharp -using Bring.Adapters; -using Microsoft.Extensions.Logging; - -public class LegacyService -{ - public LegacyService(ILogger modernLogger) - { - // Wrap modern logger to support legacy Logger API - var adapter = new LoggerAdapter(modernLogger); - - // Configure legacy verbosity level - adapter.VerboseLevel = 3; - - // Now legacy code can work unchanged - adapter.LogError("Legacy error message"); - adapter.LogWarning("Legacy warning message"); - adapter.LogDebug("Legacy debug message"); - } -} -``` - -**When to use the adapter:** -- Large codebase with many Logger references -- Gradual migration strategy -- Testing migration incrementally -- Supporting legacy code that can't be changed immediately - -**When NOT to use the adapter:** -- New code (use ILogger directly) -- Small codebases (just migrate directly) -- When you need structured logging features - ---- - -## Configuration Migration - -### Legacy Configuration -```csharp -// Somewhere in Program.cs or startup -Logger.VerboseLevel = args.Contains("--verbose") ? 3 : 1; -``` - -### Modern Configuration - -**appsettings.json:** -```json -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft": "Warning", - "Microsoft.Hosting.Lifetime": "Information", - "Bring": "Debug" - }, - "Console": { - "FormatterName": "simple", - "FormatterOptions": { - "TimestampFormat": "yyyy-MM-dd HH:mm:ss.fff ", - "SingleLine": true, - "IncludeScopes": false - } - } - } -} -``` - -**appsettings.Development.json:** -```json -{ - "Logging": { - "LogLevel": { - "Default": "Debug", - "Bring": "Trace" - } - } -} -``` - -**Command-line override:** -```bash -dotnet run --Logging:LogLevel:Default=Debug -``` - ---- - -## Best Practices - -### ✅ DO - -1. **Use structured logging:** - ```csharp - _logger.LogInformation("User {UserId} logged in from {IpAddress}", userId, ip); - ``` - -2. **Pass exceptions as first parameter:** - ```csharp - _logger.LogError(ex, "Failed to process {FileName}", fileName); - ``` - -3. **Use appropriate log levels:** - - `Trace`: Very detailed (e.g., entering/exiting methods) - - `Debug`: Developer information during debugging - - `Information`: Important business events - - `Warning`: Recoverable errors or unexpected situations - - `Error`: Failures that stop current operation - - `Critical`: Catastrophic failures requiring immediate attention - -4. **Check log level for expensive operations:** - ```csharp - if (_logger.IsEnabled(LogLevel.Debug)) - { - _logger.LogDebug("Complex data: {Data}", SerializeComplexObject()); - } - ``` - -### ❌ DON'T - -1. **Don't use string interpolation:** - ```csharp - // BAD - loses structured logging - _logger.LogInformation($"Processing {itemId}"); - - // GOOD - structured logging - _logger.LogInformation("Processing {ItemId}", itemId); - ``` - -2. **Don't catch exceptions just to log them:** - ```csharp - // BAD - try { DoWork(); } - catch (Exception ex) { _logger.LogError(ex, "Error"); throw; } - - // GOOD - let exceptions bubble, log at appropriate handler level - ``` - -3. **Don't log sensitive data:** - ```csharp - // BAD - _logger.LogDebug("Password: {Password}", password); - - // GOOD - _logger.LogDebug("Authentication attempted for user {Username}", username); - ``` - ---- - -## Testing with ILogger - -### Legacy Logger (Hard to Test) -```csharp -public void TestMethod() -{ - // No way to verify logging without capturing Console.Out - var service = new SharePointService(); - service.DoWork(); - // Can't assert what was logged -} -``` - -### Modern ILogger (Easy to Test) -```csharp -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Xunit; - -public class SharePointServiceTests -{ - [Fact] - public void DoWork_LogsInformation() - { - // Arrange - var logger = new FakeLogger(); - var service = new SharePointService(logger); - - // Act - service.DoWork(); - - // Assert - Assert.Contains(logger.LoggedMessages, - m => m.Level == LogLevel.Information && m.Message.Contains("Downloaded")); - } - - [Fact] - public void DoWork_WithNullLogger_StillWorks() - { - // NullLogger for tests where logging doesn't matter - var service = new SharePointService(NullLogger.Instance); - service.DoWork(); // No exceptions - } -} -``` - ---- - -## Migration Checklist - -- [ ] Update all service constructors to accept `ILogger` -- [ ] Replace `Logger.LogError()` with `_logger.LogError()` -- [ ] Replace `Logger.LogWarning()` with `_logger.LogWarning()` -- [ ] Replace `Logger.LogDebug()` with `_logger.LogDebug()` -- [ ] Replace `Logger.Log()` with `_logger.Log()` -- [ ] Convert string interpolation to structured logging parameters -- [ ] Update exception logging to pass exception as first parameter -- [ ] Configure logging in `appsettings.json` instead of `Logger.VerboseLevel` -- [ ] Update unit tests to inject logger mocks -- [ ] Remove `using Bring.SPODataQuality;` (legacy namespace) -- [ ] Add `using Microsoft.Extensions.Logging;` -- [ ] Consider using `LoggerAdapter` for gradual migration - ---- - -## Additional Resources - -- [Microsoft Logging Documentation](https://learn.microsoft.com/en-us/dotnet/core/extensions/logging) -- [High-performance logging](https://learn.microsoft.com/en-us/dotnet/core/extensions/high-performance-logging) -- [Logging Best Practices](https://learn.microsoft.com/en-us/dotnet/core/extensions/logging-best-practices) -- Project: `Application.cs` - Example of modern ILogger usage - ---- - -## Questions? - -If you encounter issues during migration, check: -1. Is the service registered in DI? (see `Program.cs`) -2. Is logging configured in `appsettings.json`? -3. Are you using `ILogger` where T is your class name? -4. Have you removed static `Logger` references? - -For gradual migration, use the `LoggerAdapter` class in `ConsoleApp1Net8/Adapters/LoggerAdapter.cs`. diff --git a/docs/MODERNIZATION-STATUS.md b/docs/MODERNIZATION-STATUS.md index 462e399..f9fbf4a 100644 --- a/docs/MODERNIZATION-STATUS.md +++ b/docs/MODERNIZATION-STATUS.md @@ -1,211 +1,39 @@ -# .NET 8 Modernization Status Report +# Modernization Status -**Last Updated**: 2026-03-18 14:25 UTC +**Last updated:** 2026-11-18 -## Overall Progress +## Summary -**Completed**: 9 of 27 tasks (33%) ✅ -**In Progress**: 3 tasks (11%) 🚧 -**Pending**: 15 tasks (56%) 📋 +The modernization plan from the original migration documents has been completed. +The previous phase percentages in this document were snapshots of an older state +and no longer describe the codebase. ---- +| Original phase | Outcome | +| --- | --- | +| Foundation & infrastructure | Completed: .NET 8 host, DI, options, multi-source configuration | +| Core services modernization | Completed: async SharePoint/SQL/DQ pipeline, cancellation, paging | +| Modern C# features | Completed: records, pattern matching, file-scoped namespaces, nullable enabled | +| Architecture | Completed: `SyncSettings`, credential chain, health checks, retry policy | +| Security | Completed: app-only certificate authentication, encrypted connection enforcement | +| Testing | Unit suite in place; live integration tests remain environment-specific | +| Documentation | README, configuration guide and security guide updated | -## ✅ Completed Tasks (9) - -### Phase 1: Foundation & Infrastructure (100% Complete) -1. ✅ **Add Modern NuGet Dependencies** - All Microsoft.Extensions.* packages added, Polly integrated -2. ✅ **Create New Configuration System** - IOptions with strongly-typed records (SharePoint, SQL, Application) -3. ✅ **Setup Dependency Injection** - IHostBuilder with full service registration -4. ✅ **Implement Global Usings** - Common namespaces centralized in GlobalUsings.cs - -### Phase 2: Logging & Configuration (100% Complete) -5. ✅ **Implement Structured Logging** - Production-ready ILogger with: - - LoggerMessage source generators (zero-allocation logging) - - Correlation IDs for operation tracking - - Log scopes for contextual information - - 12 semantic log methods with EventIds - - Application.cs grew from 72 to 298 lines with comprehensive logging - -### Phase 3: Modern C# Features (67% Complete) -6. ✅ **Use File-Scoped Namespaces** - All files already using modern syntax -7. ✅ **Implement Pattern Matching** - Comprehensive OperationResult utility (416 lines): - - Success/Failure discriminated union pattern - - Switch expressions, property patterns, list patterns - - 10+ extension methods demonstrating advanced patterns - - ValidationResult and ResultSummary types - - 12 usage examples in Application.cs - -### Phase 5: Security (50% Complete) -8. ✅ **Modernize Configuration** - Enhanced Application.cs with best practices -9. ✅ **Secure Credential Management** - Complete security infrastructure: - - User Secrets for development - - Environment variables for production - - SECURITY.md (578 lines) with comprehensive best practices - - setup-secrets.sh (bash) and setup-secrets.ps1 (PowerShell) scripts - - Credential rotation procedures - - Audit logging recommendations - ---- - -## 🚧 In Progress (3) - -### Phase 2: Core Services -10. 🚧 **Modernize Logger Usage** - Creating migration guide and adapter pattern -11. 🚧 **Modern LINQ Features** - Building LinqExtensions with .NET 6+ features -12. 🚧 **Use Records for DTOs** - Creating record-based data models - ---- - -## 📋 Pending Tasks (15) - -### Phase 2: Core Services Modernization -- Convert SharePoint Components to Async (SPOUser, SPOList, Context) -- Convert SQL Components to Async (SQLInteraction, RefreshSQLLists) -- Convert Data Quality Components to Async - -### Phase 3: Modern C# Features -- Enable Nullable Reference Types fully - -### Phase 4: Architecture Improvements -- Create Service Interfaces (ISharePointService, ISqlService, IDataQualityService) -- Refactor to Service Pattern -- Enhance Retry with Polly -- Improve Health Checks - -### Phase 5: Security -- Enhance Connection Security - -### Phase 6: Testing & Quality -- Add Unit Tests Foundation -- Add Integration Tests -- Improve Code Quality - -### Phase 7: Documentation -- Update Main Documentation -- Create Config Migration Tools -- Create Deployment Guide - ---- - -## 📊 Key Achievements - -### Code Metrics -- **Application.cs**: 72 → 298 lines (314% growth with production features) -- **OperationResult.cs**: 416 lines of reusable pattern matching utilities -- **SECURITY.md**: 578 lines of comprehensive security guidance -- **New Files Created**: 15+ -- **Scripts**: 2 (PowerShell + Bash for secret setup) - -### Architecture Improvements -1. **Dependency Injection**: Full IHost integration with service registration -2. **Configuration**: Multi-layer (JSON → Secrets → Env Vars → CLI) -3. **Logging**: Zero-allocation LoggerMessage with correlation tracking -4. **Security**: Never-commit-secrets architecture -5. **Modern C#**: Records, pattern matching, file-scoped namespaces - -### Developer Experience -- ✅ Clear separation of concerns -- ✅ Type-safe configuration with validation -- ✅ Comprehensive XML documentation -- ✅ Security-first defaults -- ✅ Production-ready error handling - ---- - -## 🎯 Next Steps - -### Immediate (Wave 3 - In Progress) -1. Complete logger migration adapter -2. Add modern LINQ utilities -3. Create record-based DTOs - -### Wave 4 (Ready to Start) -1. Async conversion of SharePoint components -2. Async conversion of SQL components -3. Nullable reference type enforcement - -### Wave 5 (Dependent on Wave 4) -1. Service interfaces and patterns -2. Polly integration for resilience -3. Health check improvements - ---- - -## 📁 Project Structure +## Architecture today ``` -SPOtoSQL-Net8/ -├── ConsoleApp1Net8/ -│ ├── Configuration/ ✅ Strongly-typed options -│ │ ├── ApplicationOptions.cs -│ │ ├── SharePointOptions.cs -│ │ └── SqlOptions.cs -│ ├── Utilities/ ✅ Modern C# utilities -│ │ └── OperationResult.cs (Pattern matching, 416 lines) -│ ├── Models/ 🚧 Record-based DTOs (in progress) -│ ├── Adapters/ 🚧 Logger adapter (in progress) -│ ├── Program.cs ✅ IHost with DI -│ ├── Application.cs ✅ Structured logging (298 lines) -│ ├── GlobalUsings.cs ✅ Common namespaces -│ ├── appsettings.json ✅ Default configuration -│ ├── appsettings.Development.json ✅ Dev overrides -│ └── ConsoleApp1Net8.csproj ✅ Modern packages -├── scripts/ ✅ Setup automation -│ ├── setup-secrets.sh ✅ Bash script -│ └── setup-secrets.ps1 ✅ PowerShell script -├── docs/ 🚧 Documentation (in progress) -├── SECURITY.md ✅ 578 lines of security best practices -├── README-MODERNIZATION.md ✅ Setup and migration guide -├── PROGRESS.md ✅ Detailed progress tracking -└── MODERNIZATION-STATUS.md ✅ This file - -Legacy Code (to be modernized): -SPOtoSQL-Snapshots/ConsoleApp1/ 📋 Original .NET Framework 4.8 code +src/SPO2SQL.Console Host, options, validation, health checks, exit code, CLI setup +src/SPO2SQL.Core SharePoint access, SQL transfer, credentials, XML list mappings +tests/ xUnit unit tests for both projects ``` ---- - -## 🚀 Fleet Mode Performance - -**Active Parallel Agents**: 3 (modernize-logger-usage, modern-linq, use-records) -**Completed Agent Runs**: 6 -**Total Agent Time**: ~15 minutes -**Actual Elapsed Time**: ~6 minutes (2.5x speedup from parallelization) - ---- - -## 💡 Lessons Learned - -1. **Configuration First**: Having strong configuration foundation enables everything else -2. **Logging Matters**: Investing in proper logging pays dividends in production -3. **Records are Powerful**: Pattern matching with records dramatically improves code clarity -4. **Security by Default**: User Secrets + Environment Variables = zero secrets in source -5. **Parallel Execution**: Fleet mode accelerated development significantly - ---- - -## 📈 Quality Indicators - -- ✅ All configuration validated on startup -- ✅ Comprehensive XML documentation -- ✅ Security best practices documented -- ✅ Zero secrets in source code -- ✅ Production-ready logging patterns -- ✅ Modern C# features consistently applied -- ✅ Backward compatibility maintained (legacy code still works) - ---- - -## 🎓 Learning Resources Created - -1. **SECURITY.md** - Complete security guide for production deployments -2. **README-MODERNIZATION.md** - Setup and configuration guide -3. **Application.cs** - 12 logging examples showing best practices -4. **OperationResult.cs** - 10+ pattern matching examples -5. **Setup Scripts** - Automated secret configuration +The host maps validated options into `SyncSettings` once and passes it through the +sync pipeline. The legacy XML file is used only for list/column mappings; secrets +come from the credential provider chain (environment, DPAPI on Windows, or the +values supplied through configuration/user secrets). ---- +## Known limitations -*For detailed task breakdown, see PROGRESS.md* -*For security guidelines, see SECURITY.md* -*For setup instructions, see README-MODERNIZATION.md* +- Username/password authentication is retained as a fallback and will not work in + tenants that require interactive MFA. Use app-only authentication instead. +- Metrics are logged per list; there is no metrics exporter yet. +- `dotnet format` expects LF line endings, enforced for C# files in `.gitattributes`. diff --git a/docs/MODERNIZATION.md b/docs/MODERNIZATION.md index 37c9e14..93a529d 100644 --- a/docs/MODERNIZATION.md +++ b/docs/MODERNIZATION.md @@ -6,12 +6,12 @@ This document explains the modern .NET 8 architecture and how to configure and r ### Modern Architecture - **Dependency Injection**: Full DI container with IHost pattern -- **Async/Await**: All I/O operations use async patterns (work in progress) -- **Structured Logging**: ILogger with semantic log properties -- **Configuration**: appsettings.json with IOptions pattern -- **Security**: User Secrets (dev) and Environment Variables (prod) for credentials -- **Resilience**: Polly integration for retry policies (coming soon) -- **Testing**: Unit and integration test support (coming soon) +- **Async/Await**: All SharePoint, SQL and data-quality I/O uses async patterns with cancellation +- **Structured Logging**: ILogger in the host, bridged to the Core library +- **Configuration**: appsettings.json with validated IOptions, mapped to a single `SyncSettings` record +- **Security**: User Secrets (dev), environment variables and app-only certificate authentication (prod) +- **Resilience**: Cancellation-aware retry with exponential backoff and jitter +- **Testing**: xUnit unit tests; integration tests require a live tenant/SQL Server ### Modern C# Features - Records for immutable configuration @@ -25,23 +25,19 @@ This document explains the modern .NET 8 architecture and how to configure and r ### Development Setup (User Secrets) -**Quick Setup with Helper Script**: +**Quick Setup**: ```bash -# Interactive setup wizard -./scripts/setup-secrets.sh +# Stores credentials in the current user's DPAPI-protected store (Windows) +dotnet run --project src/SPO2SQL.Console/SPO2SQL.Console.csproj -- --setup-secrets ``` -The setup script will: -- ✅ Initialize User Secrets automatically -- ✅ Guide you through configuration -- ✅ Validate email addresses and connection strings -- ✅ Ensure security best practices +For development, user secrets remain the preferred option; see the manual setup below. **Manual Setup**: 1. **Initialize User Secrets** (one-time setup): ```bash -cd SPOtoSQL-Net8/ConsoleApp1Net8 +cd src/SPO2SQL.Console dotnet user-secrets init ``` @@ -105,7 +101,7 @@ export SPO2SQL_Sql__ConnectionString="Server=myServer;Database=myDB;Integrated S ### Development ```bash -cd SPOtoSQL-Net8/ConsoleApp1Net8 +cd src/SPO2SQL.Console dotnet restore dotnet build dotnet run @@ -115,7 +111,7 @@ dotnet run ```bash dotnet publish -c Release -r win-x64 --self-contained false cd bin/Release/net8.0/win-x64/publish -./ConsoleApp1Net8.exe +dotnet run --project src/SPO2SQL.Console/SPO2SQL.Console.csproj ``` ## Logging @@ -165,7 +161,7 @@ Available log levels: Both versions can coexist: - **.NET Framework 4.8**: `SPOtoSQL-Snapshots/ConsoleApp1/` -- **.NET 8**: `SPOtoSQL-Net8/ConsoleApp1Net8/` +- **.NET 8**: `src/SPO2SQL.Console/` The .NET 8 version links to the same source files for now, so changes are shared. @@ -189,25 +185,15 @@ Key security practices: dotnet user-secrets set "SharePoint:Password" "new-password" ``` -### Helper Scripts +### Credential Setup -**Development Setup**: ```bash -# Interactive setup wizard (Linux/Mac) -./scripts/setup-secrets.sh - -# Interactive setup wizard (Windows) -.\scripts\setup-secrets.ps1 - -# Verify configuration -./scripts/verify-config.sh +# Windows: store credentials in the current user's DPAPI-protected store +dotnet run --project src/SPO2SQL.Console/SPO2SQL.Console.csproj -- --setup-secrets ``` -These scripts help you: -- ✅ Set up User Secrets securely -- ✅ Validate input (email format, connection strings, etc.) -- ✅ Verify configuration without exposing credentials -- ✅ Check for common security issues +Configuration is validated at startup by the host, so an invalid connection +string or an incomplete authentication configuration fails fast with a clear message. ## Troubleshooting diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 86cdfb8..fd87283 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -1,188 +1,40 @@ -# .NET 8 Modernization Progress - -## Completed Tasks ✅ - -### Phase 1: Foundation & Infrastructure (100% Complete) - -1. **Add Modern NuGet Dependencies** ✅ - - Added Microsoft.Extensions.* packages (DI, Configuration, Logging, Hosting, Options) - - Updated SharePoint CSOM to 16.1.24816.12000 - - Updated SqlClient to 5.2.2 - - Added Polly 8.4.2 for resilience - - Added Health Checks support - -2. **Create Configuration System** ✅ - - Created strongly-typed configuration records: - - `SharePointOptions` - SP credentials and settings - - `SqlOptions` - SQL connection and batch settings - - `ApplicationOptions` - App behavior and logging - - Implemented IOptions pattern with validation - - Created appsettings.json with sensible defaults - - Created appsettings.Development.json for dev overrides - - Added User Secrets support (ID: spo2sql-modernization-secrets) - - Environment variable support with SPO2SQL_ prefix - -3. **Setup Dependency Injection** ✅ - - Created modern Program.cs with IHostBuilder - - Configured configuration sources (JSON, secrets, env vars, command line) - - Registered options with validation - - Created Application class as IHostedService - - Implemented graceful startup and shutdown - -4. **Implement Global Usings** ✅ - - Created GlobalUsings.cs with common namespaces - - Reduced repetitive using statements across project - -### Additional Improvements - -5. **Security Enhancements** ✅ - - User Secrets for development credentials - - Environment variables for production - - Created secrets.json.template for easy setup - - Added .gitignore to prevent secret commits - - Documented security best practices - -6. **Code Quality** ✅ - - Added .editorconfig with C# coding standards - - Enabled nullable reference types - - File-scoped namespace preference configured - - Consistent formatting rules - -7. **Documentation** ✅ - - Created README-MODERNIZATION.md with: - - Architecture overview - - Configuration setup (dev & prod) - - Migration guide from .NET Framework - - Security best practices - - Troubleshooting guide - - Created PROGRESS.md (this file) - -## Next Tasks 🚧 - -### Phase 2: Core Services Modernization - -1. **Modernize Logger** (Ready to start) - - Replace Logger with ILogger - - Update all classes to use injected logger - - Add structured logging properties - -2. **Convert SharePoint Components to Async** (Ready to start) - - Update SPOUser, SPOList, Context to async/await - - Use ExecuteQueryAsync instead of ExecuteQuery - - Add CancellationToken support - -3. **Convert SQL Components to Async** (Ready to start) - - Update SQLInteraction to async patterns - - Use async ADO.NET methods - - Add CancellationToken support - -4. **Modernize Configuration Components** (Depends on logger) - - Replace ConfigHelper usage with IOptions - - Remove XML configuration dependencies - -5. **Convert Data Quality to Async** (Depends on SharePoint & SQL async) - - Update DataQualityBase to async - - Update all DQ classes (Activities, InvoiceRequest, Timesheet) - - Update RefreshSPOLists orchestration - -### Phase 3: Modern C# Features - -- Use Records for DTOs -- Implement Pattern Matching -- Leverage Modern LINQ -- Enable Nullable Reference Types fully -- Use File-Scoped Namespaces throughout - -### Phase 4: Architecture - -- Create Service Interfaces -- Refactor to Service Pattern -- Enhance Retry with Polly -- Improve Health Checks - -### Phase 5: Security - -- Implement secure credential management -- Enhance connection security - -### Phase 6: Testing - -- Add unit tests -- Add integration tests -- Improve code quality metrics - -### Phase 7: Documentation - -- Update main README -- Create config migration tools -- Create deployment guide - -## Current Structure - -``` -SPOtoSQL-Net8/ -├── ConsoleApp1Net8/ -│ ├── Configuration/ # ✅ Modern config classes -│ │ ├── ApplicationOptions.cs -│ │ ├── SharePointOptions.cs -│ │ └── SqlOptions.cs -│ ├── GlobalUsings.cs # ✅ Global usings -│ ├── Program.cs # ✅ Modern entry point with DI -│ ├── Application.cs # ✅ Main app as hosted service -│ ├── appsettings.json # ✅ Default configuration -│ ├── appsettings.Development.json # ✅ Dev overrides -│ ├── secrets.json.template # ✅ Template for User Secrets -│ ├── .gitignore # ✅ Security -│ └── ConsoleApp1Net8.csproj # ✅ Updated with modern packages -├── README-MODERNIZATION.md # ✅ Modernization docs -└── PROGRESS.md # ✅ This file - -SPOtoSQL-Snapshots/ # Legacy .NET Framework code -└── ConsoleApp1/ # (linked for now, to be modernized) -``` - -## Statistics - -- **Completed Tasks**: 7 / 27 (26%) -- **Phase 1 (Foundation)**: 4/4 (100%) ✅ -- **Phase 2 (Core Services)**: 0/5 (0%) -- **Phase 3 (C# Features)**: 1/6 (17%) -- **Phase 4 (Architecture)**: 0/4 (0%) -- **Phase 5 (Security)**: 0/2 (0%) -- **Phase 6 (Testing)**: 0/3 (0%) -- **Phase 7 (Documentation)**: 0/3 (0%) -- **Additional**: 3 extra improvements completed - -## Key Achievements - -1. **Modern Foundation**: The application now uses industry-standard .NET patterns -2. **Security First**: No credentials in source code, proper secrets management -3. **Configuration Flexibility**: JSON-based with multiple override mechanisms -4. **Validation**: Configuration validates on startup with clear error messages -5. **Logging Ready**: Structured logging infrastructure in place -6. **Extensibility**: DI container makes testing and extending easier - -## How to Test Current Progress +# Implementation Status + +## Current state (v2.0) + +The .NET 8 rewrite is complete and the legacy .NET Framework snapshot in +`SPOtoSQL-Snapshots/` is kept only as a historical reference. + +| Area | Status | +| --- | --- | +| Host, dependency injection, options validation | Done | +| Structured logging bridged from the Core library | Done | +| SharePoint paging and cancellation | Done | +| Transaction scoped to the write phase | Done | +| UTC watermark for incremental syncs | Done | +| Unified `SyncSettings` passed to the sync layer | Done | +| App-only authentication with certificate | Done | +| Real SharePoint/SQL startup health checks | Done | +| `--setup-secrets` in the real console host | Done | +| End-to-end async (CSOM, ADO.NET, retries, DQ) | Done | +| Nullable reference types enforced without blanket suppressions | Done | +| CI on Linux and Windows with format and NuGet audit | Done | + +## Validation ```bash -cd SPOtoSQL-Net8/ConsoleApp1Net8 - -# Setup secrets -dotnet user-secrets set "SharePoint:Username" "test@example.com" -dotnet user-secrets set "SharePoint:Password" "password" -dotnet user-secrets set "SharePoint:SiteUrl" "https://test.sharepoint.com" -dotnet user-secrets set "Sql:ConnectionString" "Server=test;Database=test;Integrated Security=true;" - -# Build and run -dotnet restore -dotnet build -dotnet run +dotnet restore SPO2SQL.sln +dotnet build SPO2SQL.sln --no-restore -warnaserror +dotnet test SPO2SQL.sln --no-build --no-restore +dotnet format SPO2SQL.sln --verify-no-changes --no-restore ``` -Expected output: -- Application starts -- Logs show configuration loaded -- Validates all settings -- Application name and version displayed -- Graceful shutdown +The build is warning-free; the unit test suite covers CAML generation, retry +behaviour (sync and async), configuration binding/validation, XML list mappings, +logging routing, and exit-code propagation. + +## Remaining work +- Integration tests against a real SharePoint tenant and SQL Server. +- Observability export (OpenTelemetry) behind `Application:EnableMetrics`. +- Replace the in-house `RetryPolicy` with a Polly resilience pipeline. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 26b2a49..3866f22 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -28,7 +28,7 @@ This document outlines security best practices for deploying and operating the S ```bash # Navigate to project directory -cd SPOtoSQL-Net8/ConsoleApp1Net8 +cd src/SPO2SQL.Console # Set credentials (stored encrypted outside project directory) dotnet user-secrets set "SharePoint:Username" "dev@company.com" @@ -549,7 +549,7 @@ The application automatically redacts: The `scripts/` directory contains tools to assist with secure configuration: -- **setup-secrets.sh** / **setup-secrets.ps1**: Interactive script to configure User Secrets +- **`dotnet run -- --setup-secrets`**: Interactive command to store credentials in the Windows DPAPI-protected user store; on other platforms use environment variables or user secrets. - **verify-config.sh**: Validates configuration without exposing credentials ### Documentation diff --git a/docs/Scripts-README.md b/docs/Scripts-README.md deleted file mode 100644 index ff412b1..0000000 --- a/docs/Scripts-README.md +++ /dev/null @@ -1,258 +0,0 @@ -# Helper Scripts - -This directory contains utility scripts to help with secure configuration and deployment of the SharePoint to SQL Sync Tool. - -## Scripts Overview - -### setup-secrets.sh (Linux/Mac) - -Interactive script to configure User Secrets for development. - -**Usage**: -```bash -./scripts/setup-secrets.sh -``` - -**Features**: -- ✅ Validates .NET SDK installation -- ✅ Initializes User Secrets automatically -- ✅ Validates email format for SharePoint username -- ✅ Validates SharePoint URL format -- ✅ Validates SQL connection string format -- ✅ Checks for security best practices (encryption, password length) -- ✅ Masks passwords in output -- ✅ Provides guidance for next steps - -**When to use**: -- First-time setup on a development machine -- Updating credentials -- Migrating from XML configuration - -### setup-secrets.ps1 (Windows PowerShell) - -Windows PowerShell version of the setup script with identical functionality. - -**Usage**: -```powershell -.\scripts\setup-secrets.ps1 -``` - -**Features**: -- Same as setup-secrets.sh but optimized for Windows -- Uses `Read-Host -AsSecureString` for password input -- PowerShell-native error handling - -### verify-config.sh - -Verifies that all required configuration is present and valid. - -**Usage**: -```bash -# Verify User Secrets (development) -./scripts/verify-config.sh - -# Verify Environment Variables (production) -./scripts/verify-config.sh --env-vars -``` - -**Checks performed**: -- ✅ All required configuration keys are present -- ✅ Email format validation -- ✅ URL format validation -- ✅ Connection string format validation -- ✅ Security checks (encryption, TrustServerCertificate) -- ✅ Password length validation -- ✅ Scans appsettings.json for accidentally committed secrets - -**Exit codes**: -- `0` - Configuration is valid -- `1` - Configuration has errors - -**When to use**: -- Before running the application -- After changing configuration -- In CI/CD pipelines to validate deployment -- Troubleshooting configuration issues - -## Quick Start - -### First-Time Development Setup - -1. **Run the setup wizard**: - ```bash - cd SPOtoSQL-Net8 - ./scripts/setup-secrets.sh - ``` - -2. **Follow the prompts** to enter: - - SharePoint username (email) - - SharePoint password - - SharePoint site URL - - SQL Server connection details - -3. **Verify configuration**: - ```bash - ./scripts/verify-config.sh - ``` - -4. **Run the application**: - ```bash - cd ConsoleApp1Net8 - dotnet run - ``` - -### Production Deployment - -1. **Set environment variables** (see SECURITY.md for details): - ```bash - export SPO2SQL_SharePoint__Username="user@company.com" - export SPO2SQL_SharePoint__Password="your-password" - export SPO2SQL_SharePoint__SiteUrl="https://company.sharepoint.com/sites/prod" - export SPO2SQL_Sql__ConnectionString="Server=...;Database=...;Encrypt=true;..." - ``` - -2. **Verify configuration**: - ```bash - ./scripts/verify-config.sh --env-vars - ``` - -3. **Deploy and run**: - ```bash - dotnet publish -c Release - cd bin/Release/net8.0/publish - ./ConsoleApp1Net8 - ``` - -## Troubleshooting - -### "dotnet: command not found" - -**Problem**: .NET SDK is not installed or not in PATH. - -**Solution**: Install .NET 8 SDK from https://dotnet.microsoft.com/download - -### "Project file not found" - -**Problem**: Scripts are being run from the wrong directory. - -**Solution**: Always run scripts from the repository root: -```bash -cd /path/to/SPO2SQL -./SPOtoSQL-Net8/scripts/setup-secrets.sh -``` - -Or from the SPOtoSQL-Net8 directory: -```bash -cd /path/to/SPO2SQL/SPOtoSQL-Net8 -./scripts/setup-secrets.sh -``` - -### "No secrets configured" - -**Problem**: User Secrets have not been initialized or set. - -**Solution**: Run the setup script: -```bash -./scripts/setup-secrets.sh -``` - -### Validation warnings - -**Warning**: "Connection string should contain 'Encrypt=true'" - -**Recommendation**: Always use encrypted connections in production. Update your connection string: -```bash -dotnet user-secrets set "Sql:ConnectionString" "Server=myServer;Database=myDB;User Id=user;Password=pwd;Encrypt=true;TrustServerCertificate=false;" -``` - -**Warning**: "Password is less than 8 characters" - -**Recommendation**: Use strong passwords (12+ characters, mixed case, numbers, symbols). - -## Security Notes - -### What Gets Stored Where - -**User Secrets** (Development): -- Location: `~/.microsoft/usersecrets//secrets.json` -- Format: JSON file with key-value pairs -- Permissions: User-readable only -- Automatically excluded from source control - -**Environment Variables** (Production): -- Location: System or user environment variables -- Prefix: `SPO2SQL_` -- Separator: Double underscore `__` for nested sections -- Set via: Shell, systemd service file, Docker, Kubernetes, etc. - -### Best Practices - -1. **Never commit secrets to source control** - - User Secrets are stored outside the project - - appsettings.json should NOT contain passwords - - The verify-config script checks for this - -2. **Use encryption for all connections** - - SQL: `Encrypt=true;TrustServerCertificate=false` - - SharePoint: Always HTTPS - -3. **Rotate credentials regularly** - - Update secrets with setup script or manually - - Test changes before deploying to production - -4. **Use principle of least privilege** - - SharePoint: Read-only access if possible - - SQL: Minimal required permissions - -5. **Audit and monitor** - - Enable application logging - - Review logs for authentication failures - - Monitor for unusual activity - -## Advanced Usage - -### Automated Setup (Non-Interactive) - -For CI/CD pipelines or automated deployments: - -```bash -# Set secrets via dotnet CLI -cd ConsoleApp1Net8 -dotnet user-secrets set "SharePoint:Username" "$SP_USERNAME" -dotnet user-secrets set "SharePoint:Password" "$SP_PASSWORD" -dotnet user-secrets set "SharePoint:SiteUrl" "$SP_SITEURL" -dotnet user-secrets set "Sql:ConnectionString" "$SQL_CONNSTR" - -# Verify -cd .. -./scripts/verify-config.sh -if [ $? -ne 0 ]; then - echo "Configuration validation failed" - exit 1 -fi -``` - -### Integration with CI/CD - -Example GitHub Actions workflow: - -```yaml -- name: Configure Secrets - run: | - cd SPOtoSQL-Net8/ConsoleApp1Net8 - dotnet user-secrets set "SharePoint:Username" "${{ secrets.SP_USERNAME }}" - dotnet user-secrets set "SharePoint:Password" "${{ secrets.SP_PASSWORD }}" - dotnet user-secrets set "SharePoint:SiteUrl" "${{ secrets.SP_SITEURL }}" - dotnet user-secrets set "Sql:ConnectionString" "${{ secrets.SQL_CONNSTR }}" - -- name: Verify Configuration - run: | - cd SPOtoSQL-Net8 - ./scripts/verify-config.sh -``` - -## See Also - -- **[SECURITY.md](../SECURITY.md)**: Comprehensive security documentation -- **[README-MODERNIZATION.md](../README-MODERNIZATION.md)**: Configuration and deployment guide -- **[.NET User Secrets Documentation](https://docs.microsoft.com/en-us/aspnet/core/security/app-secrets)**: Official Microsoft documentation diff --git a/global.json b/global.json new file mode 100644 index 0000000..391ba3c --- /dev/null +++ b/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "8.0.100", + "rollForward": "latestFeature" + } +} diff --git a/src/SPO2SQL.Console/Application.cs b/src/SPO2SQL.Console/Application.cs index 0623805..0b83a19 100644 --- a/src/SPO2SQL.Console/Application.cs +++ b/src/SPO2SQL.Console/Application.cs @@ -1,15 +1,16 @@ +using Microsoft.Data.SqlClient; using SPO2SQL.Configuration; -using SPO2SQL.Models; +using SPO2SQL.Logging; using SPO2SQL.Security; -using Microsoft.Data.SqlClient; namespace SPO2SQL; -public class Application : IHostedService +public sealed class Application : BackgroundService { private readonly ILogger _logger; + private readonly ILoggerFactory _loggerFactory; private readonly IHostApplicationLifetime _lifetime; - private readonly IHttpClientFactory _httpClientFactory; + private readonly SyncExitCode _exitCode; private readonly ApplicationOptions _appOptions; private readonly SharePointOptions _sharePointOptions; @@ -17,137 +18,87 @@ public class Application : IHostedService public Application( ILogger logger, + ILoggerFactory loggerFactory, IHostApplicationLifetime lifetime, - IHttpClientFactory httpClientFactory, + SyncExitCode exitCode, IOptions appOptions, IOptions sharePointOptions, IOptions sqlOptions) { _logger = logger; + _loggerFactory = loggerFactory; _lifetime = lifetime; - _httpClientFactory = httpClientFactory; + _exitCode = exitCode; _appOptions = appOptions.Value; _sharePointOptions = sharePointOptions.Value; _sqlOptions = sqlOptions.Value; } - public Task StartAsync(CancellationToken cancellationToken) + public override Task StartAsync(CancellationToken cancellationToken) { - _logger.LogInformation("═══════════════════════════════════════════════════════════"); - _logger.LogInformation(" {AppName} v{Version}", _appOptions.Name, _appOptions.Version); - _logger.LogInformation("═══════════════════════════════════════════════════════════"); + Logger.Configure(_loggerFactory.CreateLogger("SPO2SQL")); - LogConfigurationSummary(); - - if (cancellationToken.IsCancellationRequested) + if (_logger.IsEnabled(LogLevel.Information)) { - _logger.LogWarning("Application cancelled before startup."); - _lifetime.StopApplication(); - return Task.CompletedTask; + _logger.LogInformation("═══════════════════════════════════════════════════════════"); + _logger.LogInformation(" {AppName} v{Version}", _appOptions.Name, _appOptions.Version); + _logger.LogInformation("═══════════════════════════════════════════════════════════"); } - _ = Task.Run(async () => await ExecuteAsync(cancellationToken), cancellationToken) - .ContinueWith(t => - { - if (t.IsFaulted) - { - _logger.LogError(t.Exception, "Application task failed"); - } - else if (t.IsCanceled) - { - _logger.LogWarning("Application task was cancelled"); - } - - _lifetime.StopApplication(); - }, TaskContinuationOptions.ExecuteSynchronously); - - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) - { - _logger.LogInformation("{AppName} stopping gracefully...", _appOptions.Name); - return Task.CompletedTask; - } - - private void LogConfigurationSummary() - { - _logger.LogInformation("Configuration Summary:"); - _logger.LogInformation(" Environment: {Environment}", _appOptions.Environment); - _logger.LogInformation(" Metrics Enabled: {EnableMetrics}", _appOptions.EnableMetrics); - _logger.LogInformation(" Health Checks Enabled: {EnableHealthChecks}", _appOptions.EnableHealthChecks); - - _logger.LogInformation("SharePoint Configuration:"); - _logger.LogInformation(" Site URL: {SiteUrl}", _sharePointOptions.SiteUrl); - _logger.LogInformation(" Username: {Username}", MaskSensitiveData(_sharePointOptions.Username, 3)); - _logger.LogInformation(" Password: {Password}", "****** (hidden)"); - _logger.LogInformation(" Timeout: {TimeoutSeconds}s", _sharePointOptions.TimeoutSeconds); - _logger.LogInformation(" Max Retries: {MaxRetries}", _sharePointOptions.MaxRetries); - _logger.LogInformation(" Initial Retry Delay: {InitialRetryDelayMs}ms", _sharePointOptions.InitialRetryDelayMs); - - _logger.LogInformation("SQL Server Configuration:"); - _logger.LogInformation(" Connection String: {ConnectionString}", MaskConnectionString(_sqlOptions.ConnectionString)); - _logger.LogInformation(" Command Timeout: {CommandTimeoutSeconds}s", _sqlOptions.CommandTimeoutSeconds); - _logger.LogInformation(" Batch Size: {BatchSize}", _sqlOptions.BatchSize); - _logger.LogInformation(" Enforce Encryption: {EnforceEncryption}", _sqlOptions.EnforceEncryption); - - _logger.LogInformation("═══════════════════════════════════════════════════════════"); + LogConfigurationSummary(); - ValidateConfigurationInvariantsExample(); + return base.StartAsync(cancellationToken); } - private void ValidateConfigurationInvariantsExample() + public override async Task StopAsync(CancellationToken cancellationToken) { - if (_sqlOptions.BatchSize < 10 || _sqlOptions.BatchSize > 1000) + if (_logger.IsEnabled(LogLevel.Information)) { - throw new InvalidOperationException( - "This should never happen! BatchSize validation should have caught this at startup."); + _logger.LogInformation("{AppName} stopping gracefully...", _appOptions.Name); } - - if (string.IsNullOrWhiteSpace(_sharePointOptions.SiteUrl)) - { - throw new InvalidOperationException( - "This should never happen! SiteUrl validation should have caught this at startup."); - } - - _logger.LogDebug("Configuration validation checks passed (as expected)"); + await base.StopAsync(cancellationToken).ConfigureAwait(false); } - private async Task ExecuteAsync(CancellationToken cancellationToken) + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { _logger.LogInformation("Starting SharePoint to SQL synchronization..."); - ConfigureLegacyBridge(); + SyncSettings settings = BuildSyncSettings(); + ConfigureLegacyBridge(settings); if (_appOptions.EnableHealthChecks) { - await PerformHealthChecksAsync(cancellationToken); + await PerformHealthChecksAsync(settings, stoppingToken).ConfigureAwait(false); } - _logger.LogInformation("Sync configuration: DailyMode={DailyMode}, BatchSize={BatchSize}", - _appOptions.DailyMode, _sqlOptions.BatchSize); - - await Task.Run(() => + if (_logger.IsEnabled(LogLevel.Information)) { - SqlServer.RefreshSQLLists.SPOtoSQLUpdate( - daily: _appOptions.DailyMode, - cancellationToken: cancellationToken); - }, cancellationToken); + _logger.LogInformation("Sync configuration: DailyMode={DailyMode}, BatchSize={BatchSize}, Auth={AuthMode}, Metrics={EnableMetrics}", + _appOptions.DailyMode, + settings.BatchSize, + settings.HasAppOnlyAuth ? "app-only" : "username/password", + settings.EnableMetrics); + } + + await RefreshSQLLists.SPOtoSQLUpdateAsync( + settings: settings, + daily: _appOptions.DailyMode, + cancellationToken: stoppingToken).ConfigureAwait(false); _logger.LogInformation("SharePoint to SQL synchronization completed successfully"); } catch (OperationCanceledException) { _logger.LogWarning("SharePoint to SQL synchronization was cancelled."); - Environment.ExitCode = 1; + _exitCode.Fail(); } catch (Exception ex) { _logger.LogError(ex, "Application failed with error: {Message}", ex.Message); - Environment.ExitCode = 1; + _exitCode.Fail(); } finally { @@ -155,16 +106,63 @@ await Task.Run(() => } } - private void ConfigureLegacyBridge() + private void LogConfigurationSummary() + { + if (!_logger.IsEnabled(LogLevel.Information)) + { + return; + } + + _logger.LogInformation("Configuration Summary:"); + _logger.LogInformation(" Environment: {Environment}", _appOptions.Environment); + _logger.LogInformation(" Metrics Enabled: {EnableMetrics}", _appOptions.EnableMetrics); + _logger.LogInformation(" Health Checks Enabled: {EnableHealthChecks}", _appOptions.EnableHealthChecks); + + _logger.LogInformation("SharePoint Configuration:"); + _logger.LogInformation(" Site URL: {SiteUrl}", _sharePointOptions.SiteUrl); + _logger.LogInformation(" Username: {Username}", MaskSensitiveData(_sharePointOptions.Username, 3)); + _logger.LogInformation(" Password: ****** (hidden)"); + _logger.LogInformation(" Timeout: {TimeoutSeconds}s", _sharePointOptions.TimeoutSeconds); + _logger.LogInformation(" Max Retries: {MaxRetries}", _sharePointOptions.MaxRetries); + _logger.LogInformation(" Initial Retry Delay: {InitialRetryDelayMs}ms", _sharePointOptions.InitialRetryDelayMs); + + _logger.LogInformation("SQL Server Configuration:"); + _logger.LogInformation(" Connection String: {ConnectionString}", MaskConnectionString(_sqlOptions.ConnectionString)); + _logger.LogInformation(" Command Timeout: {CommandTimeoutSeconds}s", _sqlOptions.CommandTimeoutSeconds); + _logger.LogInformation(" Batch Size: {BatchSize}", _sqlOptions.BatchSize); + _logger.LogInformation(" Enforce Encryption: {EnforceEncryption}", _sqlOptions.EnforceEncryption); + + _logger.LogInformation("═══════════════════════════════════════════════════════════"); + } + + private SyncSettings BuildSyncSettings() => new() + { + Username = _sharePointOptions.Username, + Password = _sharePointOptions.Password, + SiteUrl = _sharePointOptions.SiteUrl, + SqlConnectionString = _sqlOptions.ConnectionString, + TenantId = _sharePointOptions.TenantId, + ClientId = _sharePointOptions.ClientId, + CertificatePath = _sharePointOptions.CertificatePath, + CertificatePassword = _sharePointOptions.CertificatePassword, + CommandTimeoutSeconds = _sqlOptions.CommandTimeoutSeconds, + BatchSize = _sqlOptions.BatchSize, + MaxRetries = _sharePointOptions.MaxRetries, + InitialRetryDelayMs = _sharePointOptions.InitialRetryDelayMs, + EnableMetrics = _appOptions.EnableMetrics, + LegacyConfigPath = _appOptions.LegacyConfigPath + }; + + private void ConfigureLegacyBridge(SyncSettings settings) { CredentialManager.RegisterFallbackProvider( new StaticCredentialProvider( - _sharePointOptions.Username, - _sharePointOptions.Password, - _sqlOptions.ConnectionString)); - XmlConfig.ConfigurationReader.SetSharePointBaseUrl(_sharePointOptions.SiteUrl); + settings.Username, + settings.Password, + settings.SqlConnectionString)); + XmlConfig.ConfigurationReader.SetSharePointBaseUrl(settings.SiteUrl); - string configPath = _appOptions.LegacyConfigPath ?? ""; + string configPath = settings.LegacyConfigPath ?? ""; if (!Path.IsPathRooted(configPath)) { configPath = Path.Combine(AppContext.BaseDirectory, configPath); @@ -173,7 +171,10 @@ private void ConfigureLegacyBridge() if (File.Exists(configPath)) { XmlConfig.ConfigurationReader.SetConfigPath(configPath); - _logger.LogInformation("Legacy config path set to: {ConfigPath}", configPath); + if (_logger.IsEnabled(LogLevel.Information)) + { + _logger.LogInformation("Legacy config path set to: {ConfigPath}", configPath); + } } else { @@ -181,33 +182,43 @@ private void ConfigureLegacyBridge() } } - private async Task PerformHealthChecksAsync(CancellationToken cancellationToken) + private async Task PerformHealthChecksAsync(SyncSettings settings, CancellationToken cancellationToken) { _logger.LogInformation("Performing startup health checks..."); + ValidateSqlEncryption(); + try { - using var httpClient = _httpClientFactory.CreateClient(); - httpClient.Timeout = TimeSpan.FromSeconds(_sharePointOptions.TimeoutSeconds); + using var spoUser = SPOUserFactory.Create(settings); + var checker = new HealthChecker(); + + var result = await checker.PerformHealthCheckAsync( + spoUser, + settings.SiteUrl, + settings.SqlConnectionString, + cancellationToken).ConfigureAwait(false); - using var response = await httpClient.GetAsync(_sharePointOptions.SiteUrl, cancellationToken); - _logger.LogInformation("SharePoint URL health check: {StatusCode}", response.StatusCode); + foreach (string warning in result.Warnings) + { + _logger.LogWarning("{HealthMessage}", warning); + } + + foreach (string error in result.Errors) + { + _logger.LogError("{HealthMessage}", error); + } + + if (!result.IsHealthy) + { + throw new InvalidOperationException( + "Startup health checks failed: " + string.Join("; ", result.Errors)); + } } catch (OperationCanceledException) { throw; } - catch (Exception ex) - { - _logger.LogWarning(ex, "SharePoint URL health check failed (this may be expected if authentication is required)"); - } - - if (!HasServerKeyword(_sqlOptions.ConnectionString)) - { - _logger.LogWarning("SQL connection string does not contain a server keyword (Server=/Data Source=) - verify configuration"); - } - - ValidateSqlEncryption(); _logger.LogInformation("Health checks completed"); } @@ -232,119 +243,6 @@ private void ValidateSqlEncryption() } } - private void DemonstrateRecordModels() - { - _logger.LogInformation("═══════════════════════════════════════════════════════════"); - _logger.LogInformation(" Record Models Demonstration"); - _logger.LogInformation("═══════════════════════════════════════════════════════════"); - - var listItem = new SharePointListItem - { - Id = 1, - Title = "Q1 Sales Report", - Created = DateTime.UtcNow.AddDays(-30), - Author = "john.doe@contoso.com" - }; - - var updatedItem = listItem with - { - Title = "Q1 Sales Report - Final", - Modified = DateTime.UtcNow - }; - - _logger.LogInformation("SharePoint Item Original: {Title}", listItem.Title); - _logger.LogInformation("SharePoint Item Updated: {Title} (original unchanged: {Unchanged})", - updatedItem.Title, listItem.Title == "Q1 Sales Report"); - - var operation = new SyncOperation( - Guid.NewGuid(), - SyncType.Daily, - DateTime.UtcNow.AddHours(-2), - null, - 0, - SyncStatus.Running - ); - - var (opId, syncType, start, _, _, status) = operation; - _logger.LogInformation("Sync Operation: {Type} started at {Start}, Status: {Status}", - syncType, start, status); - - var completedOp = operation with - { - EndTime = DateTime.UtcNow, - ItemsProcessed = 1250, - Status = SyncStatus.Completed - }; - - _logger.LogInformation("Operation completed: {Items} items in {Duration}", - completedOp.ItemsProcessed, completedOp.Duration); - - var issues = new[] - { - new DataQualityIssue("CustomerList", 101, "Email", "InvalidFormat", "Missing @ symbol", Severity.High), - new DataQualityIssue("OrderList", 202, "Total", "OutOfRange", "Negative total amount", Severity.Critical), - new DataQualityIssue("ProductList", 303, "Description", "TooLong", "Exceeds 500 chars", Severity.Low) - }; - - foreach (var issue in issues) - { - var action = issue switch - { - { Severity: Severity.Critical } => "BLOCK SYNC", - { Severity: Severity.High, IssueType: "InvalidFormat" } => "AUTO-FIX", - { Severity: >= Severity.Medium } => "REVIEW", - _ => "LOG ONLY" - }; - - _logger.LogInformation("DQ Issue: {ErrorMessage} \u2192 Action: {Action}", - issue.ErrorMessage, action); - } - - var stats = new SyncStatistics( - TotalItems: 1500, - Successful: 1450, - Failed: 50, - Duration: TimeSpan.FromMinutes(15) - ); - - _logger.LogInformation("Sync Statistics:"); - _logger.LogInformation(" {Summary}", stats.Summary); - _logger.LogInformation(" Success Rate: {SuccessRate:F2}%", stats.SuccessRate); - _logger.LogInformation(" Throughput: {Throughput:F2} items/sec", stats.Throughput); - _logger.LogInformation(" Avg Time/Item: {AvgTime:F2}ms", stats.AverageTimePerItem.TotalMilliseconds); - _logger.LogInformation(" Meets Quality Standards: {MeetsStandards}", stats.MeetsQualityStandards); - - var conn1 = new ConnectionInfo( - "portal.contoso.com", - "SharePointDB", - "Production", - DateTime.UtcNow.AddHours(-6) - ); - - var conn2 = new ConnectionInfo( - "portal.contoso.com", - "SharePointDB", - "Production", - conn1.LastSync - ); - - var conn3 = conn1 with { Environment = "Development" }; - - _logger.LogInformation("Connection Info:"); - _logger.LogInformation(" conn1.Description: {Description}", conn1.Description); - _logger.LogInformation(" conn1 == conn2 (value equality): {Equal}", conn1 == conn2); - _logger.LogInformation(" ReferenceEquals(conn1, conn2): {RefEqual}", ReferenceEquals(conn1, conn2)); - _logger.LogInformation(" conn1 == conn3 (different env): {Equal}", conn1 == conn3); - _logger.LogInformation(" Time Since Last Sync: {TimeSince:F1} hours", - conn1.TimeSinceLastSync?.TotalHours ?? 0); - - var connections = new HashSet { conn1, conn2, conn3 }; - _logger.LogInformation(" HashSet with conn1, conn2, conn3: {Count} unique (conn1==conn2)", - connections.Count); - - _logger.LogInformation("═══════════════════════════════════════════════════════════"); - } - private static string MaskSensitiveData(string value, int visibleChars = 3) { if (string.IsNullOrWhiteSpace(value)) @@ -360,20 +258,6 @@ private static string MaskSensitiveData(string value, int visibleChars = 3) return value[..visibleChars] + new string('*', value.Length - visibleChars); } - private static bool HasServerKeyword(string connectionString) - { - if (string.IsNullOrWhiteSpace(connectionString)) - { - return false; - } - - var match = System.Text.RegularExpressions.Regex.Match( - connectionString, - @"(?:Server|Data\s*Source|Address|Addr|Network\s*Address)\s*=", - System.Text.RegularExpressions.RegexOptions.IgnoreCase); - return match.Success; - } - private static string MaskConnectionString(string connectionString) { if (string.IsNullOrWhiteSpace(connectionString)) diff --git a/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs b/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs index e46e4d8..f9664ba 100644 --- a/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs +++ b/src/SPO2SQL.Console/Configuration/ApplicationOptions.cs @@ -24,5 +24,5 @@ public record ApplicationOptions public bool DailyMode { get; init; } = true; [Required] - public string LegacyConfigPath { get; init; } = "XmlConfig\\UserConfig.xml"; + public string LegacyConfigPath { get; init; } = "XmlConfig/UserConfig.xml"; } diff --git a/src/SPO2SQL.Console/Configuration/SharePointOptions.cs b/src/SPO2SQL.Console/Configuration/SharePointOptions.cs index 4dd5ed4..aeb9b67 100644 --- a/src/SPO2SQL.Console/Configuration/SharePointOptions.cs +++ b/src/SPO2SQL.Console/Configuration/SharePointOptions.cs @@ -2,16 +2,12 @@ namespace SPO2SQL.Configuration; -public record SharePointOptions +public record SharePointOptions : IValidatableObject { public const string SectionName = "SharePoint"; - [Required(ErrorMessage = "SharePoint username is required")] - [EmailAddress(ErrorMessage = "SharePoint username must be a valid email address")] public string Username { get; init; } = string.Empty; - [Required(ErrorMessage = "SharePoint password is required")] - [MinLength(1, ErrorMessage = "SharePoint password cannot be empty")] public string Password { get; init; } = string.Empty; [Required(ErrorMessage = "SharePoint site URL is required")] @@ -21,9 +17,54 @@ public record SharePointOptions [Range(10, 600, ErrorMessage = "Timeout must be between 10 and 600 seconds")] public int TimeoutSeconds { get; init; } = 120; - [Range(0, 10, ErrorMessage = "Max retries must be between 0 and 10")] + [Range(1, 10, ErrorMessage = "Max retries must be between 1 and 10")] public int MaxRetries { get; init; } = 3; [Range(100, 10000, ErrorMessage = "Initial retry delay must be between 100 and 10000 ms")] public int InitialRetryDelayMs { get; init; } = 1000; + + // App-only authentication. When configured, these take precedence over + // username/password and work with MFA and Conditional Access policies. + public string? TenantId { get; init; } + + public string? ClientId { get; init; } + + public string? CertificatePath { get; init; } + + public string? CertificatePassword { get; init; } + + public bool HasAppOnlyAuth => + !string.IsNullOrWhiteSpace(TenantId) && + !string.IsNullOrWhiteSpace(ClientId) && + !string.IsNullOrWhiteSpace(CertificatePath); + + public IEnumerable Validate(ValidationContext validationContext) + { + bool hasUserPassword = + !string.IsNullOrWhiteSpace(Username) && + !string.IsNullOrWhiteSpace(Password); + + if (!string.IsNullOrWhiteSpace(Username) && + !new EmailAddressAttribute().IsValid(Username)) + { + yield return new ValidationResult( + "SharePoint username must be a valid email address.", + [nameof(Username)]); + } + + if (!hasUserPassword && !HasAppOnlyAuth) + { + yield return new ValidationResult( + "Configure either Username/Password or app-only authentication " + + "(TenantId, ClientId and CertificatePath).", + [nameof(Username), nameof(TenantId), nameof(ClientId), nameof(CertificatePath)]); + } + + if (HasAppOnlyAuth && !File.Exists(CertificatePath)) + { + yield return new ValidationResult( + $"The configured SharePoint certificate was not found: '{CertificatePath}'.", + [nameof(CertificatePath)]); + } + } } diff --git a/src/SPO2SQL.Console/CredentialSetup.cs b/src/SPO2SQL.Console/CredentialSetup.cs new file mode 100644 index 0000000..b5588b9 --- /dev/null +++ b/src/SPO2SQL.Console/CredentialSetup.cs @@ -0,0 +1,57 @@ +using SPO2SQL.Security; + +namespace SPO2SQL; + +/// +/// Implements the --setup-secrets command in the real console host. +/// +internal static class CredentialSetup +{ + public static int Run() + { + Console.WriteLine("=== Secure Credential Setup ==="); + Console.WriteLine("Credentials will be encrypted and stored in the current user's profile."); + Console.WriteLine(); + + string? username = Read("SharePoint Username: "); + string? password = Read("SharePoint Password: "); + string? connectionString = Read("SQL Connection String: "); + + if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password)) + { + Console.WriteLine("ERROR: Username and password are required."); + return 1; + } + + try + { + if (!OperatingSystem.IsWindows()) + { + Console.WriteLine("ERROR: DPAPI encryption is not available on this platform."); + Console.WriteLine("Use environment variables or user secrets instead."); + return 1; + } + + DpapiCredentialProvider.SaveCredentials(username, password, connectionString ?? string.Empty); + Console.WriteLine("Credentials saved securely."); + return 0; + } + catch (PlatformNotSupportedException) + { + Console.WriteLine("ERROR: DPAPI encryption is not available on this platform."); + Console.WriteLine("Use environment variables or user secrets instead."); + return 1; + } + catch (Exception ex) + { + Console.WriteLine($"ERROR: Failed to save credentials: {ex.Message}"); + return 1; + } + } + + private static string? Read(string prompt) + { + Console.Write(prompt); + return Console.ReadLine()?.Trim(); + } +} diff --git a/src/SPO2SQL.Console/Program.cs b/src/SPO2SQL.Console/Program.cs index a9eeea4..7ee0c03 100644 --- a/src/SPO2SQL.Console/Program.cs +++ b/src/SPO2SQL.Console/Program.cs @@ -7,14 +7,22 @@ internal sealed class Program { private static async Task Main(string[] args) { + if (args.Any(a => string.Equals(a, "--setup-secrets", StringComparison.OrdinalIgnoreCase))) + { + return CredentialSetup.Run(); + } + try { var builder = CreateHostBuilder(args); - using var host = builder.Build(); + var host = builder.Build(); + + // RunAsync disposes the host when it completes, so resolve the outcome first. + var exitCode = host.Services.GetRequiredService(); await host.RunAsync(); - return 0; + return exitCode.Value; } catch (OptionsValidationException ex) { @@ -81,7 +89,7 @@ private static IHostBuilder CreateHostBuilder(string[] args) => .ValidateDataAnnotations() .ValidateOnStart(); - services.AddHttpClient(); + services.AddSingleton(); services.AddHostedService(); }); } diff --git a/src/SPO2SQL.Console/SPO2SQL.Console.csproj b/src/SPO2SQL.Console/SPO2SQL.Console.csproj index fb394ae..b5efcb4 100644 --- a/src/SPO2SQL.Console/SPO2SQL.Console.csproj +++ b/src/SPO2SQL.Console/SPO2SQL.Console.csproj @@ -9,8 +9,6 @@ - - @@ -24,11 +22,7 @@ - - - - - + diff --git a/src/SPO2SQL.Console/SyncExitCode.cs b/src/SPO2SQL.Console/SyncExitCode.cs new file mode 100644 index 0000000..cd934c8 --- /dev/null +++ b/src/SPO2SQL.Console/SyncExitCode.cs @@ -0,0 +1,15 @@ +namespace SPO2SQL; + +/// +/// Carries the synchronization outcome from the hosted service to the process +/// entry point. cannot be used here because +/// Main returns an explicit integer, which always wins. +/// +public sealed class SyncExitCode +{ + private int _value; + + public int Value => Volatile.Read(ref _value); + + public void Fail() => Interlocked.CompareExchange(ref _value, 1, 0); +} diff --git a/src/SPO2SQL.Console/Utilities/OperationResult.cs b/src/SPO2SQL.Console/Utilities/OperationResult.cs index 951be0e..b0ff6be 100644 --- a/src/SPO2SQL.Console/Utilities/OperationResult.cs +++ b/src/SPO2SQL.Console/Utilities/OperationResult.cs @@ -94,9 +94,9 @@ public static string CategorizeError(this OperationResult result) => { OperationResult.SuccessResult => "Success", OperationResult.FailureResult { ErrorCode: "NotFound" } => "Resource Not Found", - OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Auth") == true => + OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Auth", StringComparison.Ordinal) == true => "Authentication/Authorization Error", - OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Validation") == true => + OperationResult.FailureResult { ErrorCode: var code } when code?.StartsWith("Validation", StringComparison.Ordinal) == true => "Validation Error", OperationResult.FailureResult { Exception: not null } => "Exception Occurred", OperationResult.FailureResult => "General Error", diff --git a/src/SPO2SQL.Console/appsettings.json b/src/SPO2SQL.Console/appsettings.json index adbc511..693f602 100644 --- a/src/SPO2SQL.Console/appsettings.json +++ b/src/SPO2SQL.Console/appsettings.json @@ -6,7 +6,7 @@ "EnableMetrics": true, "EnableHealthChecks": true, "Environment": "Production", - "LegacyConfigPath": "XmlConfig\\UserConfig.xml" + "LegacyConfigPath": "XmlConfig/UserConfig.xml" }, "SharePoint": { "Username": "", @@ -14,7 +14,11 @@ "SiteUrl": "https://bringglobal.sharepoint.com", "TimeoutSeconds": 120, "MaxRetries": 3, - "InitialRetryDelayMs": 1000 + "InitialRetryDelayMs": 1000, + "TenantId": "", + "ClientId": "", + "CertificatePath": "", + "CertificatePassword": "" }, "Sql": { "ConnectionString": "", diff --git a/src/SPO2SQL.Console/secrets.json.template b/src/SPO2SQL.Console/secrets.json.template index 112606d..f841e26 100644 --- a/src/SPO2SQL.Console/secrets.json.template +++ b/src/SPO2SQL.Console/secrets.json.template @@ -6,7 +6,7 @@ "EnableMetrics": true, "EnableHealthChecks": true, "Environment": "Development", - "LegacyConfigPath": "XmlConfig\\UserConfig.xml" + "LegacyConfigPath": "XmlConfig/UserConfig.xml" }, "SharePoint": { "Username": "your-email@company.com", @@ -14,7 +14,11 @@ "SiteUrl": "https://yourtenant.sharepoint.com", "TimeoutSeconds": 120, "MaxRetries": 3, - "InitialRetryDelayMs": 1000 + "InitialRetryDelayMs": 1000, + "TenantId": "", + "ClientId": "", + "CertificatePath": "", + "CertificatePassword": "" }, "Sql": { "ConnectionString": "Server=localhost;Database=SPO2SQL;Integrated Security=true;TrustServerCertificate=true;", diff --git a/src/SPO2SQL.Core/Configuration/SyncSettings.cs b/src/SPO2SQL.Core/Configuration/SyncSettings.cs new file mode 100644 index 0000000..aa61a1a --- /dev/null +++ b/src/SPO2SQL.Core/Configuration/SyncSettings.cs @@ -0,0 +1,42 @@ +namespace SPO2SQL.Configuration; + +/// +/// Immutable snapshot of the settings needed by the synchronization layer. +/// The console host maps its validated options into this record once, at startup, +/// so the core never has to reach back into global configuration state. +/// +public sealed record SyncSettings +{ + public string Username { get; init; } = string.Empty; + + public string Password { get; init; } = string.Empty; + + public required string SiteUrl { get; init; } + + public required string SqlConnectionString { get; init; } + + public string? TenantId { get; init; } + + public string? ClientId { get; init; } + + public string? CertificatePath { get; init; } + + public string? CertificatePassword { get; init; } + + public int CommandTimeoutSeconds { get; init; } = 300; + + public int BatchSize { get; init; } = 80; + + public int MaxRetries { get; init; } = 3; + + public int InitialRetryDelayMs { get; init; } = 1000; + + public bool EnableMetrics { get; init; } + + public string? LegacyConfigPath { get; init; } + + public bool HasAppOnlyAuth => + !string.IsNullOrWhiteSpace(TenantId) && + !string.IsNullOrWhiteSpace(ClientId) && + !string.IsNullOrWhiteSpace(CertificatePath); +} diff --git a/src/SPO2SQL.Core/Logging/Logger.cs b/src/SPO2SQL.Core/Logging/Logger.cs index 30157ce..dabc3e8 100644 --- a/src/SPO2SQL.Core/Logging/Logger.cs +++ b/src/SPO2SQL.Core/Logging/Logger.cs @@ -1,57 +1,110 @@ -using System; +using Microsoft.Extensions.Logging; namespace SPO2SQL.Logging; +/// +/// Bridge between the legacy static logging calls used across the core library and +/// the host's . The host must call at +/// startup; until then messages are written to the console so failures are never +/// silently discarded. +/// public static class Logger { - private const string TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.fff"; - private enum LogLevel { ERROR = 1, WARNING = 2, DEBUG = 3 } + private static readonly object Sync = new(); - public static int VerboseLevel { get; set; } = 0; + private static ILogger? _sink; + private static int _verboseLevel = 2; - public static void Log(int level, string message) + /// + /// Legacy verbosity gate kept for source compatibility. 1 = errors only, + /// 2 = warnings and above, 3 = all messages. + /// + public static int VerboseLevel { - if (level < 1 || level > 3) - { - return; - } + get => Volatile.Read(ref _verboseLevel); + set => Volatile.Write(ref _verboseLevel, value); + } - if (VerboseLevel < level) - { - return; - } + /// + /// Connects the static facade to the application's logging pipeline. + /// + public static void Configure(ILogger sink) + { + ArgumentNullException.ThrowIfNull(sink); - string timestamp = DateTime.Now.ToString(TIMESTAMP_FORMAT); - string levelName = GetLevelName(level); - string formattedMessage = $"[{timestamp}] [{levelName}] {message}"; + Volatile.Write(ref _sink, sink); + VerboseLevel = sink.IsEnabled(LogLevel.Debug) ? 3 : 2; + } - Console.WriteLine(formattedMessage); + internal static void Reset() + { + Volatile.Write(ref _sink, null); + _verboseLevel = 2; } - public static void LogError(string message, Exception ex = null) + /// + /// Logs a message using the legacy numeric levels: 1 = information, + /// 2 = warning, 3 = debug. + /// + public static void Log(int level, string message) { - Log(1, message); - if (ex != null && VerboseLevel >= 3) + switch (level) { - Log(3, $"Exception Details: {ex.GetType().Name}: {ex.Message}"); + case 1: + Write(LogLevel.Information, 1, message, null); + break; + case 2: + Write(LogLevel.Warning, 2, message, null); + break; + case 3: + Write(LogLevel.Debug, 3, message, null); + break; } } - public static void LogWarning(string message) - { - Log(2, message); - } + public static void LogInformation(string message) => Write(LogLevel.Information, 1, message, null); - public static void LogDebug(string message) - { - Log(3, message); - } + public static void LogWarning(string message) => Write(LogLevel.Warning, 2, message, null); - private static string GetLevelName(int level) => level switch + public static void LogDebug(string message) => Write(LogLevel.Debug, 3, message, null); + + public static void LogError(string message, Exception? ex = null) => Write(LogLevel.Error, 1, message, ex); + + private static void Write(LogLevel severity, int legacyLevel, string message, Exception? exception) { - 1 => "ERROR", - 2 => "WARN", - 3 => "DEBUG", - _ => "INFO" - }; + if (legacyLevel > VerboseLevel) + { + return; + } + + ILogger? sink = Volatile.Read(ref _sink); + if (sink != null) + { + if (!sink.IsEnabled(severity)) + { + return; + } + + sink.Log(severity, new EventId(0, "Legacy"), message, exception, static (state, _) => state); + return; + } + + string levelName = severity switch + { + LogLevel.Error or LogLevel.Critical => "ERROR", + LogLevel.Warning => "WARN", + LogLevel.Debug or LogLevel.Trace => "DEBUG", + _ => "INFO" + }; + + string line = $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] [{levelName}] {message}"; + lock (Sync) + { + Console.WriteLine(line); + if (exception != null) + { + Console.WriteLine(exception); + } + } + } } diff --git a/src/SPO2SQL.Core/SPO2SQL.Core.csproj b/src/SPO2SQL.Core/SPO2SQL.Core.csproj index 66c8c0a..5bf826d 100644 --- a/src/SPO2SQL.Core/SPO2SQL.Core.csproj +++ b/src/SPO2SQL.Core/SPO2SQL.Core.csproj @@ -7,10 +7,10 @@ + - @@ -18,5 +18,8 @@ <_Parameter1>SPO2SQL.Console + + <_Parameter1>SPO2SQL.Core.Tests + diff --git a/src/SPO2SQL.Core/SPODataQuality/RefreshSPOLists.cs b/src/SPO2SQL.Core/SPODataQuality/RefreshSPOLists.cs deleted file mode 100644 index df4725e..0000000 --- a/src/SPO2SQL.Core/SPODataQuality/RefreshSPOLists.cs +++ /dev/null @@ -1,616 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Linq.Expressions; -using System.Threading; -using Microsoft.Data.SqlClient; -using Microsoft.SharePoint.Client; -using SPO2SQL.Logging; -using SPO2SQL.Security; -using SPO2SQL.SharePoint; -using SPO2SQL.SqlServer; -using SPO2SQL.XmlConfig; - -namespace SPO2SQL.SPODataQuality; - -internal sealed class RefreshSPOLists -{ - private static void Main(string[] args) - { - try - { - InitializeApplication(args); - - if (args.Any(a => a.Equals("--setup-secrets", StringComparison.OrdinalIgnoreCase))) - { - RunSetupSecrets(); - return; - } - - RunMainWorkflow(); - } - catch (Exception ex) - { - HandleFatalError(ex); - } - } - - private static void RunSetupSecrets() - { - Console.WriteLine("=== Secure Credential Setup ==="); - Console.WriteLine("Credentials will be encrypted and stored at:"); - Console.WriteLine(" %APPDATA%\\SPO2SQL\\credentials.enc"); - Console.WriteLine(); - - Console.Write("SharePoint Username: "); - string username = Console.ReadLine()?.Trim(); - Console.Write("SharePoint Password: "); - string password = Console.ReadLine()?.Trim(); - Console.Write("SQL Connection String: "); - string connString = Console.ReadLine()?.Trim(); - - if (string.IsNullOrEmpty(username) || string.IsNullOrEmpty(password)) - { - Console.WriteLine("ERROR: Username and password are required."); - Environment.Exit(1); - } - - try - { - DpapiCredentialProvider.SaveCredentials(username, password, connString ?? ""); - Console.WriteLine("Credentials saved securely."); - } - catch (PlatformNotSupportedException) - { - Console.WriteLine("ERROR: DPAPI encryption is not available on this platform."); - Console.WriteLine("Use environment variables instead:"); - Console.WriteLine(" set SPO_USERNAME=" + username); - Console.WriteLine(" set SPO_PASSWORD=..."); - Console.WriteLine(" set SQL_CONNECTION_STRING=..."); - Environment.Exit(1); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to save credentials: " + ex.Message); - Environment.Exit(1); - } - } - - private static void InitializeApplication(string[] args) - { - Logger.Log(1, "DEBUG: Using the Default config"); - string configPath = "XmlConfig\\UserConfig.xml"; - - int verbose = 0; - bool diagnostic = false; - - foreach (var arg in args) - { - if (arg.StartsWith("--config=")) - { - configPath = arg.Substring("--config=".Length); - } - else if (arg.StartsWith("--verbose=")) - { - if (int.TryParse(arg.Substring("--verbose=".Length), out int v) && v >= 0 && v <= 3) - { - verbose = v; - } - } - else if (arg.ToLower() == "diagnostic") - { - diagnostic = true; - } - } - - if (diagnostic && verbose == 0) - { - verbose = 1; - } - - Logger.VerboseLevel = verbose; - - ConfigurationReader.SetConfigPath(configPath); - - Logger.Log(1, "DEBUG: Application initialized"); - Logger.Log(2, "CURRENT TIME: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff")); - } - - private static void RunMainWorkflow() - { - try - { - TestSQLConnection(); - Logger.Log(1, "DEBUG: SQL connection test completed"); - - var credentials = ConfigurationReader.GetSharePointCredentials(); - if (string.IsNullOrEmpty(credentials.Username) || string.IsNullOrEmpty(credentials.Password)) - { - throw new InvalidOperationException("Failed to retrieve SharePoint credentials from configuration"); - } - - if (!TwoFactorAuth.PerformVerification()) - { - Console.WriteLine("2FA verification failed. Exiting."); - Environment.Exit(1); - } - - using (var spoUser = new SPOUser(credentials.Username, credentials.Password)) - { - Logger.Log(1, "DEBUG: SPOUser created"); - - Logger.Log(3, "DEBUG: SPOList objects configured"); - - ProcessCommandLineArguments(); - - Logger.Log(2, "End of requests."); - Logger.Log(2, ""); - } - } - catch (Exception ex) - { - throw new ApplicationException("Error in main workflow", ex); - } - } - - private static void ProcessCommandLineArguments() - { - Logger.Log(1, "DEBUG: Processing command-line arguments"); - Logger.Log(1, "DEBUG: Command-line arguments processed"); - } - - private static void HandleFatalError(Exception ex) - { - Console.WriteLine("FATAL ERROR: An unexpected error occurred."); - Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}"); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - Environment.Exit(1); - } - - private static void TestSQLConnection() - { - Logger.Log(2, "Testing SQL Server connection..."); - try - { - string connectionString = ConfigurationReader.GetSqlConnectionString(); - if (string.IsNullOrWhiteSpace(connectionString)) - { - throw new InvalidOperationException("SQL connection string is not configured"); - } - - Logger.Log(2, "Attempting to connect to SQL server..."); - - using (var connection = new SqlConnection(connectionString)) - { - try - { - connection.Open(); - } - catch (SqlException ex) when (IsNetworkRelatedSqlError(ex.Number)) - { - Console.WriteLine("ERROR: Unable to connect to the SQL Server."); - Console.WriteLine("Please check if your VPN connection is active and try again."); - Console.WriteLine("Technical details: " + ex.Message); - Console.WriteLine("Press any key to exit..."); - if (Console.KeyAvailable || !Console.IsInputRedirected) - { - try - { - Console.ReadKey(true); - } - catch (InvalidOperationException) - { - } - } - else - { - Thread.Sleep(2000); - } - Environment.Exit(1); - } - catch (SqlException ex) - { - Console.WriteLine($"SQL error: {ex.Number} - {ex.Message}"); - throw; - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Unexpected error while opening SQL connection."); - Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}"); - throw; - } - - Logger.Log(2, $"Server: {connection.DataSource}"); - Logger.Log(2, $"Database: {connection.Database}"); - Logger.Log(2, "SQL connection established successfully!"); - - using (var command = connection.CreateCommand()) - { - try - { - command.CommandText = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES"; - object result = command.ExecuteScalar(); - if (result != null) - { - int tableCount = Convert.ToInt32(result); - Logger.Log(2, $"Number of tables in database: {tableCount}"); - } - else - { - Logger.Log(2, "Number of tables in database: 0 (null result)"); - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to execute SELECT COUNT(*) on INFORMATION_SCHEMA.TABLES."); - Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}"); - } - - try - { - command.CommandText = "CREATE TABLE #TempTest (ID int); DROP TABLE #TempTest;"; - command.ExecuteNonQuery(); - Logger.Log(2, "CREATE TABLE permission: OK"); - } - catch (SqlException ex) - { - Logger.Log(2, $"Warning: No CREATE TABLE permission: {ex.Number} - {ex.Message}"); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Unexpected error during CREATE TABLE permission test."); - Console.WriteLine($"Exception: {ex.GetType().Name}: {ex.Message}"); - } - } - } - } - catch (SqlException ex) - { - Console.WriteLine($"SQL connection error: {ex.Number} - {ex.Message}"); - Console.WriteLine("Error details:"); - Console.WriteLine($" Number: {ex.Number}"); - Console.WriteLine($" State: {ex.State}"); - if (!string.IsNullOrEmpty(ex.Procedure)) - { - Console.WriteLine($" Procedure: {ex.Procedure}"); - } - Console.WriteLine("The execution is stopping."); - throw; - } - catch (InvalidOperationException ex) - { - Console.WriteLine($"Configuration error: {ex.Message}"); - throw; - } - catch (Exception ex) - { - Console.WriteLine($"General error testing SQL connection: {ex.GetType().Name}: {ex.Message}"); - throw; - } - } - - private static bool IsNetworkRelatedSqlError(int errorNumber) - { - switch (errorNumber) - { - case -2: - case -1: - case 0: - case 53: - case 64: - case 87: - case 10060: - case 10061: - case 11001: - return true; - default: - return false; - } - } - - public static void GetAllLists() - { - Logger.Log(1, "DEBUG: Entering GetAllLists"); - try - { - var (username, password) = ConfigurationReader.GetSharePointCredentials(); - using (SPOUser spoUser = new SPOUser(username, password)) - { - Context context = new Context() - { - Site = "seed", - SPOUser = spoUser - }; - foreach (Microsoft.SharePoint.Client.List allList in context.GetAllLists()) - { - try - { - Logger.Log(1, "DEBUG: Loading list - " + allList.Title); - context.Ctx.Load(allList, l => l.IsSystemList); - context.Ctx.ExecuteQuery(); - Logger.Log(2, "List Name: " + allList.Title + "; is: " + allList.IsSystemList.ToString()); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to load or display list '" + allList.Title + "'."); - Console.WriteLine("Exception: " + ex.Message); - } - } - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Exception in GetAllLists."); - Console.WriteLine("Exception: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - } - } - - private static void SPODebug(string listName, string ctxURL, SPOUser user) - { - Logger.Log(1, "DEBUG: Entering SPODebug"); - try - { - SPOList spoList = new SPOList - { - Name = listName, - Site = ctxURL, - SPOUser = user, - CAMLQuery = "1" - }; - - Logger.Log(1, "DEBUG: Executing Build"); - try - { - spoList.Build(); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to build SPOList."); - Console.WriteLine("Exception: " + ex.Message); - return; - } - - Logger.Log(1, "DEBUG: Executing PrintItemProperties"); - try - { - if (spoList.ItemCollection != null && spoList.ItemCollection.Count > 0) - { - spoList.PrintItemProperties(spoList.ItemCollection[0]); - } - else - { - Logger.Log(2, "No items found in the list."); - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to print properties of the first item."); - Console.WriteLine("Exception: " + ex.Message); - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Exception in SPODebug."); - Console.WriteLine("Exception: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - } - } - - private static void RefreshListsSPO(SPOList sourceList, SPOList destList) - { - try - { - Logger.Log(1, "DEBUG: Starting RefreshListsSPO"); - - try - { - sourceList.Build(); - Logger.Log(1, "DEBUG: sourceList.Build completed"); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to build source SPOList."); - Console.WriteLine("Exception: " + ex.Message); - return; - } - - try - { - destList.Build(); - Logger.Log(1, "DEBUG: destList.Build completed"); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to build destination SPOList."); - Console.WriteLine("Exception: " + ex.Message); - return; - } - - int num1 = 0; - int num2 = 0; - - string[,] actualFields; - try - { - actualFields = GetActualFields(sourceList, destList); - Logger.Log(1, "DEBUG: Fields obtained"); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to get actual fields mapping."); - Console.WriteLine("Exception: " + ex.Message); - return; - } - - try - { - if ((uint)sourceList.ItemCollection.Count > 0U) - { - num1 = (int)sourceList.ItemCollection[sourceList.ItemCollection.Count - 1]["ID"]; - } - - if ((uint)destList.ItemCollection.Count > 0U) - { - num2 = (int)destList.ItemCollection[destList.ItemCollection.Count - 1]["ID"]; - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to determine last IDs in source/destination lists."); - Console.WriteLine("Exception: " + ex.Message); - } - - if (num2 < num1) - { - Logger.Log(1, "DEBUG: Adding new items"); - try - { - do - { - destList.AddItem(); - ++num2; - } - while (num2 < num1); - - Logger.Log(2, "Adding new items..."); - destList.Update(); - Logger.Log(2, "Done adding items."); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to add or update new items in destination list."); - Console.WriteLine("Exception: " + ex.Message); - } - } - - try - { - for (int index1 = 0; index1 < sourceList.ItemCollection.Count; ++index1) - { - int id = (int)sourceList.ItemCollection[index1]["ID"]; - for (int index2 = 0; index2 < actualFields.Length / 2; ++index2) - { - try - { - destList.ItemCollection.GetById(id)[actualFields[index2, 0]] = sourceList.ItemCollection[index1][actualFields[index2, 1]]; - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to copy field '" + actualFields[index2, 1] + "' to '" + actualFields[index2, 0] + "' for item ID " + id + "."); - Console.WriteLine("Exception: " + ex.Message); - } - } - try - { - destList.ItemCollection.GetById(id).Update(); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to update item ID " + id + " in destination list."); - Console.WriteLine("Exception: " + ex.Message); - } - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Exception during item synchronization."); - Console.WriteLine("Exception: " + ex.Message); - } - - try - { - destList.Ctx.ExecuteQuery(); - Logger.Log(2, sourceList.Site + " " + sourceList.Name + " -> " + destList.Site + " " + destList.Name + ": Done!"); - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Failed to execute query on destination list context."); - Console.WriteLine("Exception: " + ex.Message); - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR in RefreshListsSPO: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - } - } - - private static string[,] GetActualFields(SPOList listone, SPOList listtwo) - { - Logger.Log(1, "DEBUG: Entering GetActualFields"); - try - { - List fields1 = GetFields(listone); - List fields2 = GetFields(listtwo); - - string[,] strArray = new string[fields1.Count, 2]; - for (int i = 0; i < fields1.Count; i++) - { - strArray[i, 0] = ""; - strArray[i, 1] = ""; - } - - int index1 = 0; - int index2 = 0; - - foreach (Field field1 in fields1) - { - bool found = false; - while (index2 < fields2.Count) - { - Field field2 = fields2[index2]; - if (field1.Title == field2.Title) - { - strArray[index1, 0] = field2.InternalName; - strArray[index1, 1] = field1.InternalName; - found = true; - Logger.Log(1, "DEBUG: Match found - " + field1.Title); - } - ++index2; - if (found) - { - break; - } - } - - ++index1; - index2 = 0; - } - - return strArray; - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Exception in GetActualFields."); - Console.WriteLine("Exception: " + ex.Message); - throw; - } - } - - private static List GetFields(SPOList list) - { - Logger.Log(1, "DEBUG: Entering GetFields"); - List fieldList = new List(); - try - { - foreach (Field field in (ClientObjectCollection)list.Fields) - { - if (!field.FromBaseType || field.InternalName == "Title") - { - fieldList.Add(field); - Logger.Log(1, "DEBUG: Field added - " + field.Title); - } - } - } - catch (Exception ex) - { - Console.WriteLine("ERROR: Exception in GetFields."); - Console.WriteLine("Exception: " + ex.Message); - throw; - } - - return fieldList; - } -} diff --git a/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs b/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs index 8832b67..54faadb 100644 --- a/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs +++ b/src/SPO2SQL.Core/Security/DpapiCredentialProvider.cs @@ -1,11 +1,14 @@ -using System; -using System.IO; +using System.Runtime.Versioning; using System.Security.Cryptography; using System.Text; using System.Text.Json; namespace SPO2SQL.Security; +/// +/// Windows DPAPI credential store. All entry points fail fast on other platforms so +/// callers can fall back to environment variables or user secrets. +/// public class DpapiCredentialProvider : ICredentialProvider { private sealed record StoredCredentials(string Username, string Password, string SqlConnectionString); @@ -17,13 +20,13 @@ private sealed record StoredCredentials(string Username, string Password, string public (string Username, string Password, string SqlConnectionString)? GetCredentials() { - try + if (!OperatingSystem.IsWindows() || !File.Exists(FilePath)) { - if (!File.Exists(FilePath)) - { - return null; - } + return null; + } + try + { byte[] encrypted = File.ReadAllBytes(FilePath); byte[] decrypted = ProtectedData.Unprotect(encrypted, null, DataProtectionScope.CurrentUser); string plaintext = Encoding.UTF8.GetString(decrypted); @@ -39,15 +42,22 @@ private sealed record StoredCredentials(string Username, string Password, string } } + [SupportedOSPlatform("windows")] public static void SaveCredentials(string username, string password, string connectionString) { - string dir = Path.GetDirectoryName(FilePath); - Directory.CreateDirectory(dir); + if (!OperatingSystem.IsWindows()) + { + throw new PlatformNotSupportedException("DPAPI credential storage is only available on Windows."); + } ArgumentException.ThrowIfNullOrWhiteSpace(username); ArgumentNullException.ThrowIfNull(password); ArgumentNullException.ThrowIfNull(connectionString); + string dir = Path.GetDirectoryName(FilePath) + ?? throw new InvalidOperationException("Could not determine the credential storage directory."); + Directory.CreateDirectory(dir); + string plaintext = JsonSerializer.Serialize( new StoredCredentials(username, password, connectionString)); byte[] data = Encoding.UTF8.GetBytes(plaintext); diff --git a/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs b/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs index d97179e..6cbe44d 100644 --- a/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs +++ b/src/SPO2SQL.Core/Security/EnvCredentialProvider.cs @@ -1,20 +1,15 @@ -using System; - namespace SPO2SQL.Security; public class EnvCredentialProvider : ICredentialProvider { public (string Username, string Password, string SqlConnectionString)? GetCredentials() { - string user = Environment.GetEnvironmentVariable("SPO_USERNAME"); - string pass = Environment.GetEnvironmentVariable("SPO_PASSWORD"); - string conn = Environment.GetEnvironmentVariable("SQL_CONNECTION_STRING"); - - if (!string.IsNullOrEmpty(user) && !string.IsNullOrEmpty(pass)) - { - return (user, pass, conn ?? string.Empty); - } + string user = Environment.GetEnvironmentVariable("SPO_USERNAME") ?? string.Empty; + string pass = Environment.GetEnvironmentVariable("SPO_PASSWORD") ?? string.Empty; + string conn = Environment.GetEnvironmentVariable("SQL_CONNECTION_STRING") ?? string.Empty; - return null; + return !string.IsNullOrEmpty(user) && !string.IsNullOrEmpty(pass) + ? (user, pass, conn) + : null; } } diff --git a/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs b/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs index 49c46d9..742fabf 100644 --- a/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs +++ b/src/SPO2SQL.Core/SharePoint/ActivitiesDQ.cs @@ -1,5 +1,3 @@ -using System; -using Microsoft.SharePoint.Client; using SPO2SQL.Logging; namespace SPO2SQL.SharePoint; @@ -15,7 +13,7 @@ public ActivitiesDQ(SPOUser user) : base(user) { } - public override bool Execute() + public override async Task ExecuteAsync(CancellationToken cancellationToken = default) { try { @@ -28,7 +26,8 @@ public override bool Execute() "" + ""; - var activitiesList = CreateAndBuildList(LIST_NAME, SITE, camlQuery); + using var activitiesList = await CreateAndBuildListAsync(LIST_NAME, SITE, camlQuery, cancellationToken) + .ConfigureAwait(false); if (activitiesList.ItemCollection.Count == 0) { @@ -36,7 +35,7 @@ public override bool Execute() return true; } - ProcessListItemsInBatches(activitiesList, item => + await ProcessListItemsInBatchesAsync(activitiesList, item => { var sourceValue = GetFieldValue(item, SOURCE_FIELD); if (sourceValue != null) @@ -44,14 +43,16 @@ public override bool Execute() SetFieldValue(item, TARGET_FIELD, sourceValue); item.Update(); } - }); + + return Task.CompletedTask; + }, cancellationToken: cancellationToken).ConfigureAwait(false); Logger.LogWarning($"Completed {LIST_NAME} update: {activitiesList.ItemCollection.Count} items processed"); return true; } catch (Exception ex) { - Logger.LogError($"Failed to update activities data quality", ex); + Logger.LogError("Failed to update activities data quality", ex); return false; } } diff --git a/src/SPO2SQL.Core/SharePoint/CamlQueryBuilder.cs b/src/SPO2SQL.Core/SharePoint/CamlQueryBuilder.cs index 09dc3c4..377b98c 100644 --- a/src/SPO2SQL.Core/SharePoint/CamlQueryBuilder.cs +++ b/src/SPO2SQL.Core/SharePoint/CamlQueryBuilder.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; namespace SPO2SQL.SharePoint; @@ -54,8 +55,8 @@ public static string BuildDateRangeQuery(string fieldName, DateTime from, DateTi throw new ArgumentException("'to' date must be greater than or equal to 'from' date."); } - var fromValue = EscapeXmlValue(from.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")); - var toValue = EscapeXmlValue(to.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")); + var fromValue = EscapeXmlValue(from.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)); + var toValue = EscapeXmlValue(to.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)); return $@" @@ -129,7 +130,7 @@ public static string BuildEqualNumberQuery(string fieldName, decimal value) - {EscapeXmlValue(value.ToString())} + {EscapeXmlValue(value.ToString(CultureInfo.InvariantCulture))} diff --git a/src/SPO2SQL.Core/SharePoint/ClientContextExtensions.cs b/src/SPO2SQL.Core/SharePoint/ClientContextExtensions.cs new file mode 100644 index 0000000..05e2154 --- /dev/null +++ b/src/SPO2SQL.Core/SharePoint/ClientContextExtensions.cs @@ -0,0 +1,19 @@ +using Microsoft.SharePoint.Client; + +namespace SPO2SQL.SharePoint; + +/// +/// Adds cooperative cancellation to the CSOM query API, which only exposes a +/// parameterless async overload in the current SharePoint client package. +/// +internal static class ClientContextExtensions +{ + public static async Task ExecuteQueryWithCancellationAsync( + this ClientRuntimeContext context, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + await context.ExecuteQueryAsync().ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } +} diff --git a/src/SPO2SQL.Core/SharePoint/Context.cs b/src/SPO2SQL.Core/SharePoint/Context.cs index 02e7d80..d4736c6 100644 --- a/src/SPO2SQL.Core/SharePoint/Context.cs +++ b/src/SPO2SQL.Core/SharePoint/Context.cs @@ -1,6 +1,3 @@ -using System; -using System.Linq.Expressions; -using System.Net; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; using SPO2SQL.XmlConfig; @@ -9,15 +6,15 @@ namespace SPO2SQL.SharePoint; public class Context : IDisposable { - internal Web web; + internal Web? web; - public string Site { get; set; } + public string Site { get; set; } = string.Empty; - public SPOUser SPOUser { get; set; } + public SPOUser SPOUser { get; set; } = null!; - public ClientContext Ctx { get; set; } + public ClientContext Ctx { get; set; } = null!; - public void BuildContext() + public async Task BuildContextAsync(CancellationToken cancellationToken = default) { try { @@ -32,12 +29,13 @@ public void BuildContext() SPOUser.ApplyAuthentication(clientContext, url); Ctx = clientContext; - web = Ctx.Web; + Web currentWeb = Ctx.Web; + web = currentWeb; - Ctx.Load(web, w => w.Title, w => w.Url); - Ctx.ExecuteQuery(); + Ctx.Load(currentWeb, w => w.Title, w => w.Url); + await ExecuteQueryAsync(cancellationToken).ConfigureAwait(false); - Logger.LogDebug($"SharePoint context established successfully for site: {web.Title}"); + Logger.LogDebug($"SharePoint context established successfully for site: {currentWeb.Title}"); } catch (Exception ex) { @@ -46,13 +44,19 @@ public void BuildContext() } } + protected async Task ExecuteQueryAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + await Ctx.ExecuteQueryWithCancellationAsync(cancellationToken).ConfigureAwait(false); + } + public void Dispose() { Ctx?.Dispose(); GC.SuppressFinalize(this); } - public ListCollection GetAllLists() + public async Task GetAllListsAsync(CancellationToken cancellationToken = default) { try { @@ -63,12 +67,13 @@ public ListCollection GetAllLists() if (web == null || Ctx?.Url != expectedUrl) { Logger.LogDebug("Rebuilding context for GetAllLists"); - BuildContext(); + await BuildContextAsync(cancellationToken).ConfigureAwait(false); } - var lists = web.Lists; - Ctx.Load(lists); - Ctx.ExecuteQuery(); + var currentWeb = web ?? throw new InvalidOperationException("SharePoint context is not initialized."); + var lists = currentWeb.Lists; + Ctx!.Load(lists); + await ExecuteQueryAsync(cancellationToken).ConfigureAwait(false); Logger.LogDebug($"Retrieved {lists.Count} lists from site"); return lists; diff --git a/src/SPO2SQL.Core/SharePoint/DataQualityBase.cs b/src/SPO2SQL.Core/SharePoint/DataQualityBase.cs index 737b3ba..01745a9 100644 --- a/src/SPO2SQL.Core/SharePoint/DataQualityBase.cs +++ b/src/SPO2SQL.Core/SharePoint/DataQualityBase.cs @@ -1,4 +1,3 @@ -using System; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; @@ -6,26 +5,23 @@ namespace SPO2SQL.SharePoint; internal abstract class DataQualityBase { - protected SPOUser User { get; set; } + protected SPOUser User { get; } protected DataQualityBase(SPOUser user) { User = user ?? throw new ArgumentNullException(nameof(user)); } - public abstract bool Execute(); + public abstract Task ExecuteAsync(CancellationToken cancellationToken = default); - protected SPOList CreateAndBuildList(string listName, string site, string camlQuery = null) + protected async Task CreateAndBuildListAsync( + string listName, + string site, + string? camlQuery = null, + CancellationToken cancellationToken = default) { - if (string.IsNullOrWhiteSpace(listName)) - { - throw new ArgumentException("List name cannot be null or empty", nameof(listName)); - } - - if (string.IsNullOrWhiteSpace(site)) - { - throw new ArgumentException("Site cannot be null or empty", nameof(site)); - } + ArgumentException.ThrowIfNullOrWhiteSpace(listName); + ArgumentException.ThrowIfNullOrWhiteSpace(site); Logger.LogDebug($"Creating list '{listName}' on site '{site}'"); @@ -39,80 +35,61 @@ protected SPOList CreateAndBuildList(string listName, string site, string camlQu try { - list.Build(); - Logger.LogDebug($"List '{listName}' built successfully with {list.ItemCollection?.Count ?? 0} items"); + await list.BuildAsync(cancellationToken).ConfigureAwait(false); + Logger.LogDebug($"List '{listName}' built successfully with {list.ItemCollection.Count} items"); return list; } - catch (Exception ex) + catch { - Logger.LogError($"Failed to build list '{listName}' on site '{site}'", ex); list.Dispose(); throw; } } - protected static void ProcessListItemsInBatches(SPOList list, Action processor, int batchSize = 80) + protected static async Task ProcessListItemsInBatchesAsync( + SPOList list, + Func processor, + int batchSize = 80, + CancellationToken cancellationToken = default) { - if (processor == null) - { - throw new ArgumentNullException(nameof(processor), "Item processor delegate cannot be null"); - } - + ArgumentNullException.ThrowIfNull(processor); if (batchSize <= 0) { throw new ArgumentException("batchSize must be greater than 0", nameof(batchSize)); } - if (list?.ItemCollection == null || list.ItemCollection.Count == 0) + if (list.ItemCollection.Count == 0) { - Logger.LogDebug($"No items to process in list '{list?.Name}'"); + Logger.LogDebug($"No items to process in list '{list.Name}'"); return; } - try + int batchCount = 0; + foreach (ListItem item in list.ItemCollection) { - int batchCount = 0; - foreach (ListItem item in list.ItemCollection) - { - processor(item); + cancellationToken.ThrowIfCancellationRequested(); - if (++batchCount % batchSize == 0) - { - Logger.LogDebug($"Executing batch query after {batchCount} items"); - list.Ctx.ExecuteQuery(); - } - } + await processor(item).ConfigureAwait(false); - if (batchCount % batchSize != 0) + if (++batchCount % batchSize == 0) { - Logger.LogDebug($"Executing final batch query for remaining {batchCount % batchSize} items"); - list.Ctx.ExecuteQuery(); + await list.Ctx.ExecuteQueryWithCancellationAsync(cancellationToken).ConfigureAwait(false); } - - Logger.LogDebug($"Completed processing {batchCount} items from list '{list.Name}'"); } - catch (Exception ex) + + if (batchCount % batchSize != 0) { - Logger.LogError($"Error processing items in list '{list.Name}'", ex); - throw; + await list.Ctx.ExecuteQueryWithCancellationAsync(cancellationToken).ConfigureAwait(false); } + + Logger.LogDebug($"Completed processing {batchCount} items from list '{list.Name}'"); } - protected static T GetFieldValue(ListItem item, string fieldName) + protected static T? GetFieldValue(ListItem item, string fieldName) { try { - if (item == null) - { - return default; - } - - if (item[fieldName] is T value) - { - return value; - } - - return default; + return item[fieldName] is T value ? value : default; } catch (Exception ex) { @@ -121,16 +98,5 @@ protected static T GetFieldValue(ListItem item, string fieldName) } } - protected static void SetFieldValue(ListItem item, string fieldName, object value) - { - try - { - item[fieldName] = value; - } - catch (Exception ex) - { - Logger.LogError($"Failed to set field '{fieldName}' on item: {ex.Message}"); - throw; - } - } + protected static void SetFieldValue(ListItem item, string fieldName, object? value) => item[fieldName] = value; } diff --git a/src/SPO2SQL.Core/SharePoint/HealthChecker.cs b/src/SPO2SQL.Core/SharePoint/HealthChecker.cs index 58ba8d4..b0add93 100644 --- a/src/SPO2SQL.Core/SharePoint/HealthChecker.cs +++ b/src/SPO2SQL.Core/SharePoint/HealthChecker.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using Microsoft.Data.SqlClient; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; @@ -13,8 +11,8 @@ public class HealthChecker public class HealthCheckResult { public bool IsHealthy { get; set; } = true; - public List Warnings { get; set; } = new List(); - public List Errors { get; set; } = new List(); + public List Warnings { get; set; } = []; + public List Errors { get; set; } = []; public override string ToString() { @@ -24,19 +22,13 @@ public override string ToString() if (Errors.Count > 0) { result += $"\nErrors ({Errors.Count}):"; - foreach (var error in Errors) - { - result += $"\n \u2717 {error}"; - } + result = Errors.Aggregate(result, (current, error) => current + $"\n ✗ {error}"); } if (Warnings.Count > 0) { result += $"\nWarnings ({Warnings.Count}):"; - foreach (var warning in Warnings) - { - result += $"\n \u26A0 {warning}"; - } + result = Warnings.Aggregate(result, (current, warning) => current + $"\n ⚠ {warning}"); } return result; @@ -48,7 +40,11 @@ public HealthChecker(int verbosity = 0) _verbosity = verbosity; } - public HealthCheckResult PerformHealthCheck(SPOUser spoUser, string sharePointUrl, string sqlConnectionString) + public async Task PerformHealthCheckAsync( + SPOUser spoUser, + string sharePointUrl, + string sqlConnectionString, + CancellationToken cancellationToken = default) { var result = new HealthCheckResult(); @@ -59,9 +55,11 @@ public HealthCheckResult PerformHealthCheck(SPOUser spoUser, string sharePointUr ValidateSPOUserCredentials(spoUser, result); - ValidateSharePointConnectivity(spoUser, sharePointUrl, result); + await ValidateSharePointConnectivityAsync(spoUser, sharePointUrl, result, cancellationToken) + .ConfigureAwait(false); - ValidateSqlConnectivity(sqlConnectionString, result); + await ValidateSqlConnectivityAsync(sqlConnectionString, result, cancellationToken) + .ConfigureAwait(false); result.IsHealthy = result.Errors.Count == 0; @@ -73,76 +71,59 @@ public HealthCheckResult PerformHealthCheck(SPOUser spoUser, string sharePointUr return result; } - private static void ValidateSPOUserCredentials(SPOUser spoUser, HealthCheckResult result) + private static void ValidateSPOUserCredentials(SPOUser? spoUser, HealthCheckResult result) { if (spoUser == null) { result.Errors.Add("SPOUser is null - SharePoint credentials not initialized."); - result.IsHealthy = false; return; } if (string.IsNullOrWhiteSpace(spoUser.Username)) { result.Errors.Add("SharePoint username is empty or null."); - result.IsHealthy = false; } - - // Credential material is intentionally not exposed by SPOUser; connectivity - // validation below is the authoritative authentication check. } - private void ValidateSharePointConnectivity(SPOUser spoUser, string sharePointUrl, HealthCheckResult result) + private async Task ValidateSharePointConnectivityAsync( + SPOUser spoUser, + string sharePointUrl, + HealthCheckResult result, + CancellationToken cancellationToken) { try { - if (spoUser == null) - { - result.Errors.Add("SPOUser is null - cannot validate connectivity."); - return; - } - if (string.IsNullOrWhiteSpace(sharePointUrl)) { result.Errors.Add("SharePoint URL is not configured."); return; } - if (!sharePointUrl.StartsWith("https://") && !sharePointUrl.StartsWith("http://")) + if (!sharePointUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase) && + !sharePointUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { result.Errors.Add("SharePoint URL must start with http:// or https://"); return; } - using (var clientContext = new ClientContext(sharePointUrl)) - { - if (spoUser != null) - { - spoUser.ApplyAuthentication(clientContext, sharePointUrl); - } + using var clientContext = new ClientContext(sharePointUrl); + spoUser.ApplyAuthentication(clientContext, sharePointUrl); - if (clientContext == null) - { - result.Errors.Add("Failed to create SharePoint client context."); - return; - } - - var web = clientContext.Web; - clientContext.Load(web, w => w.Title); + var web = clientContext.Web; + clientContext.Load(web, w => w.Title); - try - { - clientContext.ExecuteQuery(); - if (_verbosity >= 3) - { - Logger.Log(3, $"\u2713 SharePoint connectivity verified. Site: {web.Title}"); - } - } - catch (Exception ex) + try + { + await clientContext.ExecuteQueryWithCancellationAsync(cancellationToken).ConfigureAwait(false); + if (_verbosity >= 3) { - result.Errors.Add($"SharePoint connection failed: {ex.Message}"); + Logger.Log(3, $"✓ SharePoint connectivity verified. Site: {web.Title}"); } } + catch (Exception ex) + { + result.Errors.Add($"SharePoint connection failed: {ex.Message}"); + } } catch (Exception ex) { @@ -150,7 +131,10 @@ private void ValidateSharePointConnectivity(SPOUser spoUser, string sharePointUr } } - private void ValidateSqlConnectivity(string sqlConnectionString, HealthCheckResult result) + private async Task ValidateSqlConnectivityAsync( + string sqlConnectionString, + HealthCheckResult result, + CancellationToken cancellationToken) { try { @@ -160,15 +144,12 @@ private void ValidateSqlConnectivity(string sqlConnectionString, HealthCheckResu return; } - using (var connection = new SqlConnection(sqlConnectionString)) - { - connection.Open(); - if (_verbosity >= 3) - { - Logger.Log(3, "\u2713 SQL Server connectivity verified."); - } + await using var connection = new SqlConnection(sqlConnectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - connection.Close(); + if (_verbosity >= 3) + { + Logger.Log(3, "✓ SQL Server connectivity verified."); } } catch (SqlException sqlEx) @@ -181,7 +162,11 @@ private void ValidateSqlConnectivity(string sqlConnectionString, HealthCheckResu } } - public HealthCheckResult ValidateListAccess(SPOUser spoUser, string sharePointUrl, string listName) + public async Task ValidateListAccessAsync( + SPOUser spoUser, + string sharePointUrl, + string listName, + CancellationToken cancellationToken = default) { var result = new HealthCheckResult(); @@ -201,30 +186,25 @@ public HealthCheckResult ValidateListAccess(SPOUser spoUser, string sharePointUr return result; } - using (var clientContext = new ClientContext(sharePointUrl)) - { - if (spoUser != null) - { - spoUser.ApplyAuthentication(clientContext, sharePointUrl); - } + using var clientContext = new ClientContext(sharePointUrl); + spoUser?.ApplyAuthentication(clientContext, sharePointUrl); - var list = clientContext.Web.Lists.GetByTitle(listName); - clientContext.Load(list, l => l.Title, l => l.ItemCount); + var list = clientContext.Web.Lists.GetByTitle(listName); + clientContext.Load(list, l => l.Title, l => l.ItemCount); - try - { - clientContext.ExecuteQuery(); - if (_verbosity >= 3) - { - Logger.Log(3, $"\u2713 List '{listName}' is accessible ({list.ItemCount} items)"); - } - } - catch (Exception ex) + try + { + await clientContext.ExecuteQueryWithCancellationAsync(cancellationToken).ConfigureAwait(false); + if (_verbosity >= 3) { - result.Errors.Add($"Cannot access list '{listName}': {ex.Message}"); - result.IsHealthy = false; + Logger.Log(3, $"✓ List '{listName}' is accessible ({list.ItemCount} items)"); } } + catch (Exception ex) + { + result.Errors.Add($"Cannot access list '{listName}': {ex.Message}"); + result.IsHealthy = false; + } } catch (Exception ex) { diff --git a/src/SPO2SQL.Core/SharePoint/InvoiceRequestDQ.cs b/src/SPO2SQL.Core/SharePoint/InvoiceRequestDQ.cs index 1927f18..dccbf8c 100644 --- a/src/SPO2SQL.Core/SharePoint/InvoiceRequestDQ.cs +++ b/src/SPO2SQL.Core/SharePoint/InvoiceRequestDQ.cs @@ -1,6 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; @@ -23,7 +20,7 @@ public InvoiceRequestDQ(SPOUser user) : base(user) { } - public override bool Execute() + public override async Task ExecuteAsync(CancellationToken cancellationToken = default) { try { @@ -37,7 +34,8 @@ public override bool Execute() $"{cutoffIso}" + ""; - using var unitList = CreateAndBuildList(UNIT_LIST, UNIT_SITE, unitQuery); + using var unitList = await CreateAndBuildListAsync(UNIT_LIST, UNIT_SITE, unitQuery, cancellationToken) + .ConfigureAwait(false); var unitMap = new Dictionary(); foreach (ListItem unit in unitList.ItemCollection) @@ -55,10 +53,9 @@ public override bool Execute() return true; } - Logger.LogDebug($"Found {unitMap.Count} recently modified Unit items"); - string invoiceQuery = BuildInvoiceQuery(unitMap); - using var invoiceList = CreateAndBuildList(INVOICE_LIST, INVOICE_SITE, invoiceQuery); + using var invoiceList = await CreateAndBuildListAsync(INVOICE_LIST, INVOICE_SITE, invoiceQuery, cancellationToken) + .ConfigureAwait(false); if (invoiceList.ItemCollection.Count == 0) { @@ -66,9 +63,7 @@ public override bool Execute() return true; } - Logger.LogDebug($"Found {invoiceList.ItemCollection.Count} Invoice Request items to process"); - - ProcessListItemsInBatches(invoiceList, item => + await ProcessListItemsInBatchesAsync(invoiceList, item => { try { @@ -85,7 +80,9 @@ public override bool Execute() { Logger.LogDebug($"Failed to update invoice item {item.Id}: {ex.Message}"); } - }); + + return Task.CompletedTask; + }, cancellationToken: cancellationToken).ConfigureAwait(false); Logger.LogWarning("Completed Invoice Request approver update operation"); return true; @@ -106,11 +103,10 @@ private static string BuildInvoiceQuery(Dictionary unitMap) var conditions = unitMap.Keys.Select(key => $"" + - $"{CamlQueryBuilder.EscapeXmlValue(key)}"); + $"{CamlQueryBuilder.EscapeXmlValue(key)}").ToArray(); - var conditionArray = conditions.ToArray(); - return conditionArray.Length == 1 - ? $"{conditionArray[0]}" - : CamlQueryBuilder.BuildOrQuery(conditionArray); + return conditions.Length == 1 + ? $"{conditions[0]}" + : CamlQueryBuilder.BuildOrQuery(conditions); } } diff --git a/src/SPO2SQL.Core/SharePoint/OperationContext.cs b/src/SPO2SQL.Core/SharePoint/OperationContext.cs index fac6752..a6fe214 100644 --- a/src/SPO2SQL.Core/SharePoint/OperationContext.cs +++ b/src/SPO2SQL.Core/SharePoint/OperationContext.cs @@ -7,7 +7,7 @@ public class OperationContext { public string CorrelationId { get; set; } = Guid.NewGuid().ToString("N"); - public string OperationName { get; set; } + public string OperationName { get; set; } = string.Empty; public DateTime StartTime { get; set; } = DateTime.Now; diff --git a/src/SPO2SQL.Core/SharePoint/RetryPolicy.cs b/src/SPO2SQL.Core/SharePoint/RetryPolicy.cs index 5c01ae7..90932db 100644 --- a/src/SPO2SQL.Core/SharePoint/RetryPolicy.cs +++ b/src/SPO2SQL.Core/SharePoint/RetryPolicy.cs @@ -106,6 +106,67 @@ public T ExecuteWithRetry(Func operation, CancellationToken cancellationTo throw new InvalidOperationException($"Operation '{operationName}' failed after {_maxRetries + 1} attempts."); } + /// + /// Async variant used by the synchronization pipeline. Delays are cancellation-aware. + /// + public async Task ExecuteWithRetryAsync( + Func> operation, + CancellationToken cancellationToken, + string operationName = "Unknown") + { + ArgumentNullException.ThrowIfNull(operation); + + int attempt = 0; + while (attempt <= _maxRetries) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + if (attempt > 0) + { + Logger.LogWarning($"Retry attempt {attempt} of {_maxRetries} for operation '{operationName}'"); + } + + return await operation().ConfigureAwait(false); + } + catch (Exception ex) when (IsTransientError(ex) && attempt <= _maxRetries) + { + attempt++; + if (attempt > _maxRetries) + { + throw new InvalidOperationException($"Operation '{operationName}' failed after {_maxRetries + 1} attempts.", ex); + } + + int delayMs = CalculateBackoffDelay(attempt); + Logger.LogWarning( + $"Transient error in '{operationName}': {ex.Message}. " + + $"Retrying in {delayMs}ms (attempt {attempt}/{_maxRetries})"); + + await Task.Delay(delayMs, cancellationToken).ConfigureAwait(false); + } + } + + throw new InvalidOperationException($"Operation '{operationName}' failed after {_maxRetries + 1} attempts."); + } + + public async Task ExecuteWithRetryAsync( + Func operation, + CancellationToken cancellationToken, + string operationName = "Unknown") + { + ArgumentNullException.ThrowIfNull(operation); + + await ExecuteWithRetryAsync( + async () => + { + await operation().ConfigureAwait(false); + return true; + }, + cancellationToken, + operationName).ConfigureAwait(false); + } + public void ExecuteWithRetry(Action operation, CancellationToken cancellationToken, string operationName = "Unknown") { ArgumentNullException.ThrowIfNull(operation); diff --git a/src/SPO2SQL.Core/SharePoint/SPOList.cs b/src/SPO2SQL.Core/SharePoint/SPOList.cs index c312f27..d08b954 100644 --- a/src/SPO2SQL.Core/SharePoint/SPOList.cs +++ b/src/SPO2SQL.Core/SharePoint/SPOList.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; -using System.Linq.Expressions; using System.Text; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; @@ -12,46 +12,67 @@ public class SPOList : Context { private static readonly string[] s_nullArray = ["NULL"]; - private List _list; + private List? _list; - public string Name { get; set; } + /// + /// Number of SharePoint items requested per page. Bound to Sql:BatchSize. + /// + public int PageSize { get; set; } = 500; - public ListItemCollection ItemCollection { get; set; } + public string Name { get; set; } = string.Empty; - public FieldCollection Fields { get; set; } + /// + /// All items returned by the query, across every SharePoint page. + /// + public IReadOnlyList ItemCollection { get; private set; } = Array.Empty(); - public string CAMLQuery { get; set; } + public FieldCollection Fields { get; set; } = null!; - public void Build() + public string? CAMLQuery { get; set; } + + public async Task BuildAsync(CancellationToken cancellationToken = default) { try { if (string.IsNullOrWhiteSpace(Name)) { - throw new InvalidOperationException("List 'Name' property must be set before calling Build()"); + throw new InvalidOperationException("List 'Name' property must be set before calling BuildAsync()"); } Logger.Log(2, $"Building list '{Name}' from site '{Site}'"); if (web == null) { - BuildContext(); + await BuildContextAsync(cancellationToken).ConfigureAwait(false); } CamlQuery camlQuery = string.IsNullOrEmpty(CAMLQuery) - ? CamlQuery.CreateAllItemsQuery() + ? CamlQuery.CreateAllItemsQuery(PageSize) : new CamlQuery { ViewXml = CAMLQuery }; - _list = web.Lists.GetByTitle(Name); - ItemCollection = _list.GetItems(camlQuery); + _list = (web ?? throw new InvalidOperationException("SharePoint context is not initialized.")) + .Lists.GetByTitle(Name); Fields = _list.Fields; Ctx.Load(_list); - Ctx.Load(ItemCollection); Ctx.Load(Fields); + await ExecuteQueryAsync(cancellationToken).ConfigureAwait(false); + + var items = new List(); + do + { + cancellationToken.ThrowIfCancellationRequested(); - Logger.Log(2, "Executing query to retrieve list items"); - Ctx.ExecuteQuery(); + ListItemCollection page = _list.GetItems(camlQuery); + Ctx.Load(page); + await ExecuteQueryAsync(cancellationToken).ConfigureAwait(false); + + items.AddRange(page); + camlQuery.ListItemCollectionPosition = page.ListItemCollectionPosition; + } + while (camlQuery.ListItemCollectionPosition != null); + + ItemCollection = items; Logger.LogDebug($"Successfully loaded {ItemCollection.Count} items from list '{Name}'"); } @@ -62,13 +83,17 @@ public void Build() } } - public void Update() + public ListItem GetItemById(int id) => + ItemCollection.FirstOrDefault(item => item.Id == id) + ?? throw new InvalidOperationException($"Item with ID {id} was not loaded from list '{Name}'."); + + public async Task UpdateAsync(CancellationToken cancellationToken = default) { try { Logger.Log(2, $"Updating list '{Name}'"); - Ctx.ExecuteQuery(); - Build(); + await ExecuteQueryAsync(cancellationToken).ConfigureAwait(false); + await BuildAsync(cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -83,7 +108,7 @@ public ListItem AddItem() { if (_list == null) { - throw new InvalidOperationException($"List '{Name}' has not been built. Call Build() first."); + throw new InvalidOperationException($"List '{Name}' has not been built. Call BuildAsync() first."); } ListItem listItem = _list.AddItem(new ListItemCreationInformation()); @@ -123,12 +148,12 @@ public void PrintItemProperties(ListItem item) { object fieldValue = item[field.InternalName]; string formattedValue = FormatFieldOutput(field, fieldValue); - sb.AppendLine($"{field.Title}|{field.InternalName}|{formattedValue}|{field.CanBeDeleted}|{field.Hidden}|{field.TypeAsString}|{field.ReadOnlyField}|{field.FromBaseType}|{field.Required}|{fieldValue?.GetType().Name ?? "NULL"}"); + sb.AppendLine(CultureInfo.InvariantCulture, $"{field.Title}|{field.InternalName}|{formattedValue}|{field.CanBeDeleted}|{field.Hidden}|{field.TypeAsString}|{field.ReadOnlyField}|{field.FromBaseType}|{field.Required}|{fieldValue?.GetType().Name ?? "NULL"}"); } catch (Exception ex) { Logger.LogWarning($"Could not retrieve value for field '{field.Title}': {ex.Message}"); - sb.AppendLine($"{field.Title}|{field.InternalName}|ERROR|{field.CanBeDeleted}|{field.Hidden}|{field.TypeAsString}|{field.ReadOnlyField}|{field.FromBaseType}|{field.Required}|ERROR"); + sb.AppendLine(CultureInfo.InvariantCulture, $"{field.Title}|{field.InternalName}|ERROR|{field.CanBeDeleted}|{field.Hidden}|{field.TypeAsString}|{field.ReadOnlyField}|{field.FromBaseType}|{field.Required}|ERROR"); } } diff --git a/src/SPO2SQL.Core/SharePoint/SPOUser.cs b/src/SPO2SQL.Core/SharePoint/SPOUser.cs index 9a6f35a..8bc5704 100644 --- a/src/SPO2SQL.Core/SharePoint/SPOUser.cs +++ b/src/SPO2SQL.Core/SharePoint/SPOUser.cs @@ -1,90 +1,138 @@ -using System; using System.Net; using System.Security; -using System.Threading.Tasks; +using System.Security.Cryptography.X509Certificates; using Microsoft.Identity.Client; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; namespace SPO2SQL.SharePoint; -public class SPOUser : IDisposable +public sealed class SPOUser : IDisposable { private const string WellKnownClientId = "9bc3ab49-b65d-410a-85ad-de819febfddc"; - public string Username { get; private set; } - private readonly string _plainPassword; - private SecureString _securePassword; - private AuthenticationResult _tokenResult; - private string _lastSiteUrl; + private readonly SecureString? _securePassword; + private readonly IPublicClientApplication? _publicApp; + private readonly IConfidentialClientApplication? _confidentialApp; + private readonly X509Certificate2? _certificate; + private readonly SemaphoreSlim _tokenLock = new(1, 1); - internal ICredentials Credentials { get; private set; } + private AuthenticationResult? _tokenResult; + private string? _lastSiteUrl; + private bool _disposed; - public SPOUser(string username, string password) - { - if (string.IsNullOrWhiteSpace(username)) - { - throw new ArgumentException("Username cannot be null or empty.", nameof(username)); - } + public string Username { get; } - if (password == null) - { - throw new ArgumentNullException(nameof(password), "Password cannot be null."); - } + internal ICredentials? Credentials { get; } - Logger.LogDebug($"Initializing SPOUser for: {username}"); + /// + /// Legacy username/password (ROPC) authentication. Kept for environments that + /// cannot yet use an app-only service principal. + /// + public SPOUser(string username, string password) + { + ArgumentException.ThrowIfNullOrWhiteSpace(username); + ArgumentNullException.ThrowIfNull(password); Username = username; _plainPassword = password; Credentials = new NetworkCredential(username, password); - _securePassword = new SecureString(); + var securePassword = new SecureString(); foreach (char c in password) { - _securePassword.AppendChar(c); + securePassword.AppendChar(c); } - _securePassword.MakeReadOnly(); + securePassword.MakeReadOnly(); + _securePassword = securePassword; + + _publicApp = PublicClientApplicationBuilder.Create(WellKnownClientId) + .WithAuthority("https://login.microsoftonline.com/organizations") + .Build(); } - public async Task GetAccessTokenAsync(string siteUrl) + /// + /// App-only authentication with a certificate. This is the recommended mode for + /// unattended jobs because it is not blocked by MFA or Conditional Access. + /// + public SPOUser(AppOnlyAuthSettings settings) { - if (!string.IsNullOrEmpty(_tokenResult?.AccessToken) && _lastSiteUrl == siteUrl && _tokenResult.ExpiresOn > DateTimeOffset.UtcNow.AddMinutes(5)) - { - return _tokenResult.AccessToken; - } + ArgumentNullException.ThrowIfNull(settings); + ArgumentException.ThrowIfNullOrWhiteSpace(settings.TenantId); + ArgumentException.ThrowIfNullOrWhiteSpace(settings.ClientId); + ArgumentException.ThrowIfNullOrWhiteSpace(settings.CertificatePath); + + _plainPassword = string.Empty; + Username = $"app:{settings.ClientId}"; - var authority = "https://login.microsoftonline.com/organizations"; - var app = PublicClientApplicationBuilder.Create(WellKnownClientId) - .WithAuthority(authority) + _certificate = string.IsNullOrWhiteSpace(settings.CertificatePassword) + ? new X509Certificate2(settings.CertificatePath) + : new X509Certificate2(settings.CertificatePath, settings.CertificatePassword); + + _confidentialApp = ConfidentialClientApplicationBuilder.Create(settings.ClientId) + .WithAuthority($"https://login.microsoftonline.com/{settings.TenantId}") + .WithCertificate(_certificate) .Build(); - var uri = new Uri(siteUrl); - var scope = $"{uri.Scheme}://{uri.Host}/.default"; + Logger.LogInformation($"Using app-only authentication for client {settings.ClientId}"); + } + public async Task GetAccessTokenAsync(string siteUrl) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ArgumentException.ThrowIfNullOrWhiteSpace(siteUrl); + + await _tokenLock.WaitAsync().ConfigureAwait(false); try { - _tokenResult = await app.AcquireTokenByUsernamePassword( - new[] { scope }, - Username, - _plainPassword) - .ExecuteAsync(); + if (_tokenResult != null && + _lastSiteUrl == siteUrl && + _tokenResult.ExpiresOn > DateTimeOffset.UtcNow.AddMinutes(5)) + { + return _tokenResult.AccessToken; + } + + var uri = new Uri(siteUrl); + string scope = $"{uri.Scheme}://{uri.Host}/.default"; + + AuthenticationResult result; + if (_confidentialApp != null) + { + result = await _confidentialApp + .AcquireTokenForClient([scope]) + .ExecuteAsync() + .ConfigureAwait(false); + } + else + { + try + { +#pragma warning disable CS0618 // ROPC is kept only as a fallback for legacy tenants. + result = await _publicApp! + .AcquireTokenByUsernamePassword([scope], Username, _plainPassword) + .ExecuteAsync() + .ConfigureAwait(false); +#pragma warning restore CS0618 + } + catch (MsalUiRequiredException ex) + { + Logger.LogError("Interactive login required. MFA or Conditional Access is blocking ROPC.", ex); + throw new InvalidOperationException( + "SharePoint Online authentication failed: MFA or Conditional Access policy requires interactive login. " + + "Configure app-only authentication (TenantId, ClientId and CertificatePath).", ex); + } + } + + _tokenResult = result; _lastSiteUrl = siteUrl; Logger.LogDebug("OAuth token acquired successfully for " + siteUrl); - return _tokenResult.AccessToken; - } - catch (MsalUiRequiredException ex) - { - Logger.LogError("Interactive login required. This may be due to MFA or Conditional Access policies.", ex); - throw new InvalidOperationException( - "SharePoint Online authentication failed: MFA or Conditional Access policy requires interactive login. " + - "Consider using app-only authentication with a service principal.", ex); + return result.AccessToken; } - catch (MsalException ex) + finally { - Logger.LogError($"MSAL authentication failed: {ex.Message}", ex); - throw; + _tokenLock.Release(); } } @@ -93,7 +141,7 @@ public void ApplyAuthentication(ClientContext ctx, string siteUrl) ArgumentNullException.ThrowIfNull(ctx); ArgumentException.ThrowIfNullOrWhiteSpace(siteUrl); - ctx.ExecutingWebRequest += (sender, e) => + ctx.ExecutingWebRequest += (_, e) => { string token = GetAccessTokenAsync(siteUrl).GetAwaiter().GetResult(); e.WebRequestExecutor.RequestHeaders["Authorization"] = "Bearer " + token; @@ -102,7 +150,15 @@ public void ApplyAuthentication(ClientContext ctx, string siteUrl) public void Dispose() { + if (_disposed) + { + return; + } + _securePassword?.Dispose(); + _certificate?.Dispose(); + _tokenLock.Dispose(); + _disposed = true; GC.SuppressFinalize(this); } } diff --git a/src/SPO2SQL.Core/SharePoint/SPOUserFactory.cs b/src/SPO2SQL.Core/SharePoint/SPOUserFactory.cs new file mode 100644 index 0000000..111e893 --- /dev/null +++ b/src/SPO2SQL.Core/SharePoint/SPOUserFactory.cs @@ -0,0 +1,31 @@ +using SPO2SQL.Configuration; + +namespace SPO2SQL.SharePoint; + +/// +/// Parameters for app-only (client credentials) authentication. +/// +public sealed record AppOnlyAuthSettings( + string TenantId, + string ClientId, + string CertificatePath, + string? CertificatePassword); + +/// +/// Creates the authentication context used by every SharePoint operation. +/// +public static class SPOUserFactory +{ + public static SPOUser Create(SyncSettings settings) + { + ArgumentNullException.ThrowIfNull(settings); + + return settings.HasAppOnlyAuth + ? new SPOUser(new AppOnlyAuthSettings( + settings.TenantId!, + settings.ClientId!, + settings.CertificatePath!, + settings.CertificatePassword)) + : new SPOUser(settings.Username, settings.Password); + } +} diff --git a/src/SPO2SQL.Core/SharePoint/SharePointAuthConfig.cs b/src/SPO2SQL.Core/SharePoint/SharePointAuthConfig.cs index 6a26d47..24af760 100644 --- a/src/SPO2SQL.Core/SharePoint/SharePointAuthConfig.cs +++ b/src/SPO2SQL.Core/SharePoint/SharePointAuthConfig.cs @@ -6,15 +6,15 @@ public class SharePointAuthConfig { public bool UseAppRegistration { get; set; } - public string ClientId { get; set; } + public string ClientId { get; set; } = string.Empty; - public string ClientSecret { get; set; } + public string ClientSecret { get; set; } = string.Empty; - public string TenantId { get; set; } + public string TenantId { get; set; } = string.Empty; - public string Username { get; set; } + public string Username { get; set; } = string.Empty; - public string Password { get; set; } + public string Password { get; set; } = string.Empty; - public string SiteUrl { get; set; } + public string SiteUrl { get; set; } = string.Empty; } diff --git a/src/SPO2SQL.Core/SharePoint/TimesheetDQ.cs b/src/SPO2SQL.Core/SharePoint/TimesheetDQ.cs index 85042e2..a1f538a 100644 --- a/src/SPO2SQL.Core/SharePoint/TimesheetDQ.cs +++ b/src/SPO2SQL.Core/SharePoint/TimesheetDQ.cs @@ -1,4 +1,3 @@ -using System; using Microsoft.SharePoint.Client; using SPO2SQL.Logging; @@ -19,7 +18,7 @@ public TimesheetDQ(SPOUser user) : base(user) { } - public override bool Execute() + public override async Task ExecuteAsync(CancellationToken cancellationToken = default) { try { @@ -32,7 +31,8 @@ public override bool Execute() "" + ""; - using var timesheetList = CreateAndBuildList(TIMESHEET_LIST, TIMESHEET_SITE, camlQuery); + using var timesheetList = await CreateAndBuildListAsync(TIMESHEET_LIST, TIMESHEET_SITE, camlQuery, cancellationToken) + .ConfigureAwait(false); if (timesheetList.ItemCollection.Count == 0) { @@ -40,11 +40,11 @@ public override bool Execute() return true; } - ProcessListItemsInBatches(timesheetList, item => + await ProcessListItemsInBatchesAsync(timesheetList, async item => { try { - var unitRecord = GetUnitItem(item); + var unitRecord = await GetUnitItemAsync(item, cancellationToken).ConfigureAwait(false); if (unitRecord != null && unitRecord[MAIN_APPROVER_FIELD] != null) { SetFieldValue(item, MAIN_APPROVER_FIELD, unitRecord[MAIN_APPROVER_FIELD]); @@ -56,7 +56,7 @@ public override bool Execute() } else { - var structureApprover = GetStructureApprover(item); + var structureApprover = await GetStructureApproverAsync(item, cancellationToken).ConfigureAwait(false); if (structureApprover != null) { SetFieldValue(item, MAIN_APPROVER_FIELD, structureApprover); @@ -67,9 +67,9 @@ public override bool Execute() } catch (Exception ex) { - Logger.LogError($"Error processing Timesheet item ID {item.Id}: {ex.Message}"); + Logger.LogError($"Error processing Timesheet item ID {item.Id}", ex); } - }); + }, cancellationToken: cancellationToken).ConfigureAwait(false); Logger.LogWarning("Completed Timesheet approver backfill operation"); return true; @@ -81,7 +81,7 @@ public override bool Execute() } } - private ListItem GetUnitItem(ListItem item) + private async Task GetUnitItemAsync(ListItem item, CancellationToken cancellationToken) { try { @@ -99,7 +99,8 @@ private ListItem GetUnitItem(ListItem item) "" + ""; - using var unitList = CreateAndBuildList(UNIT_LIST, UNIT_SITE, camlUnitQuery); + using var unitList = await CreateAndBuildListAsync(UNIT_LIST, UNIT_SITE, camlUnitQuery, cancellationToken) + .ConfigureAwait(false); return unitList.ItemCollection.Count > 0 ? unitList.ItemCollection[0] : null; } catch (Exception ex) @@ -109,7 +110,7 @@ private ListItem GetUnitItem(ListItem item) } } - private FieldUserValue GetStructureApprover(ListItem item) + private async Task GetStructureApproverAsync(ListItem item, CancellationToken cancellationToken) { try { @@ -127,13 +128,11 @@ private FieldUserValue GetStructureApprover(ListItem item) "" + ""; - using var hrList = CreateAndBuildList(HR_LIST, HR_SITE, camlPersonQuery); - if (hrList.ItemCollection.Count > 0) - { - return GetFieldValue(hrList.ItemCollection[0], "Approver1"); - } - - return null; + using var hrList = await CreateAndBuildListAsync(HR_LIST, HR_SITE, camlPersonQuery, cancellationToken) + .ConfigureAwait(false); + return hrList.ItemCollection.Count > 0 + ? GetFieldValue(hrList.ItemCollection[0], "Approver1") + : null; } catch (Exception ex) { diff --git a/src/SPO2SQL.Core/SharePoint/TwoFactorAuth.cs b/src/SPO2SQL.Core/SharePoint/TwoFactorAuth.cs deleted file mode 100644 index 3f51f35..0000000 --- a/src/SPO2SQL.Core/SharePoint/TwoFactorAuth.cs +++ /dev/null @@ -1,239 +0,0 @@ -#nullable disable -using System; -using System.Collections.Generic; -using System.Security.Cryptography; -using System.Text; -using SPO2SQL.XmlConfig; - -namespace SPO2SQL.SharePoint; - -public static class TwoFactorAuth -{ - private const string APP_NAME = "SPO2SQL"; - private const int TIME_STEP = 30; - private const int CODE_DIGITS = 6; - private const int MOD_VALUE = 1000000; - - private static readonly int[] Base32Lookup = new int[128]; - - static TwoFactorAuth() - { - for (int i = 0; i < 128; i++) - { - Base32Lookup[i] = -1; - } - - string alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; - for (int i = 0; i < alpha.Length; i++) - { - Base32Lookup[alpha[i]] = i; - } - } - - public static bool IsEnabled => ConfigurationReader.IsTwoFactorEnabled(); - - public static bool PerformVerification() - { - if (!IsEnabled) - { - return true; - } - - string secret = ConfigurationReader.GetTwoFactorSecret(); - - if (string.IsNullOrEmpty(secret)) - { - return RunSetup(); - } - - return RunVerification(secret); - } - - private static bool RunSetup() - { - string secret = GenerateSecret(); - - Console.WriteLine(); - Console.WriteLine("=== Two-Factor Authentication Setup ==="); - Console.WriteLine(); - Console.WriteLine(" App: " + APP_NAME); - Console.WriteLine(); - Console.WriteLine("IMPORTANT: Save this secret key securely. It will not be shown again."); - Console.WriteLine(" Secret: " + FormatSecret(secret)); - Console.Write("Enter the 6-digit code from your authenticator app: "); - string input = Console.ReadLine(); - string code = input?.Trim(); - - if (!string.IsNullOrEmpty(code) && ValidateCode(secret, code)) - { - Console.WriteLine("Verification successful!"); - Console.WriteLine(); - Console.WriteLine("Add the following to your UserConfig.xml inside section:"); - Console.WriteLine(" " + secret + ""); - Console.WriteLine("2FA is now active."); - return true; - } - - Console.WriteLine("Invalid code. Setup aborted."); - return false; - } - - private static bool RunVerification(string secret) - { - for (int attempts = 0; attempts < 3; attempts++) - { - Console.Write("Enter 2FA code: "); - string input = Console.ReadLine(); - string code = input?.Trim(); - - if (!string.IsNullOrEmpty(code) && ValidateCode(secret, code)) - { - return true; - } - - if (attempts < 2) - { - Console.WriteLine("Invalid code. Try again."); - } - } - - Console.WriteLine("Too many invalid attempts."); - return false; - } - - public static string GenerateCode(string secretBase32) - { - byte[] secret = Base32Decode(secretBase32); - long counter = DateTimeOffset.UtcNow.ToUnixTimeSeconds() / TIME_STEP; - return ComputeTotp(secret, counter); - } - - public static bool ValidateCode(string secretBase32, string code) - { - byte[] secret = Base32Decode(secretBase32); - long counter = DateTimeOffset.UtcNow.ToUnixTimeSeconds() / TIME_STEP; - - for (int i = -1; i <= 1; i++) - { - if (ComputeTotp(secret, counter + i) == code) - { - return true; - } - } - - return false; - } - - private static string ComputeTotp(byte[] secret, long counter) - { - byte[] counterBytes = BitConverter.GetBytes(counter); - if (BitConverter.IsLittleEndian) - { - Array.Reverse(counterBytes); - } - - using var hmac = new HMACSHA1(secret); - byte[] hash = hmac.ComputeHash(counterBytes); - - int offset = hash[^1] & 0xf; - int binary = ((hash[offset] & 0x7f) << 24) - | ((hash[offset + 1] & 0xff) << 16) - | ((hash[offset + 2] & 0xff) << 8) - | (hash[offset + 3] & 0xff); - - int otp = binary % MOD_VALUE; - return otp.ToString().PadLeft(CODE_DIGITS, '0'); - } - - public static string GenerateSecret() - { - byte[] random = new byte[20]; - using var rng = RandomNumberGenerator.Create(); - rng.GetBytes(random); - return Base32Encode(random); - } - - private static string Base32Encode(byte[] data) - { - const string alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; - var result = new StringBuilder(); - - int buffer = 0; - int bitsInBuffer = 0; - - foreach (byte b in data) - { - buffer = (buffer << 8) | b; - bitsInBuffer += 8; - - while (bitsInBuffer >= 5) - { - bitsInBuffer -= 5; - int index = (buffer >> bitsInBuffer) & 0x1f; - result.Append(alphabet[index]); - } - } - - if (bitsInBuffer > 0) - { - buffer <<= (5 - bitsInBuffer); - result.Append(alphabet[buffer & 0x1f]); - } - - return result.ToString(); - } - - private static byte[] Base32Decode(string input) - { - if (input == null) - { - return Array.Empty(); - } - - input = input.Trim().ToUpperInvariant(); - var bytes = new List(); - - int bitBuffer = 0; - int bitsInBuffer = 0; - - foreach (char c in input) - { - if (c >= 128) - { - continue; - } - - int value = Base32Lookup[c]; - if (value < 0) - { - continue; - } - - bitBuffer = (bitBuffer << 5) | value; - bitsInBuffer += 5; - - if (bitsInBuffer >= 8) - { - bitsInBuffer -= 8; - bytes.Add((byte)((bitBuffer >> bitsInBuffer) & 0xff)); - } - } - - return bytes.ToArray(); - } - - private static string FormatSecret(string secret) - { - var sb = new StringBuilder(); - for (int i = 0; i < secret.Length; i++) - { - if (i > 0 && i % 4 == 0) - { - sb.Append(' '); - } - - sb.Append(secret[i]); - } - return sb.ToString(); - } -} diff --git a/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs b/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs index 16619f1..6321f76 100644 --- a/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs +++ b/src/SPO2SQL.Core/SqlServer/RefreshSQLLists.cs @@ -1,3 +1,4 @@ +using SPO2SQL.Configuration; using SPO2SQL.Logging; using SPO2SQL.SharePoint; using SPO2SQL.XmlConfig; @@ -6,79 +7,59 @@ namespace SPO2SQL.SqlServer; internal sealed class RefreshSQLLists { - public static void SPOtoSQLUpdate(bool daily, CancellationToken cancellationToken = default) + public static async Task SPOtoSQLUpdateAsync( + SyncSettings settings, + bool daily, + CancellationToken cancellationToken = default) { + ArgumentNullException.ThrowIfNull(settings); Logger.Log(2, "SPOtoSQLUpdate: Starting SPO to SQL update. Daily: " + daily); try { cancellationToken.ThrowIfCancellationRequested(); - var (username, password) = ConfigurationReader.GetSharePointCredentials(); - Logger.Log(2, $"SPOtoSQLUpdate: Username retrieved successfully"); + using var user = SPOUserFactory.Create(settings); - if (!TwoFactorAuth.PerformVerification()) + var listConfigs = ConfigurationReader.GetListConfigurations(); + int failedLists = 0; + if (listConfigs != null) { - Console.WriteLine("2FA verification failed. Exiting."); - return; - } + foreach (var kvp in listConfigs) + { + cancellationToken.ThrowIfCancellationRequested(); - SPOUser user; - try - { - user = new SPOUser(username, password); - } - catch (Exception ex) - { - Console.WriteLine("SPOtoSQLUpdate: ERROR - Failed to create SPOUser."); - Console.WriteLine("Exception: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - return; - } + var listName = kvp.Key; + var config = kvp.Value; + if (config.Ignore) + { + continue; + } - using (user) - { - var listConfigs = ConfigurationReader.GetListConfigurations(); - int failedLists = 0; - if (listConfigs != null) - { - foreach (var kvp in listConfigs) + string ctxURL = config.SharepointList; + Logger.Log(1, $"SPOtoSQLUpdate: Processing list: {listName} with URL: {ctxURL}"); + try + { + await RefreshListsSQLAsync(listName, ctxURL, settings, user, daily, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + Logger.Log(2, $"SPOtoSQLUpdate: Cancelled while processing list '{listName}'."); + throw; + } + catch (Exception ex) { - cancellationToken.ThrowIfCancellationRequested(); - - var listName = kvp.Key; - var config = kvp.Value; - if (config.Ignore) - { - continue; - } - - string ctxURL = config.SharepointList; - Logger.Log(1, $"SPOtoSQLUpdate: Processing list: {listName} with URL: {ctxURL}"); - try - { - RefreshListsSQL(listName, ctxURL, user, daily, cancellationToken); - } - catch (OperationCanceledException) - { - Logger.Log(2, $"SPOtoSQLUpdate: Cancelled while processing list '{listName}'."); - throw; - } - catch (Exception ex) - { - failedLists++; - Console.WriteLine($"SPOtoSQLUpdate: ERROR - Exception while updating list '{listName}'."); - Console.WriteLine("Exception: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - } + failedLists++; + Logger.LogError($"SPOtoSQLUpdate: Exception while updating list '{listName}'.", ex); } } + } - if (failedLists > 0) - { - throw new InvalidOperationException( - $"Synchronization failed for {failedLists} configured list(s)."); - } + if (failedLists > 0) + { + throw new InvalidOperationException( + $"Synchronization failed for {failedLists} configured list(s)."); } } catch (OperationCanceledException) @@ -88,82 +69,69 @@ public static void SPOtoSQLUpdate(bool daily, CancellationToken cancellationToke } catch (Exception ex) { - Console.WriteLine("SPOtoSQLUpdate: FATAL ERROR - Exception during SPO to SQL update process."); - Console.WriteLine("Exception: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); + Logger.LogError("SPOtoSQLUpdate: FATAL ERROR - Exception during SPO to SQL update process.", ex); throw; } Logger.Log(2, "SPOtoSQLUpdate: SPO to SQL update completed."); } - public static void RefreshListsSQL(string listName, string ctxURL, SPOUser user, bool daily, CancellationToken cancellationToken = default) + public static async Task RefreshListsSQLAsync( + string listName, + string ctxURL, + SyncSettings settings, + SPOUser user, + bool daily, + CancellationToken cancellationToken = default) { + var statistics = new OperationStatistics { StartTime = DateTime.UtcNow }; + try { cancellationToken.ThrowIfCancellationRequested(); Logger.Log(1, $"RefreshListsSQL: Initializing SPO list for: {listName}"); - SPOList spoList = null; - try - { - spoList = new SPOList - { - Site = ctxURL, - SPOUser = user, - Name = listName - }; - - Logger.Log(1, "RefreshListsSQL: SPOList initialized:"); - Logger.Log(1, $" Site: {spoList.Site}"); - Logger.Log(1, $" Name: {spoList.Name}"); - } - catch (Exception ex) + using var spoList = new SPOList { - Console.WriteLine($"RefreshListsSQL: ERROR - Failed to initialize SPOList for '{listName}'."); - Console.WriteLine("Exception: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - throw; - } + Site = ctxURL, + SPOUser = user, + Name = listName, + PageSize = settings.BatchSize + }; - SQLInteraction sqlInteraction = null; - try + using var sqlInteraction = new SQLInteraction { - sqlInteraction = new SQLInteraction - { - List = spoList, - DailyMode = daily, - CommandTimeoutSeconds = 300, - CancellationToken = cancellationToken - }; - - cancellationToken.ThrowIfCancellationRequested(); - - Logger.Log(1, "RefreshListsSQL: Building SQL interaction..."); - sqlInteraction.Build(); - - if (daily) - { - Logger.Log(2, "RefreshListsSQL: Performing daily update..."); - sqlInteraction.DailyUpdate(); - } - else - { - Logger.Log(2, "RefreshListsSQL: Performing current time update..."); - sqlInteraction.CurrentTimeUpdate(); - } + List = spoList, + DailyMode = daily, + CommandTimeoutSeconds = settings.CommandTimeoutSeconds, + CancellationToken = cancellationToken + }; + + var retryPolicy = new RetryPolicy(settings.MaxRetries, settings.InitialRetryDelayMs); + await retryPolicy.ExecuteWithRetryAsync( + () => sqlInteraction.BuildAsync(), + cancellationToken, + $"Build SQL interaction for '{listName}'").ConfigureAwait(false); + + if (daily) + { + Logger.Log(2, "RefreshListsSQL: Performing daily update..."); + await sqlInteraction.DailyUpdateAsync().ConfigureAwait(false); } - catch (Exception ex) + else { - Console.WriteLine($"RefreshListsSQL: ERROR - Failed to build SQLInteraction for '{listName}'."); - Console.WriteLine("Exception: " + ex.Message); - Console.WriteLine("Stack Trace: " + ex.StackTrace); - throw; + Logger.Log(2, "RefreshListsSQL: Performing current time update..."); + await sqlInteraction.CurrentTimeUpdateAsync().ConfigureAwait(false); } - finally + + statistics.TotalItemsProcessed = spoList.ItemCollection.Count; + statistics.SuccessfulUpdates = sqlInteraction.ProcessedItems; + statistics.FailedUpdates = sqlInteraction.FailedItems; + statistics.EndTime = DateTime.UtcNow; + + if (settings.EnableMetrics) { - sqlInteraction?.Dispose(); - spoList?.Dispose(); + Logger.LogInformation($"[Metrics] List '{listName}': {statistics}"); } Logger.Log(2, $"RefreshListsSQL: Update for list '{listName}' completed."); @@ -174,8 +142,7 @@ public static void RefreshListsSQL(string listName, string ctxURL, SPOUser user, } catch (Exception ex) { - Console.WriteLine($"RefreshListsSQL: FATAL ERROR updating list '{listName}': {ex.Message}"); - Console.WriteLine("Stack Trace: " + ex.StackTrace); + Logger.LogError($"RefreshListsSQL: FATAL ERROR updating list '{listName}'.", ex); throw; } } diff --git a/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs b/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs index 4daaea4..c116a01 100644 --- a/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs +++ b/src/SPO2SQL.Core/SqlServer/SQLInteraction.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; using System.Globalization; -using System.Linq; using System.Text; using Microsoft.Data.SqlClient; using Microsoft.SharePoint.Client; @@ -13,8 +10,7 @@ namespace SPO2SQL.SqlServer; internal sealed class SQLInteraction : IDisposable { - private const string DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.fff"; - private const string FUTURE_DATE = "2100-01-01 00:00:00.000"; + private static readonly DateTime FutureSnapshot = new(2100, 1, 1, 0, 0, 0, DateTimeKind.Utc); private static readonly HashSet ValidSqlTypes = new(StringComparer.OrdinalIgnoreCase) { @@ -29,30 +25,34 @@ internal sealed class SQLInteraction : IDisposable private bool _transactionCommitted; - public SqlConnection Connection { get; private set; } - public SqlCommand Command { get; private set; } - public SqlTransaction Transaction { get; private set; } - public SPOList List { get; set; } - public string TableName { get; private set; } - public Dictionary FNDictionary { get; private set; } - public string CurrentTime { get; private set; } + public SqlConnection? Connection { get; private set; } + public SqlCommand? Command { get; private set; } + public SqlTransaction? Transaction { get; private set; } + public SPOList List { get; set; } = null!; + public string TableName { get; private set; } = string.Empty; + public Dictionary FNDictionary { get; private set; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Watermark captured before the SharePoint read. Reusing this value on the next + /// incremental run ensures items changed while the sync was running are not lost. + /// + public DateTime CurrentTime { get; private set; } public bool DailyMode { get; set; } public int CommandTimeoutSeconds { get; set; } = 300; public CancellationToken CancellationToken { get; set; } - private HashSet IgnoredColumns { get; set; } - public Dictionary ColumnMappings { get; private set; } + /// Items successfully written during the last transfer. + public int ProcessedItems { get; private set; } - private static string SanitizeSqlType(string dataType) - { - if (dataType != null && ValidSqlTypes.Contains(dataType.Trim())) - { - return dataType.Trim(); - } + /// Items that failed to be written during the last transfer. + public int FailedItems { get; private set; } - return null; - } + private HashSet? IgnoredColumns { get; set; } + public Dictionary? ColumnMappings { get; private set; } + + private static string? SanitizeSqlType(string? dataType) => + dataType != null && ValidSqlTypes.Contains(dataType.Trim()) ? dataType.Trim() : null; public void Dispose() { @@ -61,150 +61,103 @@ public void Dispose() SafeRollback(); } + Transaction?.Dispose(); Command?.Dispose(); - Connection?.Close(); Connection?.Dispose(); } - public void Build() + public async Task BuildAsync() { - Logger.Log(1, "[Build] Starting SQL build process for list: " + (List?.Name ?? "null")); + ArgumentNullException.ThrowIfNull(List); - try - { - IgnoredColumns = ConfigurationReader.GetIgnoredColumns(); + Logger.Log(1, "[Build] Starting SQL build process for list: " + List.Name); - Logger.Log(1, "[Build] Ignored columns from config: " + (IgnoredColumns == null ? "None" : string.Join(", ", IgnoredColumns))); + IgnoredColumns = ConfigurationReader.GetIgnoredColumns(); + TableName = ToPascalCase(List.Name, false); - TableName = ToPascalCase(List.Name, false); + Connection = new SqlConnection(ConfigurationReader.GetSqlConnectionString()); + Logger.Log(1, "[Build] Establishing SQL connection..."); + await Connection.OpenAsync(CancellationToken).ConfigureAwait(false); - try - { - Connection = new SqlConnection(ConfigurationReader.GetSqlConnectionString()); - Logger.Log(1, "[Build] Establishing SQL connection..."); - Connection.Open(); - } - catch (SqlException ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.Build: Database connection failed - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - throw; - } + InitializeCommand(); + await EnsureMetadataTableAsync().ConfigureAwait(false); - InitializeCommandAndTransaction(); - EnsureMetadataTable(); - CurrentTime = DateTime.Now.ToString(DATE_FORMAT); + CurrentTime = DateTime.UtcNow; - if (DailyMode && TableExists(TableName)) + if (DailyMode && await TableExistsAsync(TableName).ConfigureAwait(false)) + { + DateTime? lastSync = await GetLastSyncDateAsync().ConfigureAwait(false); + if (lastSync.HasValue) { - string lastSync = GetLastSyncDate(); - if (lastSync != null) - { - Logger.Log(1, "[Build] Applying incremental filter: Modified >= " + lastSync); - List.CAMLQuery = CamlQueryBuilder.BuildDateRangeQuery("Modified", - DateTime.Parse(lastSync), DateTime.Now); - } + Logger.Log(1, "[Build] Applying incremental filter: Modified >= " + lastSync.Value.ToString("O", CultureInfo.InvariantCulture)); + List.CAMLQuery = CamlQueryBuilder.BuildDateRangeQuery("Modified", lastSync.Value, CurrentTime); } + } - try - { - Logger.Log(1, "[Build] Initializing SharePoint list structure..."); - List.Build(); - } - catch (Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.Build: SharePoint list initialization failed - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - throw; - } + Logger.Log(1, "[Build] Initializing SharePoint list structure..."); + await List.BuildAsync(CancellationToken).ConfigureAwait(false); - try - { - FNDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); - Logger.Log(1, "[Build] Building field dictionary..."); - BuildDictionary(); - } - catch (Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.Build: Field dictionary creation failed - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - throw; - } + FNDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + BuildDictionary(); - try - { - if (!TableExists(TableName)) - { - Logger.Log(1, "[Build] Creating new table: " + TableName); - CreateTable(); - } - else - { - Logger.Log(1, "[Build] Updating existing table: " + TableName); - UpdateTableDesign(); - } - } - catch (Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.Build: Table structure operation failed - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - throw; - } + if (!await TableExistsAsync(TableName).ConfigureAwait(false)) + { + Logger.Log(1, "[Build] Creating new table: " + TableName); + await CreateTableAsync().ConfigureAwait(false); } - catch (Exception ex) + else { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [FATAL] SQLInteraction.Build: Critical failure during build process - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - throw; + Logger.Log(1, "[Build] Updating existing table: " + TableName); + await UpdateTableDesignAsync().ConfigureAwait(false); } } - public void DailyUpdate() + public async Task DailyUpdateAsync() { try { Logger.Log(2, "[DailyUpdate] Starting daily update for table " + TableName); - Command.CommandText = $"DELETE FROM [{TableName.Replace("]", "]]")}] WHERE Snapshot = @FutureDate"; - Command.Parameters.Clear(); - Command.Parameters.AddWithValue("@FutureDate", FUTURE_DATE); - Command.ExecuteNonQuery(); + BeginTransaction(); - TransferData(FUTURE_DATE); + Command!.CommandText = $"DELETE FROM [{TableName.Replace("]", "]]")}] WHERE Snapshot = @FutureDate"; + Command.Parameters.Clear(); + Command.Parameters.AddWithValue("@FutureDate", FutureSnapshot); + await Command.ExecuteNonQueryAsync(CancellationToken).ConfigureAwait(false); - UpdateMetadata(); + await TransferDataAsync(FutureSnapshot).ConfigureAwait(false); + await UpdateMetadataAsync().ConfigureAwait(false); - Transaction.Commit(); + await Transaction!.CommitAsync(CancellationToken).ConfigureAwait(false); _transactionCommitted = true; Logger.Log(2, "[DailyUpdate] Transaction committed successfully"); } catch (Exception ex) { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [FATAL] SQLInteraction.DailyUpdate: Critical failure during daily update - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); + Logger.LogError("[DailyUpdate] Critical failure during daily update", ex); SafeRollback(); throw; } } - public void CurrentTimeUpdate() + public async Task CurrentTimeUpdateAsync() { try { Logger.Log(2, "[CurrentTimeUpdate] Starting current-time update for " + TableName); - TransferData(CurrentTime); + BeginTransaction(); - UpdateMetadata(); + await TransferDataAsync(CurrentTime).ConfigureAwait(false); + await UpdateMetadataAsync().ConfigureAwait(false); - Transaction.Commit(); + await Transaction!.CommitAsync(CancellationToken).ConfigureAwait(false); _transactionCommitted = true; Logger.Log(2, "[CurrentTimeUpdate] Transaction committed successfully"); } catch (Exception ex) { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [FATAL] SQLInteraction.CurrentTimeUpdate: Critical failure during current-time update - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); + Logger.LogError("[CurrentTimeUpdate] Critical failure during current-time update", ex); SafeRollback(); throw; } @@ -222,168 +175,142 @@ private void BuildDictionary() foreach (Field field in List.Fields) { - if (field.TypeAsString != "Computed") + if (field.TypeAsString == "Computed") { - try - { - string columnName = field.InternalName; + continue; + } + + try + { + string columnName = field.InternalName; - if (ColumnMappings != null) + if (ColumnMappings != null) + { + if (ColumnMappings.TryGetValue(columnName, out var mapping)) { - if (ColumnMappings.TryGetValue(columnName, out var mapping)) - { - if (mapping.Ignore) - { - ignoredFields++; - Logger.Log(1, "[BuildDictionary] Ignored field (by mapping) " + columnName); - continue; - } - - string destinationName = mapping.Destination; - FNDictionary.Add(GetKeyName(destinationName, 1), field); - processedFields++; - Logger.Log(1, "[BuildDictionary] Added mapped field " + columnName + " -> " + destinationName); - } - else + if (mapping.Ignore) { - skippedFields++; - Logger.Log(1, "[BuildDictionary] Skipped field (not mapped) " + columnName); + ignoredFields++; + continue; } - } - else if (IgnoredColumns != null && IgnoredColumns.Contains(columnName)) - { - ignoredFields++; - Logger.Log(1, "[BuildDictionary] Ignored field (global): " + columnName); + + string destinationName = mapping.Destination; + FNDictionary.Add(GetKeyName(destinationName, 1), field); + processedFields++; } else { - FNDictionary.Add(GetKeyName(columnName, 1), field); - processedFields++; - Logger.Log(1, "[BuildDictionary] Added field: " + columnName); + skippedFields++; } } - catch (Exception ex) + else if (IgnoredColumns != null && IgnoredColumns.Contains(columnName)) + { + ignoredFields++; + } + else { - skippedFields++; - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.BuildDictionary: Failed to process field: {field.Title} - {ex.Message}"); + FNDictionary.Add(GetKeyName(columnName, 1), field); + processedFields++; } } + catch (Exception ex) + { + skippedFields++; + Logger.LogError($"Failed to process field '{field.Title}'", ex); + } } - Logger.Log(1, "[BuildDictionary] Dictionary built. Processed: " + processedFields + ", Skipped: " + skippedFields + ", Ignored: " + ignoredFields); + Logger.Log(1, $"[BuildDictionary] Dictionary built. Processed: {processedFields}, Skipped: {skippedFields}, Ignored: {ignoredFields}"); } - private bool TableExists(string listName) + private async Task TableExistsAsync(string tableName) { - try - { - Command.CommandText = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = @TableName"; - Command.Parameters.Clear(); - Command.Parameters.AddWithValue("@TableName", listName); - bool exists = (int)Command.ExecuteScalar() != 0; - Logger.Log(1, $"[TableExists] Table '{listName}' exists: {exists}"); - return exists; - } - catch (Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.TableExists: Failed to check existence of table '{listName}' - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - throw; - } + Command!.CommandText = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = @TableName"; + Command.Parameters.Clear(); + Command.Parameters.AddWithValue("@TableName", tableName); + bool exists = (int)(await Command.ExecuteScalarAsync(CancellationToken).ConfigureAwait(false))! != 0; + Logger.Log(1, $"[TableExists] Table '{tableName}' exists: {exists}"); + return exists; } - private void CreateTable() + private async Task CreateTableAsync() { - Logger.Log(1, "[CreateTable] Creating new table: " + TableName); - var columns = new List { "[Snapshot] datetime NULL" }; + var columns = new List { "[Snapshot] datetime2 NULL" }; foreach (var fn in FNDictionary) { - string sqlType = null; - if (ColumnMappings != null && ColumnMappings.TryGetValue(fn.Value.InternalName, out var mapping) && !string.IsNullOrEmpty(mapping.DataType)) - { - sqlType = SanitizeSqlType(mapping.DataType) ?? SQLFieldType(fn.Value); - } - else - { - sqlType = SQLFieldType(fn.Value); - } - + string? sqlType = ResolveSqlType(fn.Value); if (sqlType != null) { columns.Add($"[{fn.Key.Replace("]", "]]")}] {sqlType} NULL"); } } - Command.CommandText = + Command!.CommandText = $"CREATE TABLE [{TableName.Replace("]", "]]")}] ({string.Join(", ", columns)})"; - try - { - Command.ExecuteNonQuery(); - Logger.Log(1, "[CreateTable] Successfully created table: " + TableName); - } - catch (Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.CreateTable: Failed to create table: {TableName} - {ex.Message}"); - throw; - } + + await Command.ExecuteNonQueryAsync(CancellationToken).ConfigureAwait(false); + Logger.Log(1, "[CreateTable] Successfully created table: " + TableName); } - private void UpdateTableDesign() + private async Task UpdateTableDesignAsync() { - Logger.Log(1, "[UpdateTableDesign] Updating design for table: " + TableName); int updatedColumns = 0; int failedColumns = 0; foreach (var fn in FNDictionary) { - try + string? sqlType = ResolveSqlType(fn.Value); + if (sqlType == null) { - string sqlType = null; - if (ColumnMappings != null && ColumnMappings.TryGetValue(fn.Value.InternalName, out var mapping) && !string.IsNullOrEmpty(mapping.DataType)) - { - sqlType = SanitizeSqlType(mapping.DataType) ?? SQLFieldType(fn.Value); - } - else - { - sqlType = SQLFieldType(fn.Value); - } - - if (sqlType == null) - { - failedColumns++; - continue; - } + failedColumns++; + continue; + } + try + { string colName = fn.Key; - Command.CommandText = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = @TableName AND COLUMN_NAME = @ColName"; + Command!.CommandText = "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = @TableName AND COLUMN_NAME = @ColName"; Command.Parameters.Clear(); Command.Parameters.AddWithValue("@TableName", TableName); Command.Parameters.AddWithValue("@ColName", colName); - if ((int)Command.ExecuteScalar() == 0) + if ((int)(await Command.ExecuteScalarAsync(CancellationToken).ConfigureAwait(false))! == 0) { string safeTable = $"[{TableName.Replace("]", "]]")}]"; string safeCol = $"[{colName.Replace("]", "]]")}]"; Command.CommandText = $"ALTER TABLE {safeTable} ADD {safeCol} {sqlType} NULL"; Command.Parameters.Clear(); - Command.ExecuteNonQuery(); + await Command.ExecuteNonQueryAsync(CancellationToken).ConfigureAwait(false); updatedColumns++; } } catch (Exception ex) { failedColumns++; - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.UpdateTableDesign: Failed to update column: {fn.Key} - {ex.Message}"); + Logger.LogError($"Failed to update column '{fn.Key}'", ex); } } - Logger.Log(1, "[UpdateTableDesign] Design update completed. Updated: " + updatedColumns + ", Failed: " + failedColumns); + Logger.Log(1, $"[UpdateTableDesign] Design update completed. Updated: {updatedColumns}, Failed: {failedColumns}"); } - private void TransferData(string snapDate) + private string? ResolveSqlType(Field field) + { + if (ColumnMappings != null && + ColumnMappings.TryGetValue(field.InternalName, out var mapping) && + !string.IsNullOrEmpty(mapping.DataType)) + { + return SanitizeSqlType(mapping.DataType) ?? SQLFieldType(field); + } + + return SQLFieldType(field); + } + + private async Task TransferDataAsync(DateTime snapDate) { Logger.Log(1, "[TransferData] Beginning data transfer for snapshot: " + snapDate); + string sqlColNames = GetSQLColNames(); int processedItems = 0; int failedItems = 0; @@ -405,84 +332,40 @@ private void TransferData(string snapDate) try { - Command.CommandText = insertBase; + Command!.CommandText = insertBase; Command.Parameters.Clear(); Command.Parameters.AddWithValue("@Snapshot", snapDate); - int idx = 0; - foreach (Field field in fieldList) + for (int idx = 0; idx < fieldList.Count; idx++) { + Field field = fieldList[idx]; object obj; try - { obj = listItem[field.InternalName]; } + { + obj = listItem[field.InternalName]; + } catch (Exception ex) { throw new InvalidOperationException( $"Failed to read field '{field.InternalName}' for SharePoint item.", ex); } - string paramName = $"@F{idx}"; - if (obj != null) - { - if (obj is FieldLookupValue lookup) - { - Command.Parameters.AddWithValue(paramName, lookup.LookupId); - } - else if (obj is FieldUserValue user) - { - Command.Parameters.AddWithValue(paramName, user.LookupId); - } - else if (obj is FieldUrlValue url) - { - Command.Parameters.AddWithValue(paramName, (object)url.Url ?? DBNull.Value); - } - else if (obj is ContentTypeId ctId) - { - Command.Parameters.AddWithValue(paramName, ctId.StringValue); - } - else if (obj is DateTime dt) - { - Command.Parameters.AddWithValue(paramName, dt); - } - else if (obj is FieldLookupValue[] lookups) - { - Command.Parameters.AddWithValue(paramName, lookups.Length > 0 ? (object)string.Join(";", lookups.Select(l => l.LookupId)) : DBNull.Value); - } - else if (obj is FieldUserValue[] users) - { - Command.Parameters.AddWithValue(paramName, users.Length > 0 ? (object)string.Join(";", users.Select(u => u.LookupId)) : DBNull.Value); - } - else - { - Command.Parameters.AddWithValue(paramName, obj); - } - } - else - { - Command.Parameters.AddWithValue(paramName, DBNull.Value); - } - - idx++; + Command.Parameters.AddWithValue($"@F{idx}", ConvertFieldValue(obj)); } - try - { - Command.ExecuteNonQuery(); - processedItems++; - } - catch (Exception ex) - { - failedItems++; - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.TransferData: Failed to insert item {processedItems + failedItems} - {ex.Message}"); - } + await Command.ExecuteNonQueryAsync(CancellationToken).ConfigureAwait(false); + processedItems++; } catch (Exception ex) { failedItems++; - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.TransferData: Failed to process item {processedItems + failedItems} - {ex.Message}"); + Logger.LogError($"Failed to insert item {processedItems + failedItems}", ex); } } + ProcessedItems = processedItems; + FailedItems = failedItems; + Logger.Log(1, $"[TransferData] Transfer completed. Processed: {processedItems}, Failed: {failedItems}"); if (failedItems > 0) @@ -492,9 +375,26 @@ private void TransferData(string snapDate) } } - private void EnsureMetadataTable() + private static object ConvertFieldValue(object obj) => obj switch { - Command.CommandText = @" + FieldUserValue user => user.LookupId, + FieldLookupValue lookup => lookup.LookupId, + FieldUrlValue url => (object?)url.Url ?? DBNull.Value, + ContentTypeId ctId => ctId.StringValue, + DateTime dt => dt, + FieldUserValue[] users => users.Length > 0 + ? string.Join(";", users.Select(u => u.LookupId)) + : DBNull.Value, + FieldLookupValue[] lookups => lookups.Length > 0 + ? string.Join(";", lookups.Select(l => l.LookupId)) + : DBNull.Value, + null => DBNull.Value, + _ => obj + }; + + private async Task EnsureMetadataTableAsync() + { + Command!.CommandText = @" IF OBJECT_ID(N'[Metadata]', N'U') IS NULL BEGIN CREATE TABLE [Metadata] @@ -504,48 +404,27 @@ [LastRefreshDate] datetime2 NULL ); END"; Command.Parameters.Clear(); - Command.ExecuteNonQuery(); + await Command.ExecuteNonQueryAsync(CancellationToken).ConfigureAwait(false); } - private static string SQLFieldType(Field field) + private static string? SQLFieldType(Field field) => field.TypeAsString switch { - switch (field.TypeAsString) - { - case "Attachments": - case "Boolean": - return "[bit]"; - case "Calculated": - return "[sql_variant]"; - case "Choice": - case "File": - case "LookupMulti": - case "Note": - case "Text": - case "URL": - case "UserMulti": - return "[nvarchar](MAX)"; - case "ContentTypeId": - return "[nvarchar](MAX)"; - case "Counter": - case "Integer": - case "ModStat": - case "User": - return "[int]"; - case "Currency": - case "Number": - return "[float]"; - case "DateTime": - return "[datetime]"; - case "Guid": - return "[uniqueidentifier]"; - case "Lookup": - return field.FromBaseType - ? "[nvarchar](MAX)" - : "[int]"; - default: - Logger.Log(1, $"[SQLFieldType] Unknown field type encountered - Field: {field.Title}, Type: {field.TypeAsString}"); - return null; - } + "Attachments" or "Boolean" => "[bit]", + "Calculated" => "[sql_variant]", + "Choice" or "File" or "LookupMulti" or "Note" or "Text" or "URL" or + "UserMulti" or "ContentTypeId" => "[nvarchar](MAX)", + "Counter" or "Integer" or "ModStat" or "User" => "[int]", + "Currency" or "Number" => "[float]", + "DateTime" => "[datetime]", + "Guid" => "[uniqueidentifier]", + "Lookup" => field.FromBaseType ? "[nvarchar](MAX)" : "[int]", + _ => LogUnknownFieldType(field) + }; + + private static string? LogUnknownFieldType(Field field) + { + Logger.Log(1, $"[SQLFieldType] Unknown field type encountered - Field: {field.Title}, Type: {field.TypeAsString}"); + return null; } private string GetSQLColNames() @@ -559,99 +438,83 @@ private string GetSQLColNames() return $"({string.Join(", ", columns)})"; } - private string GetLastSyncDate() + private async Task GetLastSyncDateAsync() { - try - { - Command.CommandText = "SELECT LastRefreshDate FROM Metadata WHERE TableName = @TableName"; - Command.Parameters.Clear(); - Command.Parameters.AddWithValue("@TableName", TableName); - object result = Command.ExecuteScalar(); - - if (result == null || result == DBNull.Value) - { - return null; - } - - if (result is DateTime dt) - { - return dt.ToString(DATE_FORMAT); - } + Command!.CommandText = "SELECT LastRefreshDate FROM Metadata WHERE TableName = @TableName"; + Command.Parameters.Clear(); + Command.Parameters.AddWithValue("@TableName", TableName); + object? result = await Command.ExecuteScalarAsync(CancellationToken).ConfigureAwait(false); - string str = result.ToString(); - return string.IsNullOrEmpty(str) ? null : str; + if (result == null || result == DBNull.Value) + { + return null; } - catch (Exception ex) + + if (result is DateTime dt) { - throw new InvalidOperationException( - $"Failed to read synchronization metadata for table '{TableName}'.", ex); + return DateTime.SpecifyKind(dt, DateTimeKind.Utc); } + + return DateTime.TryParse( + result.ToString(), + CultureInfo.InvariantCulture, + DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, + out DateTime parsed) + ? parsed + : null; } - private void UpdateMetadata() + private async Task UpdateMetadataAsync() { Logger.Log(1, $"[UpdateMetadata] Updating metadata for table: {TableName}"); - try - { - Command.CommandText = "DELETE FROM Metadata WHERE TableName = @TableName"; - Command.Parameters.Clear(); - Command.Parameters.AddWithValue("@TableName", TableName); - Command.ExecuteNonQuery(); - Command.CommandText = "INSERT INTO Metadata (TableName, LastRefreshDate) VALUES (@TableName, @CurrentTime)"; - Command.Parameters.Clear(); - Command.Parameters.AddWithValue("@TableName", TableName); - Command.Parameters.AddWithValue("@CurrentTime", CurrentTime); - Command.ExecuteNonQuery(); - Logger.Log(1, $"[UpdateMetadata] Metadata updated successfully"); - } - catch (Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.UpdateMetadata: Failed to update metadata - {ex.Message}"); - throw; - } + + Command!.CommandText = "DELETE FROM Metadata WHERE TableName = @TableName"; + Command.Parameters.Clear(); + Command.Parameters.AddWithValue("@TableName", TableName); + await Command.ExecuteNonQueryAsync(CancellationToken).ConfigureAwait(false); + + Command.CommandText = "INSERT INTO Metadata (TableName, LastRefreshDate) VALUES (@TableName, @CurrentTime)"; + Command.Parameters.Clear(); + Command.Parameters.AddWithValue("@TableName", TableName); + Command.Parameters.AddWithValue("@CurrentTime", CurrentTime); + await Command.ExecuteNonQueryAsync(CancellationToken).ConfigureAwait(false); + + Logger.Log(1, "[UpdateMetadata] Metadata updated successfully"); } private string GetKeyName(string key, int i = 1) { string testKey = i == 1 ? key : $"{key}{i}"; - return FNDictionary.ContainsKey(testKey) - ? GetKeyName(key, i + 1) - : testKey; + return FNDictionary.ContainsKey(testKey) ? GetKeyName(key, i + 1) : testKey; } - private string GetActualColName(Field field) + private void InitializeCommand() { - string name = ColNameConvetions(field); - int count = 0; - - foreach (Field f in List.Fields) - { - if (f.TypeAsString != "Computed" && - name.Equals(ColNameConvetions(f), StringComparison.OrdinalIgnoreCase)) - { - count++; - } - } - - return count > 1 - ? ToPascalCase(field.InternalName, true) - : name; + Command = Connection!.CreateCommand(); + Command.CommandTimeout = CommandTimeoutSeconds; } - private static string ColNameConvetions(Field field) + + /// + /// Starts the write transaction only after all SharePoint data has been read and + /// transformed, so no database locks are held during network calls. + /// + private void BeginTransaction() { - var sb = new StringBuilder(ToPascalCase(field.Title, false)); - string type = field.TypeAsString; + Transaction = Connection!.BeginTransaction($"{TableName} TXN"); + Command!.Transaction = Transaction; + } - if (type == "Choice") + private void SafeRollback() + { + try { - sb.Append("Value"); + Transaction?.Rollback(); + Logger.Log(1, "[SafeRollback] Transaction rolled back successfully"); } - else if (type == "User" || (type == "Lookup" && !field.FromBaseType)) + catch (Exception ex) { - sb.Append("Id"); + Logger.LogError("[SafeRollback] Failed to rollback transaction", ex); } - - return sb.ToString(); } private static string ToPascalCase(string text, bool internalName) @@ -661,7 +524,7 @@ private static string ToPascalCase(string text, bool internalName) return string.Empty; } - if (internalName && text.StartsWith("_")) + if (internalName && text.StartsWith('_')) { text += "IN"; } @@ -678,65 +541,4 @@ private static string ToPascalCase(string text, bool internalName) .Replace("X0020", string.Empty) .Replace("X003a", string.Empty); } - - private void InitializeCommandAndTransaction() - { - Command = Connection.CreateCommand(); - Command.CommandTimeout = CommandTimeoutSeconds; - Transaction = Connection.BeginTransaction($"{TableName} TXN"); - Command.Connection = Connection; - Command.Transaction = Transaction; - } - - private void SafeRollback() - { - try - { - Transaction?.Rollback(); - Logger.Log(1, "[SafeRollback] Transaction rolled back successfully"); - } - catch (Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.SafeRollback: Failed to rollback transaction - {ex.Message}"); - } - } - - #region Logging Methods - - private static void LogInfo(string method, string message) - { - Logger.Log(1, $"SQLInteraction.{method}: {message}"); - } - - private static void LogError(string method, string message, Exception ex, bool includeStack = false) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [ERROR] SQLInteraction.{method}: {message} - {ex.Message}"); - if (includeStack) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - } - } - - private static void LogWarning(string method, string message) - { - Logger.Log(1, $"SQLInteraction.{method}: {message}"); - } - - private static void LogDebug(string method, string message) - { - Logger.Log(1, $"SQLInteraction.{method}: {message}"); - } - - private static void LogFatal(string method, string message, Exception ex) - { - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [FATAL] SQLInteraction.{method}: {message} - {ex.Message}"); - Console.WriteLine($"[{DateTime.Now:HH:mm:ss}] [DEBUG] Stack trace: {ex.StackTrace}"); - } - - private static void LogVerbose(string method, string message) - { - Logger.Log(1, $"SQLInteraction.{method}: {message}"); - } - - #endregion } diff --git a/src/SPO2SQL.Core/XmlConfig/ConfigHelper.cs b/src/SPO2SQL.Core/XmlConfig/ConfigHelper.cs index 873aa71..d5890f1 100644 --- a/src/SPO2SQL.Core/XmlConfig/ConfigHelper.cs +++ b/src/SPO2SQL.Core/XmlConfig/ConfigHelper.cs @@ -1,25 +1,26 @@ -using System; -using System.Collections.Generic; -using System.Linq; using System.Xml; using SPO2SQL.Logging; using SPO2SQL.Security; namespace SPO2SQL.XmlConfig; +/// +/// Reads the legacy XML file used for list mappings. Credentials are intentionally +/// not read from here; the credential provider chain is the only secret source. +/// public static class ConfigurationReader { private static string _configPath = "SPO_to_SQL_config.xml"; - private static string _sharePointBaseUrlOverride; - private static readonly object _lock = new object(); - private static XmlDocument _xmlDoc; + private static string? _sharePointBaseUrlOverride; + private static readonly object Lock = new(); + private static XmlDocument? _xmlDoc; public static void SetConfigPath(string path) { ArgumentException.ThrowIfNullOrWhiteSpace(path); Logger.LogDebug($"Setting configuration path to: {path}"); - lock (_lock) + lock (Lock) { _configPath = path; _xmlDoc = null; @@ -36,25 +37,24 @@ public static void SetSharePointBaseUrl(string baseUrl) throw new ArgumentException("SharePoint base URL must be an absolute HTTP or HTTPS URL.", nameof(baseUrl)); } - lock (_lock) + lock (Lock) { _sharePointBaseUrlOverride = uri.GetLeftPart(UriPartial.Authority).TrimEnd('/'); } } - private static bool ParseBoolean(string value, bool defaultValue) - { - return bool.TryParse(value, out var result) ? result : defaultValue; - } + private static bool ParseBoolean(string? value, bool defaultValue) => + bool.TryParse(value, out bool result) ? result : defaultValue; - private static void LoadConfig() + private static XmlDocument LoadConfig() { - if (_xmlDoc != null) + XmlDocument? document = Volatile.Read(ref _xmlDoc); + if (document != null) { - return; + return document; } - lock (_lock) + lock (Lock) { if (_xmlDoc == null) { @@ -66,23 +66,26 @@ private static void LoadConfig() XmlResolver = null }; - _xmlDoc = new XmlDocument { XmlResolver = null }; + var loaded = new XmlDocument { XmlResolver = null }; using var reader = XmlReader.Create(_configPath, settings); - _xmlDoc.Load(reader); + loaded.Load(reader); + _xmlDoc = loaded; Logger.LogDebug($"Configuration file loaded successfully from {_configPath}"); } catch (Exception ex) { Logger.LogError($"Failed to load configuration file from {_configPath}", ex); - throw new Exception($"Error loading configuration file: {ex.Message}", ex); + throw new InvalidOperationException($"Error loading configuration file: {ex.Message}", ex); } } + + return _xmlDoc; } } - public static Dictionary GetSelectedColumns(string listName = null) + public static Dictionary? GetSelectedColumns(string? listName = null) { - LoadConfig(); + XmlDocument document = LoadConfig(); try { @@ -96,6 +99,7 @@ public static Dictionary GetSelectedColumns(string listNa Logger.Log(2, "List " + listName + " is configured to be ignored."); return null; } + if (listConfig.Columns != null) { Logger.Log(1, "Using specific configuration for list: " + listName); @@ -104,7 +108,7 @@ public static Dictionary GetSelectedColumns(string listNa } } - var columnNodes = _xmlDoc.SelectNodes("//Configuration/ReplicationConfiguration/SelectColumns/column"); + var columnNodes = document.SelectNodes("//Configuration/ReplicationConfiguration/SelectColumns/column"); if (columnNodes == null || columnNodes.Count == 0) { @@ -116,40 +120,42 @@ public static Dictionary GetSelectedColumns(string listNa foreach (XmlNode node in columnNodes) { - var sourceAttr = node.Attributes["source"]; - var destAttr = node.Attributes["destination"]; - var ignoreAttr = node.Attributes["ignore"]; + var sourceAttr = node.Attributes?["source"]; + if (sourceAttr == null) + { + continue; + } - if (sourceAttr != null) + var destAttr = node.Attributes?["destination"]; + var ignoreAttr = node.Attributes?["ignore"]; + + var mapping = new ColumnMapping { - var mapping = new ColumnMapping - { - Source = sourceAttr.Value, - Destination = destAttr?.Value ?? sourceAttr.Value, - Ignore = ignoreAttr != null && bool.Parse(ignoreAttr.Value) - }; + Source = sourceAttr.Value, + Destination = destAttr?.Value ?? sourceAttr.Value, + Ignore = ParseBoolean(ignoreAttr?.Value, false) + }; - columnMappings[mapping.Source] = mapping; - Logger.Log(1, "Added column mapping: " + mapping.Source + " -> " + mapping.Destination + " (Ignore: " + mapping.Ignore + ")"); - } + columnMappings[mapping.Source] = mapping; + Logger.Log(1, $"Added column mapping: {mapping.Source} -> {mapping.Destination} (Ignore: {mapping.Ignore})"); } return columnMappings.Count > 0 ? columnMappings : null; } - catch (Exception ex) + catch (Exception ex) when (ex is not InvalidOperationException) { - Console.WriteLine("Error reading selected columns: " + ex.Message); - throw new Exception("Error reading selected columns configuration: " + ex.Message); + Logger.LogError("Error reading selected columns configuration", ex); + throw new InvalidOperationException("Error reading selected columns configuration.", ex); } } - public static HashSet GetIgnoredColumns() + public static HashSet? GetIgnoredColumns() { - LoadConfig(); + XmlDocument document = LoadConfig(); try { - var columnNodes = _xmlDoc.SelectNodes("//Configuration/ReplicationConfiguration/GlobalIgnore/Column"); + var columnNodes = document.SelectNodes("//Configuration/ReplicationConfiguration/GlobalIgnore/Column"); if (columnNodes == null || columnNodes.Count == 0) { @@ -165,7 +171,6 @@ public static HashSet GetIgnoredColumns() { string columnName = node.InnerText.Trim(); ignoredColumns.Add(columnName); - Logger.Log(1, "Added ignored column: " + columnName); } } @@ -178,155 +183,22 @@ public static HashSet GetIgnoredColumns() Logger.Log(2, "Total ignored columns: " + ignoredColumns.Count); return ignoredColumns; } - catch (Exception ex) + catch (Exception ex) when (ex is not InvalidOperationException) { - Console.WriteLine("Error reading ignored columns: " + ex.Message); - throw new Exception("Error reading ignored columns configuration: " + ex.Message); + Logger.LogError("Error reading ignored columns configuration", ex); + throw new InvalidOperationException("Error reading ignored columns configuration.", ex); } } - public static (string Username, string Password) GetSharePointCredentials() - { - try - { - return CredentialManager.GetSharePointCredentials(); - } - catch - { - } - - LoadConfig(); - try - { - var spNode = _xmlDoc.SelectSingleNode("//Configuration/SharePoint"); - if (spNode == null) - { - throw new InvalidOperationException("SharePoint configuration section not found."); - } - - var usernameNode = spNode.SelectSingleNode("Username"); - var passwordNode = spNode.SelectSingleNode("Password"); - - if (usernameNode == null) - { - throw new InvalidOperationException("Username element not found in SharePoint configuration."); - } - - if (passwordNode == null) - { - throw new InvalidOperationException("Password element not found in SharePoint configuration."); - } - - var username = usernameNode.InnerText.Trim(); - var password = passwordNode.InnerText.Trim(); - - if (string.IsNullOrEmpty(username)) - { - throw new InvalidOperationException("Username cannot be empty."); - } + public static (string Username, string Password) GetSharePointCredentials() => + CredentialManager.GetSharePointCredentials(); - if (string.IsNullOrEmpty(password)) - { - throw new InvalidOperationException("Password cannot be empty."); - } + public static string GetSqlConnectionString() => + CredentialManager.GetSqlConnectionString(); - Logger.Log(2, "SharePoint credentials retrieved successfully from XML config."); - return (username, password); - } - catch (InvalidOperationException) - { - throw; - } - catch (Exception ex) - { - Console.WriteLine("Error retrieving SharePoint credentials: " + ex.Message); - throw new InvalidOperationException("Failed to retrieve SharePoint credentials.", ex); - } - } - - public static string GetSqlConnectionString() + public static string? GetSharePointBaseUrl() { - try - { - return CredentialManager.GetSqlConnectionString(); - } - catch - { - } - - LoadConfig(); - try - { - var connNode = _xmlDoc.SelectSingleNode("//Configuration/SQL/ConnectionString"); - if (connNode == null) - { - throw new InvalidOperationException("SQL ConnectionString element not found in configuration."); - } - - string connectionString = connNode.InnerText.Trim(); - if (string.IsNullOrEmpty(connectionString)) - { - throw new InvalidOperationException("SQL connection string cannot be empty."); - } - - Logger.Log(2, "SQL connection string retrieved successfully from XML config."); - return connectionString; - } - catch (InvalidOperationException) - { - throw; - } - catch (Exception ex) - { - Console.WriteLine("Error retrieving SQL connection string: " + ex.Message); - throw new InvalidOperationException("Failed to retrieve SQL connection string.", ex); - } - } - - public static bool IsTwoFactorEnabled() - { - LoadConfig(); - - try - { - var secNode = _xmlDoc.SelectSingleNode("//Configuration/Security"); - if (secNode == null) - { - return false; - } - - var enabledNode = secNode.SelectSingleNode("TwoFactorEnabled"); - if (enabledNode == null) - { - return false; - } - - return bool.TryParse(enabledNode.InnerText.Trim(), out bool result) && result; - } - catch - { - return false; - } - } - - public static string GetTwoFactorSecret() - { - LoadConfig(); - - try - { - var secNode = _xmlDoc.SelectSingleNode("//Configuration/Security"); - return secNode?.SelectSingleNode("TwoFactorSecret")?.InnerText.Trim() ?? string.Empty; - } - catch - { - return string.Empty; - } - } - - public static string GetSharePointBaseUrl() - { - lock (_lock) + lock (Lock) { if (!string.IsNullOrEmpty(_sharePointBaseUrlOverride)) { @@ -334,17 +206,11 @@ public static string GetSharePointBaseUrl() } } - LoadConfig(); - try { - var spNode = _xmlDoc.SelectSingleNode("//Configuration/SharePoint"); - if (spNode == null) - { - return null; - } - - var baseUrlNode = spNode.SelectSingleNode("BaseUrl"); + XmlDocument document = LoadConfig(); + var spNode = document.SelectSingleNode("//Configuration/SharePoint"); + var baseUrlNode = spNode?.SelectSingleNode("BaseUrl"); if (baseUrlNode == null) { Logger.LogDebug("SharePoint BaseUrl not configured in XML, using default."); @@ -370,11 +236,11 @@ public static string GetSharePointBaseUrl() public static IEnumerable GetSharePointListConfigurations() { - LoadConfig(); + XmlDocument document = LoadConfig(); try { - var listNodes = _xmlDoc.SelectNodes("//Configuration/ReplicationConfiguration/SharePointLists/List"); + var listNodes = document.SelectNodes("//Configuration/ReplicationConfiguration/SharePointLists/List"); if (listNodes == null || listNodes.Count == 0) { Logger.Log(2, "No SharePoint list configurations found."); @@ -403,10 +269,10 @@ public static IEnumerable GetSharePointListConfigurations( return configurations; } - catch (Exception ex) + catch (Exception ex) when (ex is not InvalidOperationException) { - Console.WriteLine("Error reading SharePoint list configurations: " + ex.Message); - throw new Exception("Failed to read SharePoint list configurations.", ex); + Logger.LogError("Error reading SharePoint list configurations", ex); + throw new InvalidOperationException("Failed to read SharePoint list configurations.", ex); } } @@ -418,10 +284,8 @@ private static string GetNodeValue(XmlNode parentNode, string nodeName) private static bool GetNodeValueBool(XmlNode parentNode, string nodeName, bool defaultValue) { - var value = GetNodeValue(parentNode, nodeName); - return !string.IsNullOrEmpty(value) && bool.TryParse(value, out bool result) - ? result - : defaultValue; + string value = GetNodeValue(parentNode, nodeName); + return bool.TryParse(value, out bool result) ? result : defaultValue; } private static bool IsValidListConfig(SharePointListConfig config) @@ -447,13 +311,13 @@ private static bool IsValidListConfig(SharePointListConfig config) return true; } - public static Dictionary GetListConfigurations() + public static Dictionary? GetListConfigurations() { - LoadConfig(); + XmlDocument document = LoadConfig(); try { - var listNodes = _xmlDoc.SelectNodes("//Configuration/ReplicationConfiguration/Lists/List"); + var listNodes = document.SelectNodes("//Configuration/ReplicationConfiguration/Lists/List"); if (listNodes == null || listNodes.Count == 0) { return null; @@ -463,34 +327,36 @@ public static Dictionary GetListConfigurations() foreach (XmlNode listNode in listNodes) { - var nameAttr = listNode.Attributes["name"]; - var contextAttr = listNode.Attributes["sharepointlist"]; - var ignoreAttr = listNode.Attributes["ignore"]; + var nameAttr = listNode.Attributes?["name"]; + if (nameAttr == null) + { + continue; + } - if (nameAttr != null) + var contextAttr = listNode.Attributes?["sharepointlist"]; + var ignoreAttr = listNode.Attributes?["ignore"]; + + var listConfig = new ListConfiguration { - var listConfig = new ListConfiguration - { - Name = nameAttr.Value, - SharepointList = contextAttr?.Value, - Ignore = ParseBoolean(ignoreAttr?.Value, false), - Columns = GetListColumns(listNode) - }; + Name = nameAttr.Value, + SharepointList = contextAttr?.Value ?? string.Empty, + Ignore = ParseBoolean(ignoreAttr?.Value, false), + Columns = GetListColumns(listNode) + }; - listConfigs[listConfig.Name] = listConfig; - } + listConfigs[listConfig.Name] = listConfig; } return listConfigs.Count > 0 ? listConfigs : null; } - catch (Exception ex) + catch (Exception ex) when (ex is not InvalidOperationException) { - Console.WriteLine("Error reading list configurations: " + ex.Message); - throw; + Logger.LogError("Error reading list configurations", ex); + throw new InvalidOperationException("Error reading list configurations.", ex); } } - private static Dictionary GetListColumns(XmlNode listNode) + private static Dictionary? GetListColumns(XmlNode listNode) { var columnNodes = listNode.SelectNodes(".//Columns/column"); if (columnNodes == null || columnNodes.Count == 0) @@ -502,50 +368,52 @@ private static Dictionary GetListColumns(XmlNode listNode foreach (XmlNode node in columnNodes) { - var sourceAttr = node.Attributes["source"]; - var destAttr = node.Attributes["destination"]; - var ignoreAttr = node.Attributes["ignore"]; - var datatypeAttr = node.Attributes["datatype"]; + var sourceAttr = node.Attributes?["source"]; + if (sourceAttr == null) + { + continue; + } - if (sourceAttr != null) + var destAttr = node.Attributes?["destination"]; + var ignoreAttr = node.Attributes?["ignore"]; + var datatypeAttr = node.Attributes?["datatype"]; + + var mapping = new ColumnMapping { - var mapping = new ColumnMapping - { - Source = sourceAttr.Value, - Destination = destAttr?.Value ?? sourceAttr.Value, - Ignore = ParseBoolean(ignoreAttr?.Value, false), - DataType = datatypeAttr?.Value - }; + Source = sourceAttr.Value, + Destination = destAttr?.Value ?? sourceAttr.Value, + Ignore = ParseBoolean(ignoreAttr?.Value, false), + DataType = datatypeAttr?.Value + }; - columnMappings[mapping.Source] = mapping; - } + columnMappings[mapping.Source] = mapping; } return columnMappings.Count > 0 ? columnMappings : null; } } -public class SharePointListConfig +public sealed class SharePointListConfig { - public string SiteUrl { get; set; } - public string ListTitle { get; set; } - public string SqlTable { get; set; } + public string SiteUrl { get; set; } = string.Empty; + public string ListTitle { get; set; } = string.Empty; + public string SqlTable { get; set; } = string.Empty; public bool AutoAddNewColumns { get; set; } public bool Disabled { get; set; } } -public class ColumnMapping +public sealed class ColumnMapping { - public string Source { get; set; } - public string Destination { get; set; } + public string Source { get; set; } = string.Empty; + public string Destination { get; set; } = string.Empty; public bool Ignore { get; set; } - public string DataType { get; set; } + public string? DataType { get; set; } } -public class ListConfiguration +public sealed class ListConfiguration { - public string Name { get; set; } - public string SharepointList { get; set; } + public string Name { get; set; } = string.Empty; + public string SharepointList { get; set; } = string.Empty; public bool Ignore { get; set; } - public Dictionary Columns { get; set; } + public Dictionary? Columns { get; set; } } diff --git a/tests/SPO2SQL.Console.Tests/ConfigurationOptionsTests.cs b/tests/SPO2SQL.Console.Tests/ConfigurationOptionsTests.cs index 438f400..65f7c06 100644 --- a/tests/SPO2SQL.Console.Tests/ConfigurationOptionsTests.cs +++ b/tests/SPO2SQL.Console.Tests/ConfigurationOptionsTests.cs @@ -33,6 +33,79 @@ public void SharePointOptions_DefaultValues_AreSet() options.Password.Should().BeEmpty(); options.SiteUrl.Should().BeEmpty(); } + + [Fact] + public void Validate_WithUsernameAndPassword_UsesLegacyAuthentication() + { + var options = new SharePointOptions + { + SiteUrl = "https://contoso.sharepoint.com", + Username = "user@contoso.com", + Password = "secret" + }; + + Validate(options).Should().BeEmpty(); + options.HasAppOnlyAuth.Should().BeFalse(); + } + + [Fact] + public void Validate_WithoutAnyAuthenticationMode_ReturnsError() + { + var options = new SharePointOptions + { + SiteUrl = "https://contoso.sharepoint.com" + }; + + Validate(options).Should().NotBeEmpty(); + } + + [Fact] + public void Validate_WithCompleteAppOnlySettings_AcceptsExistingCertificate() + { + string certificatePath = Path.GetTempFileName(); + try + { + var options = new SharePointOptions + { + SiteUrl = "https://contoso.sharepoint.com", + TenantId = "tenant-id", + ClientId = "client-id", + CertificatePath = certificatePath + }; + + Validate(options).Should().BeEmpty(); + options.HasAppOnlyAuth.Should().BeTrue(); + } + finally + { + File.Delete(certificatePath); + } + } + + [Fact] + public void Validate_WithMissingCertificate_ReturnsError() + { + var options = new SharePointOptions + { + SiteUrl = "https://contoso.sharepoint.com", + TenantId = "tenant-id", + ClientId = "client-id", + CertificatePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".pfx") + }; + + Validate(options).Should().NotBeEmpty(); + } + + private static List Validate(SharePointOptions options) + { + var results = new List(); + System.ComponentModel.DataAnnotations.Validator.TryValidateObject( + options, + new System.ComponentModel.DataAnnotations.ValidationContext(options), + results, + validateAllProperties: true); + return results; + } } public class SqlOptionsTests diff --git a/tests/SPO2SQL.Console.Tests/SyncExitCodeTests.cs b/tests/SPO2SQL.Console.Tests/SyncExitCodeTests.cs new file mode 100644 index 0000000..2b93906 --- /dev/null +++ b/tests/SPO2SQL.Console.Tests/SyncExitCodeTests.cs @@ -0,0 +1,34 @@ +using FluentAssertions; +using Xunit; + +namespace SPO2SQL.Console.Tests; + +public class SyncExitCodeTests +{ + [Fact] + public void NewInstance_DefaultsToSuccess() + { + new SyncExitCode().Value.Should().Be(0); + } + + [Fact] + public void Fail_SetsExitCodeToOne() + { + var exitCode = new SyncExitCode(); + + exitCode.Fail(); + + exitCode.Value.Should().Be(1); + } + + [Fact] + public void Fail_IsIdempotent() + { + var exitCode = new SyncExitCode(); + + exitCode.Fail(); + exitCode.Fail(); + + exitCode.Value.Should().Be(1); + } +} diff --git a/tests/SPO2SQL.Core.Tests/CamlQueryBuilderTests.cs b/tests/SPO2SQL.Core.Tests/CamlQueryBuilderTests.cs index d9b80b9..e37ba63 100644 --- a/tests/SPO2SQL.Core.Tests/CamlQueryBuilderTests.cs +++ b/tests/SPO2SQL.Core.Tests/CamlQueryBuilderTests.cs @@ -40,6 +40,17 @@ public void BuildDateRangeQuery_WithValidDates_ReturnsValidCaml() .And.Contain(""); } + [Fact] + public void BuildDateRangeQuery_WithUtcDates_DoesNotShiftValues() + { + var from = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + var result = CamlQueryBuilder.BuildDateRangeQuery("Modified", from, from.AddHours(1)); + + result.Should().Contain("2024-01-01T00:00:00Z") + .And.Contain("2024-01-01T01:00:00Z"); + } + [Fact] public void BuildDateRangeQuery_WithInvalidRange_ThrowsArgumentException() { diff --git a/tests/SPO2SQL.Core.Tests/ConfigurationReaderTests.cs b/tests/SPO2SQL.Core.Tests/ConfigurationReaderTests.cs new file mode 100644 index 0000000..69bc004 --- /dev/null +++ b/tests/SPO2SQL.Core.Tests/ConfigurationReaderTests.cs @@ -0,0 +1,85 @@ +using FluentAssertions; +using SPO2SQL.XmlConfig; +using Xunit; + +namespace SPO2SQL.Core.Tests; + +[CollectionDefinition("ConfigurationReaderTests", DisableParallelization = true)] +public class ConfigurationReaderTestDefinition +{ +} + +[Collection("ConfigurationReaderTests")] +public sealed class ConfigurationReaderTests : IDisposable +{ + private readonly string _configPath = Path.Combine(Path.GetTempPath(), $"spo2sql-{Guid.NewGuid():N}.xml"); + + public ConfigurationReaderTests() + { + File.WriteAllText(_configPath, """ + + + https://contoso.sharepoint.com + + + + + + + + InternalField + + + + + + + + """); + + ConfigurationReader.SetConfigPath(_configPath); + } + + [Fact] + public void GetSelectedColumns_UsesMappedDestinationAndIgnoreFlags() + { + var columns = ConfigurationReader.GetSelectedColumns("Activities"); + + columns.Should().NotBeNull(); + columns!["Title"].Destination.Should().Be("ListItemTitle"); + columns["Hidden"].Ignore.Should().BeTrue(); + } + + [Fact] + public void GetIgnoredColumns_ReturnsGlobalIgnores() + { + var columns = ConfigurationReader.GetIgnoredColumns(); + + columns.Should().NotBeNull(); + columns!.Should().Contain("InternalField"); + } + + [Fact] + public void GetListConfigurations_SkipsIgnoredLists() + { + var lists = ConfigurationReader.GetListConfigurations(); + + lists.Should().NotBeNull(); + lists!["Activities"].Ignore.Should().BeFalse(); + lists["Ignored"].Ignore.Should().BeTrue(); + lists["Activities"].SharepointList.Should().Be("wolf/activities"); + } + + [Fact] + public void GetSharePointBaseUrl_ReturnsConfiguredOverride() + { + ConfigurationReader.SetSharePointBaseUrl("https://override.sharepoint.com/sites/dev"); + + ConfigurationReader.GetSharePointBaseUrl().Should().Be("https://override.sharepoint.com"); + } + + public void Dispose() + { + File.Delete(_configPath); + } +} diff --git a/tests/SPO2SQL.Core.Tests/LoggerTests.cs b/tests/SPO2SQL.Core.Tests/LoggerTests.cs new file mode 100644 index 0000000..f7001d1 --- /dev/null +++ b/tests/SPO2SQL.Core.Tests/LoggerTests.cs @@ -0,0 +1,91 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging; +using NSubstitute; +using SPO2SQL.Logging; +using Xunit; + +// CA1873 targets production logging calls; the calls in this file are NSubstitute +// verifications whose argument matchers must be evaluated eagerly. +#pragma warning disable CA1873 + +namespace SPO2SQL.Core.Tests; + +[CollectionDefinition("LoggingTests", DisableParallelization = true)] +public class LoggingTestDefinition +{ +} + +[Collection("LoggingTests")] +public class LoggerTests +{ + [Fact] + public void Configure_WithNullSink_ThrowsArgumentNullException() + { + var act = () => Logger.Configure(null!); + + act.Should().Throw(); + } + + [Fact] + public void LegacyLogLevels_AreRoutedToTheHostLogger() + { + var sink = Substitute.For(); + sink.IsEnabled(Arg.Any()).Returns(true); + Logger.Configure(sink); + + try + { + Logger.Log(1, "legacy-info"); + Logger.LogWarning("legacy-warning"); + Logger.LogDebug("legacy-debug"); + Logger.LogError("legacy-error", new InvalidOperationException("boom")); + + AssertLogged(sink, LogLevel.Information, "legacy-info"); + AssertLogged(sink, LogLevel.Warning, "legacy-warning"); + AssertLogged(sink, LogLevel.Debug, "legacy-debug"); + sink.Received(1).Log( + LogLevel.Error, + Arg.Any(), + Arg.Is(s => s == "legacy-error"), + Arg.Is(e => e is InvalidOperationException), + Arg.Any>()); + } + finally + { + Logger.Reset(); + } + } + + [Fact] + public void VerboseLevel_FiltersDebugMessages() + { + var sink = Substitute.For(); + sink.IsEnabled(Arg.Any()).Returns(true); + Logger.Configure(sink); + Logger.VerboseLevel = 1; + + try + { + Logger.Log(3, "hidden-debug"); + + sink.DidNotReceive().Log( + LogLevel.Debug, + Arg.Any(), + Arg.Is(s => s == "hidden-debug"), + (Exception?)null, + Arg.Any>()); + } + finally + { + Logger.Reset(); + } + } + + private static void AssertLogged(ILogger sink, LogLevel level, string message, Exception? exception = null) => + sink.Received(1).Log( + level, + Arg.Any(), + Arg.Is(s => s == message), + exception, + Arg.Any>()); +} diff --git a/tests/SPO2SQL.Core.Tests/RetryPolicyTests.cs b/tests/SPO2SQL.Core.Tests/RetryPolicyTests.cs index ec41530..4374a25 100644 --- a/tests/SPO2SQL.Core.Tests/RetryPolicyTests.cs +++ b/tests/SPO2SQL.Core.Tests/RetryPolicyTests.cs @@ -56,6 +56,42 @@ public void ExecuteWithRetry_SucceedsOnRetry_ReturnsResult() attempts.Should().Be(2); } + [Fact] + public async Task ExecuteWithRetryAsync_SucceedsOnRetry_ReturnsResult() + { + var policy = new RetryPolicy(maxRetries: 3, initialDelayMs: 100); + var attempts = 0; + + var result = await policy.ExecuteWithRetryAsync(async () => + { + attempts++; + await Task.Yield(); + if (attempts < 3) + { + throw new TimeoutException("Simulated transient error"); + } + + return 42; + }, CancellationToken.None); + + result.Should().Be(42); + attempts.Should().Be(3); + } + + [Fact] + public async Task ExecuteWithRetryAsync_WithCancellation_ThrowsOperationCanceledException() + { + var policy = new RetryPolicy(maxRetries: 3, initialDelayMs: 100); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + var act = async () => await policy.ExecuteWithRetryAsync( + () => Task.FromResult(1), + cts.Token); + + await act.Should().ThrowAsync(); + } + [Fact] public void IsTransientError_WithTimeoutException_ReturnsTrue() { @@ -67,7 +103,7 @@ public void IsTransientError_WithTimeoutException_ReturnsTrue() [Fact] public void IsTransientError_WithNull_ReturnsFalse() { - var result = RetryPolicy.IsTransientError(null); + var result = RetryPolicy.IsTransientError(null!); result.Should().BeFalse(); } diff --git a/tests/SPO2SQL.Core.Tests/SyncSettingsTests.cs b/tests/SPO2SQL.Core.Tests/SyncSettingsTests.cs new file mode 100644 index 0000000..2532281 --- /dev/null +++ b/tests/SPO2SQL.Core.Tests/SyncSettingsTests.cs @@ -0,0 +1,51 @@ +using FluentAssertions; +using SPO2SQL.Configuration; +using Xunit; + +namespace SPO2SQL.Core.Tests; + +public class SyncSettingsTests +{ + [Fact] + public void HasAppOnlyAuth_WithCompleteAppOnlySettings_ReturnsTrue() + { + var settings = new SyncSettings + { + SiteUrl = "https://contoso.sharepoint.com", + SqlConnectionString = "Server=.;Database=x;", + TenantId = "tenant-id", + ClientId = "client-id", + CertificatePath = "/certs/app.pfx" + }; + + settings.HasAppOnlyAuth.Should().BeTrue(); + } + + [Fact] + public void HasAppOnlyAuth_WithIncompleteSettings_ReturnsFalse() + { + var settings = new SyncSettings + { + SiteUrl = "https://contoso.sharepoint.com", + SqlConnectionString = "Server=.;Database=x;", + TenantId = "tenant-id", + ClientId = "client-id" + }; + + settings.HasAppOnlyAuth.Should().BeFalse(); + } + + [Fact] + public void HasAppOnlyAuth_WithOnlyUserPassword_ReturnsFalse() + { + var settings = new SyncSettings + { + SiteUrl = "https://contoso.sharepoint.com", + SqlConnectionString = "Server=.;Database=x;", + Username = "user@contoso.com", + Password = "secret" + }; + + settings.HasAppOnlyAuth.Should().BeFalse(); + } +}