From 9f3ab7bae1669e7c6f836331cbc8e32c8ec3571d Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 16:19:01 +0100 Subject: [PATCH 01/16] added newtonsoft.json 13.0.4 to the Core --- idou.Core/idou.Core.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/idou.Core/idou.Core.csproj b/idou.Core/idou.Core.csproj index 237d661..3d6217c 100644 --- a/idou.Core/idou.Core.csproj +++ b/idou.Core/idou.Core.csproj @@ -6,4 +6,8 @@ enable + + + + From 776c6c7259e2128505bf96c25d05b81914c20179 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 16:19:24 +0100 Subject: [PATCH 02/16] added a valid flag to the IMappingPlan --- idou.Core.Tests/Engine/ChangePipelineTests.cs | 2 ++ idou.Core/Mapping/DefaultMappingPlan.cs | 2 ++ idou.Core/Mapping/IMappingPlan.cs | 1 + 3 files changed, 5 insertions(+) diff --git a/idou.Core.Tests/Engine/ChangePipelineTests.cs b/idou.Core.Tests/Engine/ChangePipelineTests.cs index 2315741..433b144 100644 --- a/idou.Core.Tests/Engine/ChangePipelineTests.cs +++ b/idou.Core.Tests/Engine/ChangePipelineTests.cs @@ -180,6 +180,8 @@ public void MapAndTransform_IfTransformerTurnsEventIntoNoOp_ItIsFilteredOut() private sealed class RecordingMappingPlan : IMappingPlan { + public bool IsValid => true; + private readonly Func _mapEntityType; private readonly Func _mapKey; private readonly Func _mapRecord; diff --git a/idou.Core/Mapping/DefaultMappingPlan.cs b/idou.Core/Mapping/DefaultMappingPlan.cs index ad45796..361c408 100644 --- a/idou.Core/Mapping/DefaultMappingPlan.cs +++ b/idou.Core/Mapping/DefaultMappingPlan.cs @@ -4,6 +4,8 @@ namespace idou.Core.Mapping; public class DefaultMappingPlan : IMappingPlan { + public bool IsValid => true; + public EntityType MapEntityType(EntityType sourceType) { return sourceType; diff --git a/idou.Core/Mapping/IMappingPlan.cs b/idou.Core/Mapping/IMappingPlan.cs index d52ac97..53f425e 100644 --- a/idou.Core/Mapping/IMappingPlan.cs +++ b/idou.Core/Mapping/IMappingPlan.cs @@ -4,6 +4,7 @@ namespace idou.Core.Mapping; public interface IMappingPlan { + bool IsValid { get; } EntityType MapEntityType(EntityType sourceType); EntityKey MapKey(EntityType sourceType, EntityKey sourceKey); EntityRecord MapRecord(EntityRecord sourceRecord); From da470455ff4b5049eb28d4d79e4d9bd6ce4f8532 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 16:19:49 +0100 Subject: [PATCH 03/16] started implementing json mappings --- .../Mapping/JsonMappingPlanTests.cs | 28 +++++++++++++++ idou.Core.Tests/Mapping/assets/empty.json | 2 ++ idou.Core.Tests/Mapping/assets/valid.json | 6 ++++ idou.Core.Tests/idou.Core.Tests.csproj | 9 +++++ idou.Core/Mapping/JsonMapping.cs | 7 ++++ idou.Core/Mapping/JsonMappingPlan.cs | 36 +++++++++++++++++++ 6 files changed, 88 insertions(+) create mode 100644 idou.Core.Tests/Mapping/JsonMappingPlanTests.cs create mode 100644 idou.Core.Tests/Mapping/assets/empty.json create mode 100644 idou.Core.Tests/Mapping/assets/valid.json create mode 100644 idou.Core/Mapping/JsonMapping.cs create mode 100644 idou.Core/Mapping/JsonMappingPlan.cs diff --git a/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs new file mode 100644 index 0000000..fef9e6f --- /dev/null +++ b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs @@ -0,0 +1,28 @@ +using idou.Core.Mapping; + +namespace idou.Core.Tests.Mapping; + +public class JsonMappingPlanTests +{ + [Fact] + public void MappingPlanIsValid_WhenTheFileIsCorrectAndHasMappings() + { + var plan = new JsonMappingPlan("Mapping/assets/valid.json"); + + Assert.True(plan.IsValid); + } + + [Fact] + public void Throws_WhenTheFileIsInvalid() + { + Assert.Throws(() => new JsonMappingPlan("Mapping/assets/invalid.json")); + } + + [Fact] + public void MappingPlanIsInvalid_WhenTheMappingsAreEmpty() + { + var plan = new JsonMappingPlan("Mapping/assets/empty.json"); + + Assert.False(plan.IsValid); + } +} \ No newline at end of file diff --git a/idou.Core.Tests/Mapping/assets/empty.json b/idou.Core.Tests/Mapping/assets/empty.json new file mode 100644 index 0000000..32960f8 --- /dev/null +++ b/idou.Core.Tests/Mapping/assets/empty.json @@ -0,0 +1,2 @@ +[ +] \ No newline at end of file diff --git a/idou.Core.Tests/Mapping/assets/valid.json b/idou.Core.Tests/Mapping/assets/valid.json new file mode 100644 index 0000000..10e1614 --- /dev/null +++ b/idou.Core.Tests/Mapping/assets/valid.json @@ -0,0 +1,6 @@ +[ + { + "source": "column1", + "target": "targetColumn1" + } +] \ No newline at end of file diff --git a/idou.Core.Tests/idou.Core.Tests.csproj b/idou.Core.Tests/idou.Core.Tests.csproj index 5ec0c9a..b1b110f 100644 --- a/idou.Core.Tests/idou.Core.Tests.csproj +++ b/idou.Core.Tests/idou.Core.Tests.csproj @@ -22,4 +22,13 @@ + + + Always + + + Always + + + \ No newline at end of file diff --git a/idou.Core/Mapping/JsonMapping.cs b/idou.Core/Mapping/JsonMapping.cs new file mode 100644 index 0000000..9bf0a87 --- /dev/null +++ b/idou.Core/Mapping/JsonMapping.cs @@ -0,0 +1,7 @@ +namespace idou.Core.Mapping; + +public class JsonMapping +{ + public string Source { get; set; } + public string Target { get; set; } +} \ No newline at end of file diff --git a/idou.Core/Mapping/JsonMappingPlan.cs b/idou.Core/Mapping/JsonMappingPlan.cs new file mode 100644 index 0000000..ea15c73 --- /dev/null +++ b/idou.Core/Mapping/JsonMappingPlan.cs @@ -0,0 +1,36 @@ +using idou.Core.Domain; +using Newtonsoft.Json; + +namespace idou.Core.Mapping; + +public class JsonMappingPlan : IMappingPlan +{ + private readonly IList _mappings; + + public JsonMappingPlan(string filePath) + { + if (!File.Exists(filePath)) + { + throw new FileNotFoundException(filePath); + } + + _mappings = JsonConvert.DeserializeObject>(File.ReadAllText(filePath))?.ToList() ?? []; + } + + public bool IsValid => _mappings.Any(); + + public EntityType MapEntityType(EntityType sourceType) + { + return sourceType; + } + + public EntityKey MapKey(EntityType sourceType, EntityKey sourceKey) + { + return sourceKey; + } + + public EntityRecord MapRecord(EntityRecord sourceRecord) + { + return sourceRecord; + } +} \ No newline at end of file From ca25705d7731b2a1df8eded3be12ed88661c5fa7 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 17:34:58 +0100 Subject: [PATCH 04/16] added flow diagram --- docs/flow.puml | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/flow.puml diff --git a/docs/flow.puml b/docs/flow.puml new file mode 100644 index 0000000..feaa067 --- /dev/null +++ b/docs/flow.puml @@ -0,0 +1,82 @@ +@startuml + +title Sync Job Execution Flow + +start + +:SyncEngine.RunAsync(job); + +:JobStateStore.SetStatus(Running); + +:Load Checkpoint; +note right +ICheckpointStore.LoadAsync(jobId) +end note + +if (Checkpoint exists?) then (no) + :Bootstrap mode; + repeat + :Read snapshot page; + note right + IDataSource.ReadSnapshotAsync + end note + + :Map & Transform; + note right + ChangePipeline + - IMappingPlan + - ITransformer[] + end note + + :Apply upserts; + note right + IDataTarget.UpsertAsync + end note + + :Update progress; + note right + IJobStateStore.UpdateProgress + end note + repeat while (More pages?) +else (yes) + :Incremental mode; + repeat + :Read changes; + note right + IDataSource.ReadChangesAsync + end note + + :Map & Transform; + note right + ChangePipeline + end note + + if (Conflict resolution enabled?) then (yes) + :Resolve conflicts; + note right + IConflictResolver.Resolve + (may call IDataTarget.TryGetAsync) + end note + endif + + :Apply changes; + note right + IDataTarget.ApplyChangesAsync + end note + + :Save checkpoint; + note right + ICheckpointStore.SaveAsync + end note + + :Update progress; + note right + IJobStateStore.UpdateProgress + end note + repeat while (Continue & not cancelled) +endif + +:JobStateStore.SetStatus(Stopped); + +stop +@enduml From 5415f781abd349b659937dfc796ed0843c31a07b Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 17:35:18 +0100 Subject: [PATCH 05/16] working inital JSON mapping plan --- .../Mapping/JsonMappingPlanTests.cs | 32 ++++++++++++++++--- idou.Core.Tests/Mapping/assets/valid.json | 4 +-- idou.Core/Mapping/JsonMappingPlan.cs | 28 +++++++++++----- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs index fef9e6f..90ca235 100644 --- a/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs +++ b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs @@ -1,3 +1,4 @@ +using idou.Core.Domain; using idou.Core.Mapping; namespace idou.Core.Tests.Mapping; @@ -5,24 +6,45 @@ namespace idou.Core.Tests.Mapping; public class JsonMappingPlanTests { [Fact] - public void MappingPlanIsValid_WhenTheFileIsCorrectAndHasMappings() + public void MappingPlanIsValid_WhenTheFileExistsAndHasMappings() { var plan = new JsonMappingPlan("Mapping/assets/valid.json"); - Assert.True(plan.IsValid); } [Fact] - public void Throws_WhenTheFileIsInvalid() + public void Throws_WhenTheFileDoesNotExist() { - Assert.Throws(() => new JsonMappingPlan("Mapping/assets/invalid.json")); + Assert.Throws(() => new JsonMappingPlan("Mapping/assets/does-not-exist.json")); } [Fact] public void MappingPlanIsInvalid_WhenTheMappingsAreEmpty() { var plan = new JsonMappingPlan("Mapping/assets/empty.json"); - Assert.False(plan.IsValid); } + + [Fact] + public void MapsTheEntityType_WhenTheMappingExists() + { + var plan = new JsonMappingPlan("Mapping/assets/valid.json"); + + var sourceType = new EntityType("users"); + var mapped = plan.MapEntityType(sourceType); + + Assert.True(plan.IsValid); + Assert.Equal("accounts", mapped.Name); + } + + [Fact] + public void DoesNotMapTheEntityType_WhenNoMappingExists() + { + var plan = new JsonMappingPlan("Mapping/assets/valid.json"); + + var sourceType = new EntityType("unknown"); + var mapped = plan.MapEntityType(sourceType); + + Assert.Equal(sourceType.Name, mapped.Name); + } } \ No newline at end of file diff --git a/idou.Core.Tests/Mapping/assets/valid.json b/idou.Core.Tests/Mapping/assets/valid.json index 10e1614..8b46f84 100644 --- a/idou.Core.Tests/Mapping/assets/valid.json +++ b/idou.Core.Tests/Mapping/assets/valid.json @@ -1,6 +1,6 @@ [ { - "source": "column1", - "target": "targetColumn1" + "source": "users", + "target": "accounts" } ] \ No newline at end of file diff --git a/idou.Core/Mapping/JsonMappingPlan.cs b/idou.Core/Mapping/JsonMappingPlan.cs index ea15c73..99d8654 100644 --- a/idou.Core/Mapping/JsonMappingPlan.cs +++ b/idou.Core/Mapping/JsonMappingPlan.cs @@ -3,25 +3,29 @@ namespace idou.Core.Mapping; -public class JsonMappingPlan : IMappingPlan +public sealed class JsonMappingPlan : IMappingPlan { - private readonly IList _mappings; + private readonly IReadOnlyDictionary _bySource; public JsonMappingPlan(string filePath) { if (!File.Exists(filePath)) - { throw new FileNotFoundException(filePath); - } - _mappings = JsonConvert.DeserializeObject>(File.ReadAllText(filePath))?.ToList() ?? []; + var mappings = JsonConvert.DeserializeObject>( + File.ReadAllText(filePath) + ) ?? throw new InvalidOperationException("Invalid mapping file"); + + _bySource = mappings.ToDictionary(m => m.Source, StringComparer.OrdinalIgnoreCase); } - public bool IsValid => _mappings.Any(); + public bool IsValid => _bySource.Count > 0; public EntityType MapEntityType(EntityType sourceType) { - return sourceType; + return _bySource.TryGetValue(sourceType.Name, out var mapping) + ? new EntityType(mapping.Target) + : sourceType; } public EntityKey MapKey(EntityType sourceType, EntityKey sourceKey) @@ -31,6 +35,14 @@ public EntityKey MapKey(EntityType sourceType, EntityKey sourceKey) public EntityRecord MapRecord(EntityRecord sourceRecord) { - return sourceRecord; + var targetType = MapEntityType(sourceRecord.Type); + var targetKey = MapKey(sourceRecord.Type, sourceRecord.Key); + + return new EntityRecord + { + Key = targetKey, + Type = targetType, + Attributes = new Dictionary(sourceRecord.Attributes) + }; } } \ No newline at end of file From d3b1f36e30f1b5f7beb758dfd4da1fa069ddd58a Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 17:39:11 +0100 Subject: [PATCH 06/16] added idea configs and excluded copilot settings --- .gitignore | 340 ++++++++++++++++++++++++- .idea/.idea.idou/.idea/.gitignore | 15 ++ .idea/.idea.idou/.idea/indexLayout.xml | 8 + .idea/.idea.idou/.idea/vcs.xml | 6 + 4 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 .idea/.idea.idou/.idea/.gitignore create mode 100644 .idea/.idea.idou/.idea/indexLayout.xml create mode 100644 .idea/.idea.idou/.idea/vcs.xml diff --git a/.gitignore b/.gitignore index 4b1dfcb..6a6c68d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,341 @@ -bin/ -obj/ +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper *.DotSettings.user +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs +Resource.designer.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake # +/tools/** +/build/tools/** +/build/temp/** + +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf +.idea/*/.idea/projectSettingsUpdater.xml +.idea/*/.idea/encodings.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +.idea/modules.xml +.idea/*.iml +.idea/modules +*.iml +*.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +# fastlane +fastlane/report.xml + +# inspectcode +inspectcodereport.xml +inspectcode + +# BenchmarkDotNet +/BenchmarkDotNet.Artifacts + +*.GeneratedMSBuildEditorConfig.editorconfig + + +.idea/.idea.idou/.idea/misc.xml +.idea/.idea.idou/.idea/copilot.* diff --git a/.idea/.idea.idou/.idea/.gitignore b/.idea/.idea.idou/.idea/.gitignore new file mode 100644 index 0000000..e247bb2 --- /dev/null +++ b/.idea/.idea.idou/.idea/.gitignore @@ -0,0 +1,15 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Rider ignored files +/projectSettingsUpdater.xml +/modules.xml +/contentModel.xml +/.idea.idou.iml +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/.idea.idou/.idea/indexLayout.xml b/.idea/.idea.idou/.idea/indexLayout.xml new file mode 100644 index 0000000..7b08163 --- /dev/null +++ b/.idea/.idea.idou/.idea/indexLayout.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/.idea.idou/.idea/vcs.xml b/.idea/.idea.idou/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/.idea.idou/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file From 440ea467d2c6f231e752070f551fa31487a01ad1 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 20:08:21 +0100 Subject: [PATCH 07/16] removed the two user specific files from git --- .gitignore | 2 ++ .idea/.idea.idou/.idea/indexLayout.xml | 8 -------- .idea/.idea.idou/.idea/vcs.xml | 6 ------ 3 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 .idea/.idea.idou/.idea/indexLayout.xml delete mode 100644 .idea/.idea.idou/.idea/vcs.xml diff --git a/.gitignore b/.gitignore index 6a6c68d..4f2872d 100644 --- a/.gitignore +++ b/.gitignore @@ -264,6 +264,8 @@ __pycache__/ .idea/**/shelf .idea/*/.idea/projectSettingsUpdater.xml .idea/*/.idea/encodings.xml +.idea/*/.idea/vcs.xml +.idea/*/.idea/indexLayout.xml # Generated files .idea/**/contentModel.xml diff --git a/.idea/.idea.idou/.idea/indexLayout.xml b/.idea/.idea.idou/.idea/indexLayout.xml deleted file mode 100644 index 7b08163..0000000 --- a/.idea/.idea.idou/.idea/indexLayout.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/.idea.idou/.idea/vcs.xml b/.idea/.idea.idou/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/.idea.idou/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From cbef203000f2fd9524b88b2b2d673c5bbbd084e7 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 20:13:59 +0100 Subject: [PATCH 08/16] improved the tests --- idou.Core.Tests/Mapping/JsonMappingPlanTests.cs | 10 ++++++++++ idou.Core/Mapping/JsonMappingPlan.cs | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs index 90ca235..5ceaf86 100644 --- a/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs +++ b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs @@ -37,6 +37,16 @@ public void MapsTheEntityType_WhenTheMappingExists() Assert.Equal("accounts", mapped.Name); } + [Fact] + public void MapsTheEntityType_CaseInsensitive_WhenTheMappingExists() + { + var plan = new JsonMappingPlan("Mapping/assets/valid.json"); + var sourceType = new EntityType("UsErS"); + var mapped = plan.MapEntityType(sourceType); + Assert.True(plan.IsValid); + Assert.Equal("accounts", mapped.Name); + } + [Fact] public void DoesNotMapTheEntityType_WhenNoMappingExists() { diff --git a/idou.Core/Mapping/JsonMappingPlan.cs b/idou.Core/Mapping/JsonMappingPlan.cs index 99d8654..0e77d20 100644 --- a/idou.Core/Mapping/JsonMappingPlan.cs +++ b/idou.Core/Mapping/JsonMappingPlan.cs @@ -10,7 +10,7 @@ public sealed class JsonMappingPlan : IMappingPlan public JsonMappingPlan(string filePath) { if (!File.Exists(filePath)) - throw new FileNotFoundException(filePath); + throw new FileNotFoundException($"Mapping file not found: {filePath}"); var mappings = JsonConvert.DeserializeObject>( File.ReadAllText(filePath) From c690f63b843dae90df35df99368d9c5592d09cf4 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 20:23:23 +0100 Subject: [PATCH 09/16] added tests for JsonMapping --- idou.Core.Tests/Mapping/JsonMappingTests.cs | 22 +++++++++++++++++++++ idou.Core/Mapping/JsonMapping.cs | 6 +++--- 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 idou.Core.Tests/Mapping/JsonMappingTests.cs diff --git a/idou.Core.Tests/Mapping/JsonMappingTests.cs b/idou.Core.Tests/Mapping/JsonMappingTests.cs new file mode 100644 index 0000000..a4dd648 --- /dev/null +++ b/idou.Core.Tests/Mapping/JsonMappingTests.cs @@ -0,0 +1,22 @@ +using idou.Core.Mapping; + +namespace idou.Core.Tests.Mapping; + +public class JsonMappingTests +{ + [Fact] + public void JsonMapping_Creation_Works() + { + const string source = "sourceField"; + const string target = "targetField"; + + var mapping = new JsonMapping + { + Source = source, + Target = target + }; + + Assert.Equal(source, mapping.Source); + Assert.Equal(target, mapping.Target); + } +} \ No newline at end of file diff --git a/idou.Core/Mapping/JsonMapping.cs b/idou.Core/Mapping/JsonMapping.cs index 9bf0a87..c659ddc 100644 --- a/idou.Core/Mapping/JsonMapping.cs +++ b/idou.Core/Mapping/JsonMapping.cs @@ -1,7 +1,7 @@ namespace idou.Core.Mapping; -public class JsonMapping +public record JsonMapping { - public string Source { get; set; } - public string Target { get; set; } + public required string Source { get; init; } = null!; + public required string Target { get; init; } = null!; } \ No newline at end of file From 6e4fd048b3ef55a773c2cac61a3f42a0a0851776 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 20:35:03 +0100 Subject: [PATCH 10/16] maxed out test coverage --- idou.Core.Tests/Domain/EntityKeyTests.cs | 20 +++++++++++ idou.Core.Tests/Domain/EntityTypeTests.cs | 19 +++++++++++ .../Mapping/DefaultMappingPlanTests.cs | 22 +++++++++++++ .../Mapping/JsonMappingPlanTests.cs | 33 +++++++++++++++++++ 4 files changed, 94 insertions(+) create mode 100644 idou.Core.Tests/Domain/EntityKeyTests.cs create mode 100644 idou.Core.Tests/Domain/EntityTypeTests.cs create mode 100644 idou.Core.Tests/Mapping/DefaultMappingPlanTests.cs diff --git a/idou.Core.Tests/Domain/EntityKeyTests.cs b/idou.Core.Tests/Domain/EntityKeyTests.cs new file mode 100644 index 0000000..de36be0 --- /dev/null +++ b/idou.Core.Tests/Domain/EntityKeyTests.cs @@ -0,0 +1,20 @@ +using idou.Core.Domain; + +namespace idou.Core.Tests.Domain; + +public class EntityKeyTests +{ + [Fact] + public void EntityKey_Creation_Works() + { + const string keyName = "TestKey"; + + var entityKey = new EntityKey(keyName); + + Assert.Equal(keyName, entityKey.Value); + Assert.Equal(keyName, entityKey.ToString()); + Assert.Equal(keyName.GetHashCode(), entityKey.GetHashCode()); + Assert.True(entityKey.Equals(new EntityKey(keyName))); + Assert.False(entityKey.Equals(new EntityKey("DifferentKey"))); + } +} \ No newline at end of file diff --git a/idou.Core.Tests/Domain/EntityTypeTests.cs b/idou.Core.Tests/Domain/EntityTypeTests.cs new file mode 100644 index 0000000..07f53b1 --- /dev/null +++ b/idou.Core.Tests/Domain/EntityTypeTests.cs @@ -0,0 +1,19 @@ +using idou.Core.Domain; + +namespace idou.Core.Tests.Domain; + +public class EntityTypeTests +{ + [Fact] + public void EntityType_Creation_Works() + { + const string typeName = "TestType"; + + var entityType = new EntityType(typeName); + + Assert.Equal(typeName, entityType.ToString()); + Assert.Equal(typeName.GetHashCode(), entityType.GetHashCode()); + Assert.True(entityType.Equals(new EntityType(typeName))); + Assert.False(entityType.Equals(new EntityType("DifferentType"))); + } +} \ No newline at end of file diff --git a/idou.Core.Tests/Mapping/DefaultMappingPlanTests.cs b/idou.Core.Tests/Mapping/DefaultMappingPlanTests.cs new file mode 100644 index 0000000..4f01c44 --- /dev/null +++ b/idou.Core.Tests/Mapping/DefaultMappingPlanTests.cs @@ -0,0 +1,22 @@ +using idou.Core.Mapping; + +namespace idou.Core.Tests.Mapping; + +public class DefaultMappingPlanTests +{ + [Fact] + public void DefaultMappingPlan_Creation_Works() + { + var mappingPlan = new DefaultMappingPlan(); + + Assert.NotNull(mappingPlan); + } + + [Fact] + public void DefaultMappingPlan_IsValidByDefault() + { + var mappingPlan = new DefaultMappingPlan(); + + Assert.True(mappingPlan.IsValid); + } +} \ No newline at end of file diff --git a/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs index 5ceaf86..070fc7f 100644 --- a/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs +++ b/idou.Core.Tests/Mapping/JsonMappingPlanTests.cs @@ -57,4 +57,37 @@ public void DoesNotMapTheEntityType_WhenNoMappingExists() Assert.Equal(sourceType.Name, mapped.Name); } + + [Fact] + public void MapsTheEntityKey_WhenTheMappingExists() + { + var plan = new JsonMappingPlan("Mapping/assets/valid.json"); + var sourceType = new EntityType("users"); + var sourceKey = new EntityKey("u1"); + + var mappedKey = plan.MapKey(sourceType, sourceKey); + + Assert.Equal(sourceKey.Value, mappedKey.Value); + } + + [Fact] + public void MapsTheEntityRecord_WhenTheMappingExists() + { + var plan = new JsonMappingPlan("Mapping/assets/valid.json"); + var sourceRecord = new EntityRecord + { + Type = new EntityType("users"), + Key = new EntityKey("u1"), + Attributes = new Dictionary + { + { "name", "Alice" } + } + }; + + var mappedRecord = plan.MapRecord(sourceRecord); + + Assert.Equal("accounts", mappedRecord.Type.Name); + Assert.Equal(sourceRecord.Key.Value, mappedRecord.Key.Value); + Assert.Equal(sourceRecord.Attributes["name"], mappedRecord.Attributes["name"]); + } } \ No newline at end of file From 05e599e3a93f39fef5d2f84afe73ba5c3133bd9c Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 21:31:29 +0100 Subject: [PATCH 11/16] excluded the tests --- .github/workflows/test-pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 397304d..7efdbfd 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -39,6 +39,7 @@ jobs: --logger "trx" --results-directory "${{ github.workspace }}/TestResults" --collect:"XPlat Code Coverage" + /p:Exclude="[*.Tests]*" - name: Combine Coverage Reports if: always() From fc2189272494c26da57dbf6c3a8ce2c7c4c377aa Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 21:34:10 +0100 Subject: [PATCH 12/16] fixed the exclusion --- .github/workflows/test-pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 7efdbfd..53f04e4 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -39,7 +39,7 @@ jobs: --logger "trx" --results-directory "${{ github.workspace }}/TestResults" --collect:"XPlat Code Coverage" - /p:Exclude="[*.Tests]*" + /p:Exclude="[idou.*.Tests*]*" - name: Combine Coverage Reports if: always() From 96ef4f7eb202169943afd8c205a7740ec20ef49b Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 22:49:19 +0100 Subject: [PATCH 13/16] added more tests --- idou.Core.Tests/Domain/ChangeFilterTests.cs | 22 ++++++++++++++++++++ idou.Core.Tests/Domain/EntityRecordTests.cs | 23 +++++++++++++++++++++ idou.Core/Domain/ChangeFilter.cs | 6 +++--- idou.Core/Engine/RetryPolicy.cs | 6 ------ idou.Core/Engine/SyncEngine.cs | 6 ------ idou.Core/Engine/SyncJob.cs | 6 ------ 6 files changed, 48 insertions(+), 21 deletions(-) create mode 100644 idou.Core.Tests/Domain/ChangeFilterTests.cs create mode 100644 idou.Core.Tests/Domain/EntityRecordTests.cs delete mode 100644 idou.Core/Engine/RetryPolicy.cs delete mode 100644 idou.Core/Engine/SyncEngine.cs delete mode 100644 idou.Core/Engine/SyncJob.cs diff --git a/idou.Core.Tests/Domain/ChangeFilterTests.cs b/idou.Core.Tests/Domain/ChangeFilterTests.cs new file mode 100644 index 0000000..13ec6c9 --- /dev/null +++ b/idou.Core.Tests/Domain/ChangeFilterTests.cs @@ -0,0 +1,22 @@ +using idou.Core.Domain; + +namespace idou.Core.Tests.Domain; + +public class ChangeFilterTests +{ + [Fact] + public void ChangeFilter_Constructor_SetsProperties() + { + var entityTypes = new List + { + new("TypeA"), + new("TypeB") + }; + var since = DateTimeOffset.UtcNow.AddDays(-1); + + var changeFilter = new ChangeFilter(entityTypes, since); + + Assert.Equal(entityTypes, changeFilter.EntityTypes); + Assert.Equal(since, changeFilter.Since); + } +} \ No newline at end of file diff --git a/idou.Core.Tests/Domain/EntityRecordTests.cs b/idou.Core.Tests/Domain/EntityRecordTests.cs new file mode 100644 index 0000000..98fbe5a --- /dev/null +++ b/idou.Core.Tests/Domain/EntityRecordTests.cs @@ -0,0 +1,23 @@ +using idou.Core.Domain; + +namespace idou.Core.Tests.Domain; + +public class EntityRecordTests +{ + [Fact] + public void EntityRecord_Creation_Works() + { + var entityRecord = new EntityRecord + { + Key = new EntityKey("TestKey"), + Type = new EntityType("TestType"), + Attributes = new Dictionary() + }; + + Assert.NotNull(entityRecord); + + Assert.Equal("TestKey", entityRecord.Key.Value); + Assert.Equal("TestType", entityRecord.Type.ToString()); + Assert.NotNull(entityRecord.Attributes); + } +} \ No newline at end of file diff --git a/idou.Core/Domain/ChangeFilter.cs b/idou.Core/Domain/ChangeFilter.cs index 43a37f0..007cde4 100644 --- a/idou.Core/Domain/ChangeFilter.cs +++ b/idou.Core/Domain/ChangeFilter.cs @@ -1,7 +1,7 @@ namespace idou.Core.Domain; -public class ChangeFilter +public class ChangeFilter(IReadOnlyCollection entityTypes, DateTimeOffset? since) { - public IReadOnlyCollection EntityTypes { get; } - public DateTimeOffset? Since { get; } + public IReadOnlyCollection EntityTypes { get; } = entityTypes; + public DateTimeOffset? Since { get; } = since; } \ No newline at end of file diff --git a/idou.Core/Engine/RetryPolicy.cs b/idou.Core/Engine/RetryPolicy.cs deleted file mode 100644 index 707deb8..0000000 --- a/idou.Core/Engine/RetryPolicy.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace idou.Core.Engine; - -public class RetryPolicy -{ - -} \ No newline at end of file diff --git a/idou.Core/Engine/SyncEngine.cs b/idou.Core/Engine/SyncEngine.cs deleted file mode 100644 index f9996b0..0000000 --- a/idou.Core/Engine/SyncEngine.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace idou.Core.Engine; - -public class SyncEngine -{ - -} \ No newline at end of file diff --git a/idou.Core/Engine/SyncJob.cs b/idou.Core/Engine/SyncJob.cs deleted file mode 100644 index 0cd331a..0000000 --- a/idou.Core/Engine/SyncJob.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace idou.Core.Engine; - -public class SyncJob -{ - -} \ No newline at end of file From 9a5e094f1a0b8be775485856166ed578c8e71b7f Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 22:55:57 +0100 Subject: [PATCH 14/16] removed more unused files --- idou.Core/Mapping/JsonMapping.cs | 2 +- idou.Core/Results/ApplyResult.cs | 8 -------- idou.Core/Results/WriteResult.cs | 8 -------- 3 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 idou.Core/Results/ApplyResult.cs delete mode 100644 idou.Core/Results/WriteResult.cs diff --git a/idou.Core/Mapping/JsonMapping.cs b/idou.Core/Mapping/JsonMapping.cs index c659ddc..40e7e23 100644 --- a/idou.Core/Mapping/JsonMapping.cs +++ b/idou.Core/Mapping/JsonMapping.cs @@ -1,6 +1,6 @@ namespace idou.Core.Mapping; -public record JsonMapping +public class JsonMapping { public required string Source { get; init; } = null!; public required string Target { get; init; } = null!; diff --git a/idou.Core/Results/ApplyResult.cs b/idou.Core/Results/ApplyResult.cs deleted file mode 100644 index 1ee7d8a..0000000 --- a/idou.Core/Results/ApplyResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace idou.Core.Results; - -public class ApplyResult -{ - public int AppliedCount { get; } - public int FailedCount { get; } - public IReadOnlyList Errors { get; } -} \ No newline at end of file diff --git a/idou.Core/Results/WriteResult.cs b/idou.Core/Results/WriteResult.cs deleted file mode 100644 index 6c05df0..0000000 --- a/idou.Core/Results/WriteResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace idou.Core.Results; - -public class WriteResult -{ - public int WrittenCount { get; } - public int FailedCount { get; } - IReadOnlyList Errors { get; } -} \ No newline at end of file From d99f7a482dc19cbeba0eb0de8be2e559d30d482c Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 23:16:01 +0100 Subject: [PATCH 15/16] added moq to tests and dedicated tests for ChangeEvent --- idou.Core.Tests/Domain/ChangeEventTests.cs | 27 ++++++++++++++++++++++ idou.Core.Tests/idou.Core.Tests.csproj | 1 + 2 files changed, 28 insertions(+) create mode 100644 idou.Core.Tests/Domain/ChangeEventTests.cs diff --git a/idou.Core.Tests/Domain/ChangeEventTests.cs b/idou.Core.Tests/Domain/ChangeEventTests.cs new file mode 100644 index 0000000..9776ad5 --- /dev/null +++ b/idou.Core.Tests/Domain/ChangeEventTests.cs @@ -0,0 +1,27 @@ +using idou.Core.Domain; +using idou.Core.Domain.Enums; + +namespace idou.Core.Tests.Domain; + +public class ChangeEventTests +{ + [Fact] + public void ChangeEvent_Creation_Works() + { + var upsertMode = ChangeOperation.Upsert; + var offset = DateTimeOffset.UtcNow; + + var changeEvent = new ChangeEvent(offset) + { + Operation = upsertMode, + Key = new EntityKey("TestKey"), + Type = new EntityType("TestType") + }; + + Assert.NotNull(changeEvent); + Assert.Equal(upsertMode, changeEvent.Operation); + Assert.Equal("TestKey", changeEvent.Key.Value); + Assert.Equal("TestType", changeEvent.Type.ToString()); + Assert.Equal(offset, changeEvent.Timestamp); + } +} \ No newline at end of file diff --git a/idou.Core.Tests/idou.Core.Tests.csproj b/idou.Core.Tests/idou.Core.Tests.csproj index b1b110f..a9fbde5 100644 --- a/idou.Core.Tests/idou.Core.Tests.csproj +++ b/idou.Core.Tests/idou.Core.Tests.csproj @@ -10,6 +10,7 @@ + From 5998bd4a56d88558ccd0e0844ff2f3651a336bf6 Mon Sep 17 00:00:00 2001 From: Joshua Hegedus Date: Sun, 28 Dec 2025 23:20:43 +0100 Subject: [PATCH 16/16] finished the tests --- idou.Core.Tests/Domain/ChangeEventTests.cs | 16 ++++++++++++++-- idou.Core/Capabilities/CapabilitySet.cs | 11 ----------- 2 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 idou.Core/Capabilities/CapabilitySet.cs diff --git a/idou.Core.Tests/Domain/ChangeEventTests.cs b/idou.Core.Tests/Domain/ChangeEventTests.cs index 9776ad5..203b703 100644 --- a/idou.Core.Tests/Domain/ChangeEventTests.cs +++ b/idou.Core.Tests/Domain/ChangeEventTests.cs @@ -8,14 +8,23 @@ public class ChangeEventTests [Fact] public void ChangeEvent_Creation_Works() { - var upsertMode = ChangeOperation.Upsert; + const ChangeOperation upsertMode = ChangeOperation.Upsert; var offset = DateTimeOffset.UtcNow; + var payload = new EntityRecord + { + Type = new EntityType("TestType"), + Key = new EntityKey("TestKey"), + Attributes = new Dictionary() + }; var changeEvent = new ChangeEvent(offset) { Operation = upsertMode, Key = new EntityKey("TestKey"), - Type = new EntityType("TestType") + Type = new EntityType("TestType"), + Version = "1.0.0", + Payload = payload, + Metadata = new Dictionary() }; Assert.NotNull(changeEvent); @@ -23,5 +32,8 @@ public void ChangeEvent_Creation_Works() Assert.Equal("TestKey", changeEvent.Key.Value); Assert.Equal("TestType", changeEvent.Type.ToString()); Assert.Equal(offset, changeEvent.Timestamp); + Assert.Equal("1.0.0", changeEvent.Version); + Assert.Equal(payload, changeEvent.Payload); + Assert.NotNull(changeEvent.Metadata); } } \ No newline at end of file diff --git a/idou.Core/Capabilities/CapabilitySet.cs b/idou.Core/Capabilities/CapabilitySet.cs deleted file mode 100644 index 27dd041..0000000 --- a/idou.Core/Capabilities/CapabilitySet.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace idou.Core.Capabilities; - -public class CapabilitySet -{ - public bool SupportsChangeFeed { get; init; } - public bool SupportsSnapshotRead { get; init; } - public bool SupportsBulkWrite { get; init; } - public bool SupportsTransactions { get; init; } - public bool SupportsConditionalWrite { get; init; } - -} \ No newline at end of file