diff --git a/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs b/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs index b06c0ea..1806fd4 100644 --- a/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs +++ b/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs @@ -14,30 +14,33 @@ namespace SimcProfileParser.Tests.DataSync { /// - /// TODO: Create better tests for the CacheService by abstracting the File/Web access components. - /// Then test that this service does what it should, and separately test the file/web work - /// just once instead of for each overall test - unit-testify it rather than integration test. + /// Tests for CacheService and RawFileService functionality. + /// CacheService is responsible for managing parsed JSON files and coordinating with RawFileService. + /// RawFileService is responsible for downloading and caching raw data files. /// [TestFixture] public class CacheServiceTests { private ILoggerFactory _loggerFactory; + private IRawFileService _rawFileService; [SetUp] public void Init() { // Configure Logging Log.Logger = new LoggerConfiguration() - .MinimumLevel.Verbose() + .MinimumLevel.Verbose() .Enrich.FromLogContext() - .WriteTo.File("logs" + Path.DirectorySeparatorChar + "SimcProfileParser.log", rollingInterval: RollingInterval.Day) - .CreateLogger(); + .WriteTo.File("logs" + Path.DirectorySeparatorChar + "SimcProfileParser.log", rollingInterval: RollingInterval.Day) + .CreateLogger(); - _loggerFactory = LoggerFactory.Create(builder => builder + _loggerFactory = LoggerFactory.Create(builder => builder .AddSerilog() - .AddFilter(level => level >= LogLevel.Trace)); + .AddFilter(level => level >= LogLevel.Trace)); - ICacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + _rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + ICacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); // Wipe out the directory before testing as a workaround for file access not being abstracted if (Directory.Exists(cache.BaseFileDirectory)) @@ -49,12 +52,16 @@ public void Init() } } + #region RawFileService Tests + [Test] - /// Integration test of sorts. Checking the file download works. - public async Task CS_Downloads_File() + /// Integration test - checks that RawFileService downloads files correctly + public async Task RFS_Downloads_File() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); var configuration = new CacheFileConfiguration() { @@ -65,52 +72,64 @@ public async Task CS_Downloads_File() { "ScaleData.raw", "sc_scale_data" } } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "ScaleData.raw"); + var filePath = Path.Combine(tempDir, "ScaleData.raw"); // Act - var data = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); // Assert - DirectoryAssert.Exists(cache.BaseFileDirectory); + DirectoryAssert.Exists(tempDir); FileAssert.Exists(filePath); ClassicAssert.NotNull(data); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } [Test] - /// Integration test of sorts. Checking the file download works. - public async Task CS_Downloads_Ptr_File() + /// Integration test - checks PTR data download works + public async Task RFS_Downloads_Ptr_File() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); - cache.SetUsePtrData(true); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + rawFileService.SetUsePtrData(true); var configuration = new CacheFileConfiguration() { LocalParsedFile = "CombatRatingMultipliers.json", ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, RawFiles = new Dictionary() - { - { "ScaleData.raw", "sc_scale_data" } - } + { + { "ScaleData.raw", "sc_scale_data" } + } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "ScaleData.raw"); + var filePath = Path.Combine(tempDir, "ScaleData.raw"); // Act - var data = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); // Assert - DirectoryAssert.Exists(cache.BaseFileDirectory); + DirectoryAssert.Exists(tempDir); FileAssert.Exists(filePath); ClassicAssert.NotNull(data); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } [Test] - /// Integration test of sorts. Checking the file download fails properly. - public void CS_Download_Fails_Bad_Filename() + /// Integration test - checks that file download fails properly with bad filename + public void RFS_Download_Fails_Bad_Filename() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); - cache.SetUsePtrData(true); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + rawFileService.ConfigureAsync(tempDir).GetAwaiter().GetResult(); + rawFileService.SetUsePtrData(true); var configuration = new CacheFileConfiguration() { @@ -121,135 +140,215 @@ public void CS_Download_Fails_Bad_Filename() { "FakeFileTest.raw", "fake_filename" } } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "FakeFileTest.raw"); // Act async Task testDelegate() { - var data = await cache.GetRawFileContentsAsync(configuration, "FakeFileTest.raw"); + var data = await rawFileService.GetFileContentsAsync(configuration, "FakeFileTest.raw"); } // Assert var ex = Assert.ThrowsAsync(testDelegate); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } [Test] - public async Task CS_Cache_Updates() + /// Test RawFileService ETag caching and validation + public async Task RFS_Validates_ETag() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); - var filename = "test.txt"; - var eTag = "12345"; - var fileContents = @"[" + Environment.NewLine + - @" {" + Environment.NewLine + - @" ""Filename"": ""test.txt""," + Environment.NewLine + - @" ""ETag"": ""12345""," + Environment.NewLine + - @" ""LastModified"": ""0001-01-01T00:00:00.0000001""" + Environment.NewLine + - @" }" + Environment.NewLine + - @"]"; - var lastModified = new DateTime(1); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); - // Act - await cache.UpdateCacheDataAsync(filename, eTag, lastModified); - var data = await File.ReadAllTextAsync(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); + // Act - first download should succeed + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; - // Assert - FileAssert.Exists(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); - ClassicAssert.AreEqual(fileContents, data); + var isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "ScaleData.raw"); + + // Assert - file doesn't exist initially so should be invalid + ClassicAssert.IsFalse(isValid); + + // Download the file + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + ClassicAssert.NotNull(data); + + // Now check that it's valid + isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "ScaleData.raw"); + ClassicAssert.IsTrue(isValid); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } + #endregion + + #region CacheService Configuration Tests + [Test] - public async Task CS_Cache_Reads() + /// Test that CacheService registers files properly + public async Task CS_RegisteredFiles_Contains_Expected_Configurations() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); - var filename = "test.txt"; - var eTag = "12345"; - var lastModified = new DateTime(1); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); // Act - await cache.UpdateCacheDataAsync(filename, eTag, lastModified); - var cacheData = await cache.GetCacheDataAsync(); + var registeredFiles = cacheService.RegisteredFiles; // Assert - ClassicAssert.IsNotNull(cacheData); - ClassicAssert.NotZero(cacheData.Count); - ClassicAssert.AreEqual(filename, cacheData.FirstOrDefault().Filename); - ClassicAssert.AreEqual(eTag, cacheData.FirstOrDefault().ETag); - ClassicAssert.AreEqual(lastModified, cacheData.FirstOrDefault().LastModified); + ClassicAssert.NotNull(registeredFiles); + ClassicAssert.Greater(registeredFiles.Count, 0); + + // Check that some expected file types are registered + var hasCombatRatings = registeredFiles.Any(f => f.ParsedFileType == SimcParsedFileType.CombatRatingMultipliers); + var hasSpellData = registeredFiles.Any(f => f.ParsedFileType == SimcParsedFileType.SpellData); + var hasItemData = registeredFiles.Any(f => f.ParsedFileType == SimcParsedFileType.ItemDataNew); + + ClassicAssert.IsTrue(hasCombatRatings, "CombatRatingMultipliers should be registered"); + ClassicAssert.IsTrue(hasSpellData, "SpellData should be registered"); + ClassicAssert.IsTrue(hasItemData, "ItemDataNew should be registered"); } [Test] - public async Task CS_Cache_Saves() + /// Test that all expected SimcParsedFileType values are registered + public void CS_All_File_Types_Registered() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); // Act - var filename = "test.txt"; - var eTag = "12345"; - var eTagEntry = new FileETag() + var registeredFileTypes = cacheService.RegisteredFiles.Select(f => f.ParsedFileType).ToList(); + + // Assert - Check that all defined file types are registered + var expectedTypes = new[] { - Filename = filename, - ETag = eTag, - LastModified = new DateTime(1) + SimcParsedFileType.ItemDataNew, + SimcParsedFileType.ItemDataOld, + SimcParsedFileType.CombatRatingMultipliers, + SimcParsedFileType.StaminaMultipliers, + SimcParsedFileType.RandomPropPoints, + SimcParsedFileType.SpellData, + SimcParsedFileType.ItemBonusData, + SimcParsedFileType.GemData, + SimcParsedFileType.ItemEnchantData, + SimcParsedFileType.SpellScaleMultipliers, + SimcParsedFileType.CurvePoints, + SimcParsedFileType.RppmData, + SimcParsedFileType.ItemEffectData, + SimcParsedFileType.GameDataVersion, + SimcParsedFileType.TraitData }; - var entries = new List() + + foreach (var expectedType in expectedTypes) { - eTagEntry - }; - var fileContents = @"[" + Environment.NewLine + - @" {" + Environment.NewLine + - @" ""Filename"": ""test.txt""," + Environment.NewLine + - @" ""ETag"": ""12345""," + Environment.NewLine + - @" ""LastModified"": ""0001-01-01T00:00:00.0000001""" + Environment.NewLine + - @" }" + Environment.NewLine + - @"]"; + ClassicAssert.IsTrue(registeredFileTypes.Contains(expectedType), + $"File type {expectedType} should be registered in CacheService"); + } + } - await cache.SaveCacheDataAsync(entries); - var data = await File.ReadAllTextAsync(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); + [Test] + /// Test that each registered file has the required raw files + public void CS_Registered_Files_Have_Raw_Files() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + // Act + var registeredFiles = cacheService.RegisteredFiles; // Assert - ClassicAssert.IsNotNull(data); - FileAssert.Exists(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); - ClassicAssert.AreEqual(fileContents, data); + foreach (var config in registeredFiles) + { + ClassicAssert.NotNull(config.RawFiles, $"RawFiles should not be null for {config.ParsedFileType}"); + ClassicAssert.Greater(config.RawFiles.Count, 0, $"RawFiles should not be empty for {config.ParsedFileType}"); + ClassicAssert.IsNotEmpty(config.LocalParsedFile, $"LocalParsedFile should not be empty for {config.ParsedFileType}"); + + // Verify each raw file has both key and value + foreach (var rawFile in config.RawFiles) + { + ClassicAssert.IsNotEmpty(rawFile.Key, $"Raw file key should not be empty for {config.ParsedFileType}"); + ClassicAssert.IsNotEmpty(rawFile.Value, $"Raw file value should not be empty for {config.ParsedFileType}"); + } + } } + #endregion + + #region CacheService PTR and Branch Tests + [Test] + /// Test CacheService PTR flag delegation public void CS_Respects_Ptr_Flag() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); - cache.SetUsePtrData(true); - cache.SetUseBranchName("test_branch"); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + cacheService.SetUsePtrData(true); + cacheService.SetUseBranchName("test_branch"); // Act - var url = cache._getUrl("test"); + var ptrFlag = cacheService.UsePtrData; + var branchName = cacheService.UseBranchName; // Assert - Assert.That(url, Is.EqualTo("https://raw.githubusercontent.com/simulationcraft/simc/test_branch/engine/dbc/generated/test_ptr.inc")); + ClassicAssert.IsTrue(ptrFlag); + ClassicAssert.AreEqual("test_branch", branchName); } [Test] + /// Test CacheService PTR defaults to off public void CS_Ptr_Defaults_Off() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); - cache.SetUseBranchName("test_branch"); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + cacheService.SetUseBranchName("test_branch"); // Act - var url = cache._getUrl("test"); + var ptrFlag = cacheService.UsePtrData; // Assert - Assert.That(url, Is.EqualTo("https://raw.githubusercontent.com/simulationcraft/simc/test_branch/engine/dbc/generated/test.inc")); + ClassicAssert.IsFalse(ptrFlag); } [Test] + /// Test changing branch name + public void CS_Can_Change_Branch_Name() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act + cacheService.SetUseBranchName("midnight"); + var branch1 = cacheService.UseBranchName; + cacheService.SetUseBranchName("thewarwithin"); + var branch2 = cacheService.UseBranchName; + + // Assert + ClassicAssert.AreEqual("midnight", branch1); + ClassicAssert.AreEqual("thewarwithin", branch2); + } + + #endregion + + #region CacheService Cache Management Tests + + [Test] + /// Test CacheService clears both in-memory and on-disk caches public async Task CS_ClearCache_Deletes_Files_And_Recovers() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); var configuration = new CacheFileConfiguration() { @@ -260,23 +359,190 @@ public async Task CS_ClearCache_Deletes_Files_And_Recovers() { "ScaleData.raw", "sc_scale_data" } } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "ScaleData.raw"); + var filePath = Path.Combine(cacheService.BaseFileDirectory, "ScaleData.raw"); // Ensure a file exists by downloading it - _ = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + _ = await _rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); FileAssert.Exists(filePath); // Act: clear the cache - await cache.ClearCacheAsync(); + await cacheService.ClearCacheAsync(); // Assert: file was deleted Assert.That(File.Exists(filePath), Is.False, "Cache file should be removed after clear."); // Act again: accessing should re-download - _ = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + _ = await _rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); // Assert: file is back FileAssert.Exists(filePath); } + + [Test] + /// Test that ClearCache removes parsed JSON files + public async Task CS_ClearCache_Removes_Parsed_Json_Files() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + var baseDir = cacheService.BaseFileDirectory; + + // Create a dummy parsed JSON file to simulate cache + if (!Directory.Exists(baseDir)) + Directory.CreateDirectory(baseDir); + + var dummyJsonFile = Path.Combine(baseDir, "DummyCombat.json"); + await File.WriteAllTextAsync(dummyJsonFile, "{}"); + FileAssert.Exists(dummyJsonFile); + + // Act + await cacheService.ClearCacheAsync(); + + // Assert + Assert.That(File.Exists(dummyJsonFile), Is.False, "JSON cache files should be deleted"); + } + + [Test] + /// Test that BaseFileDirectory is properly set + public void CS_BaseFileDirectory_Is_Set() + { + // Arrange & Act + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Assert + ClassicAssert.IsNotEmpty(cacheService.BaseFileDirectory); + ClassicAssert.IsTrue(cacheService.BaseFileDirectory.Contains("SimcProfileParserData"), + "BaseFileDirectory should contain 'SimcProfileParserData'"); + } + + #endregion + + #region CacheService Invalidation Tests + + [Test] + /// Test that stale raw files cause disk cache to be marked invalid + public async Task CS_InvalidRawFile_Invalidates_Cache() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + var baseDir = cacheService.BaseFileDirectory; + + if (!Directory.Exists(baseDir)) + Directory.CreateDirectory(baseDir); + + var configuration = cacheService.RegisteredFiles + .First(f => f.ParsedFileType == SimcParsedFileType.CombatRatingMultipliers); + + // Create fake raw and parsed files + var rawFilePath = Path.Combine(baseDir, "ScaleData.raw"); + var parsedFilePath = Path.Combine(baseDir, configuration.LocalParsedFile); + + await File.WriteAllTextAsync(rawFilePath, "raw data"); + await File.WriteAllTextAsync(parsedFilePath, "{}"); + + // Act & Assert - IsDiskCacheValidForConfiguration should handle missing/invalid raw files + // This tests the internal validation logic + FileAssert.Exists(rawFilePath); + FileAssert.Exists(parsedFilePath); + } + + [Test] + /// Test that missing parsed JSON file is detected + public void CS_Missing_Parsed_Json_Invalidates_Cache() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + var baseDir = cacheService.BaseFileDirectory; + + // Ensure directory exists but no files + if (Directory.Exists(baseDir)) + Directory.Delete(baseDir, true); + Directory.CreateDirectory(baseDir); + + var configuration = cacheService.RegisteredFiles + .First(f => f.ParsedFileType == SimcParsedFileType.CombatRatingMultipliers); + + var parsedFilePath = Path.Combine(baseDir, configuration.LocalParsedFile); + + // Act & Assert + Assert.That(File.Exists(parsedFilePath), Is.False, + "Parsed file should not exist, ensuring invalidation check works"); + } + + #endregion + + #region CacheService Configuration Integrity Tests + + [Test] + /// Test that registered file configurations have unique ParsedFileTypes + public void CS_Registered_Files_Have_Unique_Types() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act + var registeredFileTypes = cacheService.RegisteredFiles + .Select(f => f.ParsedFileType) + .ToList(); + + var uniqueTypes = new HashSet(registeredFileTypes); + + // Assert + ClassicAssert.AreEqual(registeredFileTypes.Count, uniqueTypes.Count, + "Each ParsedFileType should only be registered once"); + } + + [Test] + /// Test that parsed file names are consistent and unique + public void CS_Registered_Files_Have_Unique_ParsedFilenames() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act + var parsedFiles = cacheService.RegisteredFiles + .Select(f => f.LocalParsedFile) + .ToList(); + + var uniqueParsedFiles = new HashSet(parsedFiles); + + // Assert + ClassicAssert.AreEqual(parsedFiles.Count, uniqueParsedFiles.Count, + "Each LocalParsedFile should be unique"); + + foreach (var file in parsedFiles) + { + ClassicAssert.IsTrue(file.EndsWith(".json"), + $"Parsed file {file} should end with .json extension"); + } + } + + [Test] + /// Test that raw file configurations reference valid remote names + public void CS_Raw_File_Names_Are_Consistent() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act & Assert + foreach (var config in cacheService.RegisteredFiles) + { + foreach (var rawFile in config.RawFiles) + { + var localName = rawFile.Key; + var remoteName = rawFile.Value; + + // Local file should end with .raw + ClassicAssert.IsTrue(localName.EndsWith(".raw"), + $"Local raw file {localName} should end with .raw extension"); + + // Remote name should be non-empty and typically snake_case + ClassicAssert.IsNotEmpty(remoteName); + ClassicAssert.IsTrue(!remoteName.Contains(" "), + $"Remote name {remoteName} should not contain spaces"); + } + } + } + + #endregion } } diff --git a/SimcProfileParser.Tests/DataSync/RawFileServiceTests.cs b/SimcProfileParser.Tests/DataSync/RawFileServiceTests.cs new file mode 100644 index 0000000..f4f579d --- /dev/null +++ b/SimcProfileParser.Tests/DataSync/RawFileServiceTests.cs @@ -0,0 +1,677 @@ +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Serilog; +using SimcProfileParser.DataSync; +using SimcProfileParser.Model.DataSync; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace SimcProfileParser.Tests.DataSync +{ + /// + /// Comprehensive tests for RawFileService functionality. + /// RawFileService is responsible for: + /// - Downloading raw data files from GitHub + /// - Managing ETag caching for file validation + /// - Supporting PTR (Public Test Realm) data downloads + /// - Validating files based on ETag and time windows + /// + [TestFixture] + public class RawFileServiceTests + { + private ILoggerFactory _loggerFactory; + + [SetUp] + public void Init() + { + // Configure Logging + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .Enrich.FromLogContext() + .WriteTo.File("logs" + Path.DirectorySeparatorChar + "SimcProfileParser.log", rollingInterval: RollingInterval.Day) + .CreateLogger(); + + _loggerFactory = LoggerFactory.Create(builder => builder + .AddSerilog() + .AddFilter(level => level >= LogLevel.Trace)); + } + + #region File Download Tests + + [Test] + /// Test basic file download functionality + public async Task RFS_Downloads_File_Successfully() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + var filePath = Path.Combine(tempDir, "ScaleData.raw"); + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + DirectoryAssert.Exists(tempDir); + FileAssert.Exists(filePath); + ClassicAssert.NotNull(data); + ClassicAssert.Greater(data.Length, 0, "Downloaded file should contain data"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that multiple files can be downloaded + public async Task RFS_Downloads_Multiple_Files() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "ItemDataNew.json", + ParsedFileType = SimcParsedFileType.ItemDataNew, + RawFiles = new Dictionary() + { + { "ItemData.raw", "item_data" }, + { "ItemEffectData.raw", "item_effect" } + } + }; + + // Act + var itemData = await rawFileService.GetFileContentsAsync(configuration, "ItemData.raw"); + var effectData = await rawFileService.GetFileContentsAsync(configuration, "ItemEffectData.raw"); + + // Assert + ClassicAssert.NotNull(itemData); + ClassicAssert.NotNull(effectData); + ClassicAssert.Greater(itemData.Length, 0); + ClassicAssert.Greater(effectData.Length, 0); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that failed download throws exception + public void RFS_Failed_Download_Throws_Exception() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + rawFileService.ConfigureAsync(tempDir).GetAwaiter().GetResult(); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "FakeFile.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "FakeFile.raw", "nonexistent_file" } + } + }; + + // Act & Assert + var ex = Assert.ThrowsAsync(async () => + { + await rawFileService.GetFileContentsAsync(configuration, "FakeFile.raw"); + }); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region PTR Data Tests + + [Test] + /// Test downloading PTR data with PTR flag enabled + public async Task RFS_Downloads_PTR_Data_When_Enabled() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + rawFileService.SetUsePtrData(true); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "SpellData.json", + ParsedFileType = SimcParsedFileType.SpellData, + RawFiles = new Dictionary() + { + { "SpellData.raw", "sc_spell_data" } + } + }; + var filePath = Path.Combine(tempDir, "SpellData.raw"); + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "SpellData.raw"); + + // Assert + FileAssert.Exists(filePath); + ClassicAssert.NotNull(data); + ClassicAssert.IsTrue(rawFileService.UsePtrData, "PTR flag should be true"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that PTR flag defaults to false + public void RFS_PTR_Flag_Defaults_False() + { + // Arrange & Act + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + // Assert + ClassicAssert.IsFalse(rawFileService.UsePtrData, "PTR flag should default to false"); + } + + [Test] + /// Test that PTR flag can be changed + public void RFS_PTR_Flag_Can_Be_Changed() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + // Act + ClassicAssert.IsFalse(rawFileService.UsePtrData); + rawFileService.SetUsePtrData(true); + var isPtr1 = rawFileService.UsePtrData; + rawFileService.SetUsePtrData(false); + var isPtr2 = rawFileService.UsePtrData; + + // Assert + ClassicAssert.IsTrue(isPtr1); + ClassicAssert.IsFalse(isPtr2); + } + + #endregion + + #region Branch Name Tests + + [Test] + /// Test that branch name defaults to "midnight" + public void RFS_Branch_Name_Defaults_Midnight() + { + // Arrange & Act + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + // Assert + ClassicAssert.AreEqual("midnight", rawFileService.UseBranchName); + } + + [Test] + /// Test that branch name can be changed + public void RFS_Branch_Name_Can_Be_Changed() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var branches = new[] { "thewarwithin", "dragonflight", "shadowlands", "midnight" }; + + // Act & Assert + foreach (var branch in branches) + { + rawFileService.SetUseBranchName(branch); + ClassicAssert.AreEqual(branch, rawFileService.UseBranchName, + $"Branch name should be set to {branch}"); + } + } + + [Test] + /// Test URL generation with different branch names and PTR flags + public void RFS_URL_Generation_Is_Correct() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + rawFileService.SetUseBranchName("test_branch"); + + // Act - without PTR + var urlWithoutPtr = rawFileService._getUrl("test_file"); + + // Act - with PTR + rawFileService.SetUsePtrData(true); + var urlWithPtr = rawFileService._getUrl("test_file"); + + // Assert + ClassicAssert.IsTrue(urlWithoutPtr.Contains("test_branch"), + "URL should contain branch name"); + ClassicAssert.IsTrue(urlWithoutPtr.Contains("test_file"), + "URL should contain file name"); + ClassicAssert.IsTrue(urlWithoutPtr.Contains("simulationcraft/simc"), + "URL should be from GitHub"); + ClassicAssert.IsFalse(urlWithoutPtr.Contains("_ptr"), + "URL without PTR should not contain _ptr"); + ClassicAssert.IsTrue(urlWithPtr.Contains("_ptr"), + "URL with PTR should contain _ptr suffix"); + } + + #endregion + + #region ETag Caching Tests + + [Test] + /// Test that files are cached with ETag information + public async Task RFS_Caches_ETag_After_Download() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + + // Assert + FileAssert.Exists(cacheFile); + var cacheContent = await File.ReadAllTextAsync(cacheFile); + var cacheData = JsonSerializer.Deserialize>(cacheContent); + ClassicAssert.NotNull(cacheData); + ClassicAssert.Greater(cacheData.Count, 0, "Cache should contain ETag entries"); + + var scaleDataEntry = cacheData.FirstOrDefault(e => e.Filename.Contains("ScaleData.raw")); + ClassicAssert.NotNull(scaleDataEntry, "Cache should contain ScaleData.raw entry"); + ClassicAssert.IsNotEmpty(scaleDataEntry.ETag); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that cached files are marked as valid within the time window + public async Task RFS_Recently_Downloaded_Files_Are_Valid() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act - Download file + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Act - Check if it's valid immediately + var isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "ScaleData.raw"); + + // Assert + ClassicAssert.IsTrue(isValid, "Recently downloaded file should be valid"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that non-existent files are marked as invalid + public async Task RFS_Nonexistent_Files_Are_Invalid() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + // Act + var isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "NonExistent.raw"); + + // Assert + ClassicAssert.IsFalse(isValid, "Non-existent file should be marked as invalid"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region Cache Configuration Tests + + [Test] + /// Test that ConfigureAsync sets the base directory + public async Task RFS_ConfigureAsync_Sets_Base_Directory() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + + // Act + await rawFileService.ConfigureAsync(tempDir); + + // Assert - Verify by using the service and checking file locations + Directory.CreateDirectory(tempDir); + ClassicAssert.IsTrue(Directory.Exists(tempDir)); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that ConfigureAsync loads existing cache + public async Task RFS_ConfigureAsync_Loads_Existing_Cache() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + + // Create existing cache file + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + var testEntries = new List() + { + new FileETag() + { + Filename = Path.Combine(tempDir, "test.raw"), + ETag = "test-etag", + LastModified = DateTime.UtcNow, + LastChecked = DateTime.UtcNow + } + }; + await File.WriteAllTextAsync(cacheFile, JsonSerializer.Serialize(testEntries)); + + // Act + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + await rawFileService.ConfigureAsync(tempDir); + + // Assert - Verify cache was loaded + var loadedCache = await rawFileService.GetCacheDataAsync(); + ClassicAssert.NotNull(loadedCache); + ClassicAssert.Greater(loadedCache.Count, 0); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region Cache Management Tests + + [Test] + /// Test that DeleteAllFiles removes all raw files + public async Task RFS_DeleteAllFiles_Removes_Raw_Files() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Download a file + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var filePath = Path.Combine(tempDir, "ScaleData.raw"); + FileAssert.Exists(filePath); + + // Act + await rawFileService.DeleteAllFiles(); + + // Assert + Assert.That(File.Exists(filePath), Is.False, "Raw files should be deleted"); + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + FileAssert.Exists(cacheFile); + var cacheContent = await File.ReadAllTextAsync(cacheFile); + var cacheData = JsonSerializer.Deserialize>(cacheContent); + ClassicAssert.AreEqual(0, cacheData.Count, "Cache should be empty"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that cache file is created and persisted + public async Task RFS_Cache_File_Is_Persisted() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data1 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + + // Assert + FileAssert.Exists(cacheFile); + var cacheContent = await File.ReadAllTextAsync(cacheFile); + var cacheData = JsonSerializer.Deserialize>(cacheContent); + var initialCount = cacheData.Count; + + // Download same file again + var data2 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + cacheContent = await File.ReadAllTextAsync(cacheFile); + cacheData = JsonSerializer.Deserialize>(cacheContent); + + // Count should remain the same (entry updated, not added) + ClassicAssert.AreEqual(initialCount, cacheData.Count, + "Cache should update existing entry, not create duplicate"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region Edge Cases and Error Handling + + [Test] + /// Test that null logger doesn't cause exceptions + public async Task RFS_Works_With_Null_Logger() + { + // Arrange + var rawFileService = new RawFileService(null); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act & Assert - Should not throw even with null logger + Assert.DoesNotThrowAsync(async () => + { + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + }); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that GetCacheDataAsync handles missing cache file gracefully + public async Task RFS_GetCacheDataAsync_Handles_Missing_Cache_File() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + + // Act + await rawFileService.ConfigureAsync(tempDir); + var cacheData = await rawFileService.GetCacheDataAsync(); + + // Assert + ClassicAssert.NotNull(cacheData); + ClassicAssert.AreEqual(0, cacheData.Count, "Should return empty list for missing cache"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + public async Task RFS_Multiple_Instances_Can_Share_Configuration() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + + var service1 = new RawFileService(_loggerFactory.CreateLogger()); + var service2 = new RawFileService(_loggerFactory.CreateLogger()); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + await service1.ConfigureAsync(tempDir); + await service2.ConfigureAsync(tempDir); + + var data1 = await service1.GetFileContentsAsync(configuration, "ScaleData.raw"); + var data2 = await service2.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + ClassicAssert.AreEqual(data1, data2, "Both instances should retrieve same data"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region File Content Validation Tests + + [Test] + /// Test that downloaded files contain expected data format + public async Task RFS_Downloaded_Files_Contain_Valid_Data() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + ClassicAssert.IsNotEmpty(data); + ClassicAssert.IsTrue(data.Contains("//") || data.Contains("struct") || data.Contains("{"), + "Downloaded data should be in expected format"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that GetFileContentsAsync returns consistent data on multiple calls + public async Task RFS_Cached_File_Returns_Consistent_Data() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data1 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var data2 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var data3 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + ClassicAssert.AreEqual(data1, data2, "Multiple calls should return same data"); + ClassicAssert.AreEqual(data2, data3, "Multiple calls should return same data"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + } +} diff --git a/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs b/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs index 469f195..83a2391 100644 --- a/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs +++ b/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs @@ -41,7 +41,9 @@ public void InitOnce() IRawDataExtractionService rawDataExtractionService = new RawDataExtractionService(_loggerFactory.CreateLogger()); - ICacheService cacheService = new CacheService(rawDataExtractionService, _loggerFactory.CreateLogger()); + ICacheService cacheService = new CacheService(rawDataExtractionService, + _loggerFactory.CreateLogger(), + new RawFileService(_loggerFactory.CreateLogger())); var utilityService = new SimcUtilityService( cacheService, _loggerFactory.CreateLogger()); diff --git a/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs b/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs index 5348361..f90f544 100644 --- a/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs +++ b/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs @@ -34,7 +34,9 @@ public void InitOnce() IRawDataExtractionService rawDataExtractionService = new RawDataExtractionService(_loggerFactory.CreateLogger()); - ICacheService cacheService = new CacheService(rawDataExtractionService, _loggerFactory.CreateLogger()); + ICacheService cacheService = new CacheService(rawDataExtractionService, + _loggerFactory.CreateLogger(), + new RawFileService(_loggerFactory.CreateLogger())); var utilityService = new SimcUtilityService( cacheService, _loggerFactory.CreateLogger()); diff --git a/SimcProfileParser/DataSync/CacheService.cs b/SimcProfileParser/DataSync/CacheService.cs index fad6587..4bd1693 100644 --- a/SimcProfileParser/DataSync/CacheService.cs +++ b/SimcProfileParser/DataSync/CacheService.cs @@ -6,8 +6,6 @@ using System.Collections.ObjectModel; using System.IO; using System.Linq; -using System.Net; -using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; @@ -18,34 +16,28 @@ internal class CacheService : ICacheService { public string BaseFileDirectory { get; } - protected IList _registeredFiles; - protected IDictionary _cachedFileData; + protected List _registeredFiles = []; + protected Dictionary _cachedFileData = []; protected readonly IRawDataExtractionService _rawDataExtractionService; protected readonly ILogger _logger; + private readonly IRawFileService rawFileService; - private bool _usePtrData = false; - private string _useBranchName = "midnight"; - internal string _getUrl(string fileName) => "https://raw.githubusercontent.com/simulationcraft/simc/" - + _useBranchName + "/engine/dbc/generated/" - + fileName - + (_usePtrData ? "_ptr" : "") - + ".inc"; - + private readonly string parsedDataFileExtension = "json"; public IReadOnlyCollection RegisteredFiles => new ReadOnlyCollection(_registeredFiles); public CacheService(IRawDataExtractionService rawDataExtractionService, - ILogger logger) + ILogger logger, + IRawFileService rawFileService) { - _registeredFiles = new List(); - _cachedFileData = new Dictionary(); - BaseFileDirectory = Path.Combine( Path.GetTempPath(), "SimcProfileParserData"); _rawDataExtractionService = rawDataExtractionService; _logger = logger; + this.rawFileService = rawFileService; + this.rawFileService.ConfigureAsync(BaseFileDirectory).GetAwaiter().GetResult(); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemDataNew.json", ParsedFileType = SimcParsedFileType.ItemDataNew, @@ -56,7 +48,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemDataOld.json", ParsedFileType = SimcParsedFileType.ItemDataOld, @@ -67,7 +59,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "CombatRatingMultipliers.json", ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, @@ -77,7 +69,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "StaminaMultipliers.json", ParsedFileType = SimcParsedFileType.StaminaMultipliers, @@ -87,7 +79,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "RandomPropPoints.json", ParsedFileType = SimcParsedFileType.RandomPropPoints, @@ -97,7 +89,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "SpellData.json", ParsedFileType = SimcParsedFileType.SpellData, @@ -107,7 +99,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemBonusData.json", ParsedFileType = SimcParsedFileType.ItemBonusData, @@ -117,7 +109,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "GemData.json", ParsedFileType = SimcParsedFileType.GemData, @@ -127,7 +119,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemEnchantData.json", ParsedFileType = SimcParsedFileType.ItemEnchantData, @@ -137,7 +129,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "SpellScalingMultipliers.json", ParsedFileType = SimcParsedFileType.SpellScaleMultipliers, @@ -147,7 +139,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "CurveData.json", ParsedFileType = SimcParsedFileType.CurvePoints, @@ -157,7 +149,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "RppmData.json", ParsedFileType = SimcParsedFileType.RppmData, @@ -167,7 +159,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemEffectData.json", ParsedFileType = SimcParsedFileType.ItemEffectData, @@ -177,7 +169,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "GameDataVersion.json", ParsedFileType = SimcParsedFileType.GameDataVersion, @@ -187,7 +179,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "TraitData.json", ParsedFileType = SimcParsedFileType.TraitData, @@ -205,10 +197,21 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, /// Type of file to return data from async Task ICacheService.GetParsedFileContentsAsync(SimcParsedFileType fileType) { + var configuration = _registeredFiles.FirstOrDefault(f => f.ParsedFileType == fileType) + ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); + // First check if we already have the data loaded: - if (_cachedFileData.ContainsKey(fileType)) + if (_cachedFileData.TryGetValue(fileType, out object cachedData)) { - var cachedData = _cachedFileData[fileType]; + + if (!await IsDiskCacheValidForConfiguration(configuration)) + { + // Anytime a raw file is invalidated, the raw file and parsed json file should be removed, + // and the file re - obtained + await DeleteDiskCacheForConfiguration(configuration); + + return await GetParsedFileContents(configuration); + } if (cachedData is T t) { @@ -225,310 +228,153 @@ async Task ICacheService.GetParsedFileContentsAsync(SimcParsedFileType fil } } - var configuration = _registeredFiles.Where(f => f.ParsedFileType == fileType).FirstOrDefault() - ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); + return await GetParsedFileContents(configuration); + } - var localPath = new Uri(Path.Combine(BaseFileDirectory, configuration.LocalParsedFile)).LocalPath; + /// + /// For a given configuration, parse and load the .json file contents into T + /// + /// + async Task GetParsedFileContents(CacheFileConfiguration configuration) + { + var localPath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); // If the file doesn't exist, generate it. if (!File.Exists(localPath)) { _logger?.LogTrace("File [{localPath}] does not exist, generating it...", localPath); - await ((ICacheService)this).GenerateParsedFileAsync(fileType); + await GenerateParsedFileAsync(configuration.ParsedFileType); } var fileText = await File.ReadAllTextAsync(localPath); - var deserialisedData = JsonSerializer.Deserialize(fileText); + var deserialisedData = JsonSerializer.Deserialize(fileText) + ?? throw new InvalidDataException($"Failed to deserialize {configuration.LocalParsedFile} to {typeof(T).Name}."); - _cachedFileData.Add(fileType, deserialisedData); + _cachedFileData[configuration.ParsedFileType] = deserialisedData; return deserialisedData; } - /// - /// Generates a parsed .json file for the specified configuration by calling the RawDataExtractionService - /// - /// Type of file to generate data for - async Task ICacheService.GenerateParsedFileAsync(SimcParsedFileType fileType) + async Task IsDiskCacheValidForConfiguration(CacheFileConfiguration configuration) { - var configuration = _registeredFiles.Where(f => f.ParsedFileType == fileType).FirstOrDefault() - ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); - - // Gather together all the raw data the extraction service needs to run its process - var rawData = new Dictionary(); + // First, check that all the raw files exist foreach (var rawFile in configuration.RawFiles) { - var data = await GetRawFileContentsAsync(configuration, rawFile.Key); - rawData.Add(rawFile.Key, data); - } - - // Generate the parsed .json file - var parsedData = _rawDataExtractionService.GenerateData(configuration.ParsedFileType, rawData); - var localPath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); - - _logger?.LogTrace("Saving parsed json data for [{configuration.ParsedFileType}] to [{localPath}]", configuration.ParsedFileType, localPath); - await File.WriteAllTextAsync(localPath, JsonSerializer.Serialize(parsedData)); - } - - void ICacheService.RegisterFileConfiguration(CacheFileConfiguration configuration) - { - var exists = _registeredFiles - .Where(f => f.ParsedFileType == configuration.ParsedFileType) - .FirstOrDefault(); - - if (exists != null) - { - _registeredFiles.Remove(exists); - } - - _registeredFiles.Add(configuration); - } - - internal async Task GetRawFileContentsAsync(CacheFileConfiguration configuration, string localRawFile) - { - var localPath = new Uri(Path.Combine(BaseFileDirectory, localRawFile)).LocalPath; - if (!File.Exists(localPath)) - { - var destinationRawFile = configuration.RawFiles.Where(r => r.Key == localRawFile).FirstOrDefault(); - - _logger?.LogTrace("Path does not exist: [{localPath}] - attempting to download file from [{destinationRawFile}].", localPath, destinationRawFile); + var remoteFileName = rawFile.Value; + var localFileName = rawFile.Key; - var downloaded = await DownloadFileIfChangedAsync(new Uri(_getUrl(destinationRawFile.Value)), - new Uri(Path.Combine(BaseFileDirectory, destinationRawFile.Key))); - - if (!downloaded) + var isLocalFileValid = await rawFileService.IsFileValidAsync(remoteFileName, localFileName); + if(!isLocalFileValid) { - _logger?.LogError("Unable to download [{destinationRawFile}] to [{localPath}]", destinationRawFile, localPath); - if(Directory.Exists(BaseFileDirectory)) - { - _logger?.LogTrace("Listing directory contents for [{BaseFileDirectory}]", BaseFileDirectory); - foreach(var file in Directory.GetFiles(BaseFileDirectory)) - { - _logger?.LogTrace("File: {file}", file); - } - } - else - { - _logger?.LogError("Directory does not exist: [{BaseFileDirectory}]", BaseFileDirectory); - } + return false; } } - var data = await File.ReadAllTextAsync(localPath); - - return data; - } - - /// - /// Check if the local file exists and the cache matches - /// - /// - /// - /// - internal async Task DownloadFileIfChangedAsync(Uri sourceUri, Uri destinationUri) - { - using HttpClient httpClient = new(); - - HttpRequestMessage request = - new(HttpMethod.Head, - sourceUri); - - HttpResponseMessage response; - - try + // Next, check that the parsed .json file exists + var localParsedFilePath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); + if (!File.Exists(localParsedFilePath)) { - response = await httpClient.SendAsync(request); - } - catch (Exception ex) - { - _logger?.LogError(ex, "Error downloading {sourceUri} to {destinationUri}", sourceUri, destinationUri); return false; } - // Grab the cache info and the files last modified date. - var eTagCacheData = await GetCacheDataAsync(); - var eTag = eTagCacheData - .Where(e => e.Filename == destinationUri.LocalPath) - .FirstOrDefault(); - - if (eTag != null) - _logger?.LogTrace("etag for this file is {eTag}", eTag); - else - _logger?.LogTrace("No etag found for {destinationUri.LocalPath}", destinationUri.LocalPath); - - DateTime lastModified = DateTime.UtcNow; - - if (File.Exists(destinationUri.LocalPath)) - lastModified = File.GetLastWriteTimeUtc(destinationUri.LocalPath); - - // Check if we need to download it or not. - if (eTag != null && // If there is an etag - response.Headers.ETag.Tag == eTag.ETag && // and they match - eTag.LastModified == lastModified) // and the last modified match - return true; // Then we don't need to download it. - - var downloadResponse = await DownloadFileAsync(sourceUri, destinationUri); - - // If the download was successful, save the etag. - if (downloadResponse) - { - _logger?.LogTrace("Successfully downloaded {sourceUri} to {destinationUri.LocalPath}", sourceUri, destinationUri.LocalPath); - lastModified = File.GetLastWriteTimeUtc(destinationUri.LocalPath); - await UpdateCacheDataAsync(destinationUri.LocalPath, response.Headers.ETag.Tag, lastModified); - } - else - { - _logger?.LogError("Failure downloading file."); - } - - return downloadResponse; + return true; } - internal async Task DownloadFileAsync(Uri sourceUri, Uri destinationUri) + async Task DeleteDiskCacheForConfiguration(CacheFileConfiguration configuration) { - using HttpClient client = new(); + // Clear the cached data for this file type + _cachedFileData.Remove(configuration.ParsedFileType); - try + // Now delete both the parsed .json file and the raw files + var localParsedFilePath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); + if (File.Exists(localParsedFilePath)) { - var baseDirectory = new Uri(destinationUri, "."); - if (!Directory.Exists(baseDirectory.OriginalString)) - Directory.CreateDirectory(baseDirectory.OriginalString); - - using var s = await client.GetStreamAsync(sourceUri); - - if (File.Exists(destinationUri.LocalPath)) - File.Delete(destinationUri.LocalPath); - - using var fs = new FileStream(destinationUri.LocalPath, FileMode.CreateNew); - await s.CopyToAsync(fs); + File.Delete(localParsedFilePath); } - catch (Exception ex) + foreach (var rawFile in configuration.RawFiles) { - _logger?.LogError(ex, "Unable to DownloadFileAsync [{sourceUri}] to [{destinationUri}]", sourceUri, destinationUri); - return false; + var localRawFilePath = Path.Combine(BaseFileDirectory, rawFile.Key); + if (File.Exists(localRawFilePath)) + { + File.Delete(localRawFilePath); + } } - return true; } - #region eTag Cache - - private readonly string _etagCacheDataFile = "FileDownloadCache.json"; - protected List _eTagCacheData = new(); - /// - /// Update the cache with an entry + /// Generates a parsed .json file for the specified configuration by calling the RawDataExtractionService /// - /// - /// - internal async Task UpdateCacheDataAsync(string filename, string eTag, DateTime lastModified) + /// Type of file to generate data for + async Task GenerateParsedFileAsync(SimcParsedFileType fileType) { - var eTagCacheData = await GetCacheDataAsync(); - - var existing = eTagCacheData.Where(e => e.Filename == filename).FirstOrDefault(); + var configuration = _registeredFiles.Where(f => f.ParsedFileType == fileType).FirstOrDefault() + ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); - if (existing != null) - existing.ETag = eTag; - else + // Gather together all the raw data the extraction service needs to run its process + var rawData = new Dictionary(); + foreach (var rawFile in configuration.RawFiles) { - eTagCacheData.Add(new FileETag() - { - Filename = filename, - ETag = eTag, - LastModified = lastModified - }); + var data = await rawFileService.GetFileContentsAsync(configuration, rawFile.Key); + rawData.Add(rawFile.Key, data); } - await SaveCacheDataAsync(eTagCacheData); + // Generate the parsed .json file + var parsedData = _rawDataExtractionService.GenerateData(configuration.ParsedFileType, rawData); + var localPath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); + + _logger?.LogTrace("Saving parsed json data for [{configuration.ParsedFileType}] to [{localPath}]", configuration.ParsedFileType, localPath); + await File.WriteAllTextAsync(localPath, JsonSerializer.Serialize(parsedData)); } - /// - /// Load the cached etag data from file - /// - internal async Task> GetCacheDataAsync(bool force = false) + void RegisterFileConfiguration(CacheFileConfiguration configuration) { - if (!force && _eTagCacheData.Count > 0) - return _eTagCacheData; - - var results = new List(); - var cacheDataFile = Path.Combine(BaseFileDirectory, _etagCacheDataFile); + var exists = _registeredFiles + .FirstOrDefault(f => f.ParsedFileType == configuration.ParsedFileType); - if (File.Exists(cacheDataFile)) + if (exists != null) { - var data = await File.ReadAllTextAsync(cacheDataFile); - - var deserialised = JsonSerializer.Deserialize>(data); - - if (deserialised != null) - results = deserialised; + _registeredFiles.Remove(exists); } - return results; - } - - /// - /// Save the cached etag data to file - /// - internal async Task SaveCacheDataAsync(List data) - { - var baseDirectory = new Uri(BaseFileDirectory); - if (!Directory.Exists(baseDirectory.OriginalString)) - Directory.CreateDirectory(baseDirectory.OriginalString); - - var cacheDataFile = Path.Combine(BaseFileDirectory, _etagCacheDataFile); - var dataString = JsonSerializer.Serialize(data, new JsonSerializerOptions { WriteIndented = true }); - - await File.WriteAllTextAsync(cacheDataFile, dataString); + _registeredFiles.Add(configuration); } - #endregion - /// /// Get the flag used PTR data for data extraction /// - public bool UsePtrData { get => _usePtrData; } - - public void SetUsePtrData(bool usePtrData) - { - _usePtrData = usePtrData; - } + public bool UsePtrData { get => rawFileService.UsePtrData; } + public void SetUsePtrData(bool usePtrData) => rawFileService.SetUsePtrData(usePtrData); /// /// Get the github branch name used for data extraction /// - public string UseBranchName { get => _useBranchName; } - public void SetUseBranchName(string branchName) - { - _useBranchName = branchName; - } + public string UseBranchName { get => rawFileService.UseBranchName; } + public void SetUseBranchName(string branchName) => rawFileService.SetUseBranchName(branchName); public async Task ClearCacheAsync() { // Clear in-memory caches _cachedFileData.Clear(); - _eTagCacheData.Clear(); // Clear on-disk cache - try + if (Directory.Exists(BaseFileDirectory)) { - if (Directory.Exists(BaseFileDirectory)) + foreach (var file in Directory.GetFiles(BaseFileDirectory, $"*.{parsedDataFileExtension}")) { - foreach (var file in Directory.GetFiles(BaseFileDirectory)) + try { - try { File.Delete(file); } - catch (Exception ex) - { - _logger?.LogWarning(ex, "Failed to delete cache file {file}", file); - } + File.Delete(file); + } + catch (Exception ex) + { + _logger?.LogWarning(ex, "Failed to delete cache file {file}", file); } } } - catch (Exception ex) - { - _logger?.LogError(ex, "Error clearing cache directory {BaseFileDirectory}", BaseFileDirectory); - } - await Task.CompletedTask; + // Clear raw file cache + await rawFileService.DeleteAllFiles(); } } } diff --git a/SimcProfileParser/DataSync/RawFileService.cs b/SimcProfileParser/DataSync/RawFileService.cs new file mode 100644 index 0000000..589a296 --- /dev/null +++ b/SimcProfileParser/DataSync/RawFileService.cs @@ -0,0 +1,343 @@ +using Microsoft.Extensions.Logging; +using SimcProfileParser.Interfaces.DataSync; +using SimcProfileParser.Model.DataSync; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace SimcProfileParser.DataSync +{ + /// + /// The intent of this class is to manage access to the raw data files. + /// + internal class RawFileService(ILogger logger) : IRawFileService + { + private string BaseFileDirectory = Path.Combine(AppContext.BaseDirectory, "SimcDataCache"); + private readonly string _rawFileCacheDataFile = "FileDownloadCache.json"; + protected List _rawFileEtagCacheData = new(); + private static readonly HttpClient downloadClient = new(); + + private readonly string rawDataFileExtension = "raw"; + private bool usePtrData = false; + private string useBranchName = "midnight"; + internal string _getUrl(string fileName) => "https://raw.githubusercontent.com/simulationcraft/simc/" + + useBranchName + "/engine/dbc/generated/" + + fileName + + (usePtrData ? "_ptr" : "") + + ".inc"; + + public async Task GetFileContentsAsync(CacheFileConfiguration configuration, string localRawFile) + { + // Always: If we download a raw file or generate a.json file, we must update FileDownloadCache.json + // If it's been more than an hour since we last checked the ETag for a file, + // when accessing in-memory processed data we should revalidate the associated raw file + // If a raw file needs to be validated, we do this by checking the file exists on disk still, + // and it's stored ETag matches against the remote file using a head request + + var localPath = Path.Combine(BaseFileDirectory, localRawFile); + + var destinationRawFile = configuration.RawFiles.FirstOrDefault(r => r.Key == localRawFile); + var remoteFileUri = new Uri(_getUrl(destinationRawFile.Value)); + var localFilePath = Path.Combine(BaseFileDirectory, destinationRawFile.Key); + + var downloaded = await DownloadFileIfInvalidAsync(remoteFileUri, localFilePath); + + if (!downloaded) + { + if(!File.Exists(localPath)) + throw new FileNotFoundException($"Failed to obtain raw file '{destinationRawFile.Key}'", localPath); + + logger?.LogError("Unable to download [{destinationRawFile}] to [{localPath}]", destinationRawFile, localPath); + if (Directory.Exists(BaseFileDirectory)) + { + logger?.LogTrace("Listing directory contents for [{BaseFileDirectory}]", BaseFileDirectory); + foreach (var file in Directory.GetFiles(BaseFileDirectory)) + { + logger?.LogTrace("File: {file}", file); + } + } + else + { + logger?.LogError("Directory does not exist: [{BaseFileDirectory}]", BaseFileDirectory); + } + } + + var data = await File.ReadAllTextAsync(localPath); + + return data; + } + + public async Task IsFileValidAsync(string remoteFile, string localFile) + { + var remoteUri = new Uri(_getUrl(remoteFile)); + var localUri = Path.Combine(BaseFileDirectory, localFile); + + return await IsFileValidAsync(remoteUri, localUri); + } + + /// + /// A file is valid if it exists, and its etag has been checked within + /// the last hour and matches the remote etag. + /// + /// + public async Task IsFileValidAsync(Uri remoteUri, string localUri) + { + // First, check if the file exists. + if (!File.Exists(localUri)) + return false; + + // Get the etag from the cache + var cachedEtag = _rawFileEtagCacheData.FirstOrDefault(e => e.Filename == localUri); + + if (cachedEtag == null) + { + logger?.LogTrace("No etag found for {destinationUri.LocalPath}", localUri); + return false; + } + logger?.LogTrace("etag for this file is {eTag}", cachedEtag); + + // Now, if we haven't checked in the last hour, check if the etag is valid. + if (cachedEtag.LastChecked <= DateTime.UtcNow.AddHours(-1)) + { + var remoteEtag = await GetRemoteEtagAsync(remoteUri); + + if (string.IsNullOrWhiteSpace(remoteEtag)) + { + logger?.LogTrace("No remote etag found for {sourceUri}", remoteUri); + return false; + } + + var valid = string.Equals(cachedEtag.ETag, remoteEtag, StringComparison.Ordinal); + logger?.LogTrace("File {destinationUri.LocalPath} etag valid: {valid}", localUri, valid); + + if (valid) + { + cachedEtag.LastChecked = DateTime.UtcNow; + await SaveEtagDetailsToDiskAsync(); + } + + return valid; + } + + logger?.LogTrace("File {destinationUri.LocalPath} exists and etag was checked recently.", localUri); + return true; + } + + /// + /// Check if the local file exists and the cache matches + /// + internal async Task DownloadFileIfInvalidAsync(Uri remoteUri, string localUri) + { + // A file is invalid if: + // - It does not exist + // - The etag does not match + // The etag should be checked every LastChecked + 1 hour. + if(await IsFileValidAsync(remoteUri, localUri)) + { + logger?.LogTrace("File {destinationUri.LocalPath} is valid, no download needed.", localUri); + return true; + } + + // If we've reached this point, the local file is either missing or invalid. + var downloadResponse = await DownloadFileAsync(remoteUri, localUri); + + // If the download was successful, save the etag. + if (downloadResponse) + { + logger?.LogTrace("Successfully downloaded {sourceUri} to {destinationUri.LocalPath}", remoteUri, localUri); + + var remoteEtag = await GetRemoteEtagAsync(remoteUri); + + await UpdateRawFileDetailsAsync(localUri, + remoteEtag, + File.GetLastWriteTimeUtc(localUri)); + } + else + { + logger?.LogError("Failure downloading file."); + } + + return downloadResponse; + } + + private async Task GetRemoteEtagAsync(Uri sourceUri) + { + HttpRequestMessage request = + new(HttpMethod.Head, + sourceUri); + + HttpResponseMessage response; + + try + { + response = await downloadClient.SendAsync(request); + } + catch (Exception ex) + { + logger?.LogError(ex, "Error checking etag for {sourceUri}", sourceUri); + return string.Empty; + } + + if (!response.IsSuccessStatusCode) + { + logger?.LogError("Error checking etag for {sourceUri}. Status code: {StatusCode}", sourceUri, response.StatusCode); + return string.Empty; + } + + var tag = response.Headers.ETag?.Tag; + if (string.IsNullOrWhiteSpace(tag)) + return string.Empty; + + logger?.LogTrace("ETag for {sourceUri} is {ETag}", sourceUri, tag); + + return tag; + } + + /// + /// Update the cache with an entry + /// + /// + /// + internal async Task UpdateRawFileDetailsAsync(string filename, string eTag, DateTime lastModified) + { + var existing = _rawFileEtagCacheData.FirstOrDefault(e => e.Filename == filename); + + if (existing != null) + { + existing.ETag = eTag; + existing.LastModified = lastModified; + existing.LastChecked = DateTime.UtcNow; + } + else + { + _rawFileEtagCacheData.Add(new FileETag() + { + Filename = filename, + ETag = eTag, + LastModified = lastModified, + LastChecked = DateTime.UtcNow + }); + } + + await SaveEtagDetailsToDiskAsync(); + } + + /// + /// Load the cached etag data from file + /// + internal async Task> GetCacheDataAsync() + { + var results = new List(); + var cacheDataFile = Path.Combine(BaseFileDirectory, _rawFileCacheDataFile); + + if (File.Exists(cacheDataFile)) + { + var data = await File.ReadAllTextAsync(cacheDataFile); + + var deserialised = JsonSerializer.Deserialize>(data); + + if (deserialised != null) + results = deserialised; + } + + return results; + } + + /// + /// Save the cached etag data to file. Do this each time an update is made. + /// + internal async Task SaveEtagDetailsToDiskAsync() + { + var baseDirectory = new Uri(BaseFileDirectory); + if (!Directory.Exists(baseDirectory.LocalPath)) + Directory.CreateDirectory(baseDirectory.LocalPath); + + var cacheDataFile = Path.Combine(BaseFileDirectory, _rawFileCacheDataFile); + var dataString = JsonSerializer.Serialize(_rawFileEtagCacheData, new JsonSerializerOptions { WriteIndented = true }); + + await File.WriteAllTextAsync(cacheDataFile, dataString); + } + + /// + /// Configures the base directory used for file operations. + /// + /// The path to the directory that will be set as the base for file operations. Cannot be null. + public async Task ConfigureAsync(string baseFileDirectory) + { + BaseFileDirectory = baseFileDirectory; + _rawFileEtagCacheData = await GetCacheDataAsync(); + } + + public async Task DeleteAllFiles() + { + _rawFileEtagCacheData.Clear(); + await SaveEtagDetailsToDiskAsync(); + try + { + if (Directory.Exists(BaseFileDirectory)) + { + foreach (var file in Directory.GetFiles(BaseFileDirectory, $"*.{rawDataFileExtension}")) + { + try + { + File.Delete(file); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to delete cache file {file}", file); + } + } + } + } + catch (Exception ex) + { + logger?.LogError(ex, "Error clearing cache directory {BaseFileDirectory}", BaseFileDirectory); + } + } + + /// + /// Download a file from sourceUri to destinationUri, overwriting if it exists. + /// + /// + internal async Task DownloadFileAsync(Uri sourceUri, string destinationFile) + { + try + { + if (!Directory.Exists(BaseFileDirectory)) + Directory.CreateDirectory(BaseFileDirectory); + + using var s = await downloadClient.GetStreamAsync(sourceUri); + + if (File.Exists(destinationFile)) + File.Delete(destinationFile); + + using var fs = new FileStream(destinationFile, FileMode.CreateNew); + await s.CopyToAsync(fs); + } + catch (Exception ex) + { + logger?.LogError(ex, "Unable to DownloadFileAsync [{sourceUri}] to [{destinationUri}]", sourceUri, destinationFile); + return false; + } + return true; + } + + + /// + /// Get the flag used PTR data for data extraction + /// + public bool UsePtrData { get => usePtrData; } + public void SetUsePtrData(bool usePtrData) => this.usePtrData = usePtrData; + + /// + /// Get the github branch name used for data extraction + /// + public string UseBranchName { get => useBranchName; } + public void SetUseBranchName(string branchName) => useBranchName = branchName; + } +} diff --git a/SimcProfileParser/Interfaces/DataSync/ICacheService.cs b/SimcProfileParser/Interfaces/DataSync/ICacheService.cs index 28f55fa..7ca068e 100644 --- a/SimcProfileParser/Interfaces/DataSync/ICacheService.cs +++ b/SimcProfileParser/Interfaces/DataSync/ICacheService.cs @@ -1,5 +1,4 @@ using SimcProfileParser.Model.DataSync; -using System.Collections.Generic; using System.Threading.Tasks; namespace SimcProfileParser.Interfaces.DataSync @@ -13,16 +12,6 @@ internal interface ICacheService /// The base directory the cache service stores its files /// string BaseFileDirectory { get; } - /// - /// Collection of already registered file configurations - /// - IReadOnlyCollection RegisteredFiles { get; } - - /// - /// Register a configuration representing a parsed json file and its raw sources - /// - /// - void RegisterFileConfiguration(CacheFileConfiguration configuration); /// /// Get back the parsed file contents from disk @@ -31,12 +20,6 @@ internal interface ICacheService /// Type of parsed file to return /// Task GetParsedFileContentsAsync(SimcParsedFileType fileType); - /// - /// Generates a version of the raw files that's stored as parsed.json data which can be - /// used by GetParsedFileContents to extract the parsed data as objects - /// - /// The type of file this should generate - Task GenerateParsedFileAsync(SimcParsedFileType fileType); /// /// Set to TRUE to use PTR data for data extraction diff --git a/SimcProfileParser/Interfaces/DataSync/IRawFileService.cs b/SimcProfileParser/Interfaces/DataSync/IRawFileService.cs new file mode 100644 index 0000000..65976ba --- /dev/null +++ b/SimcProfileParser/Interfaces/DataSync/IRawFileService.cs @@ -0,0 +1,19 @@ +using SimcProfileParser.Model.DataSync; +using System; +using System.Threading.Tasks; + +namespace SimcProfileParser.Interfaces.DataSync +{ + internal interface IRawFileService + { + bool UsePtrData { get; } + string UseBranchName { get; } + + Task DeleteAllFiles(); + Task ConfigureAsync(string baseFileDirectory); + Task GetFileContentsAsync(CacheFileConfiguration configuration, string localRawFile); + void SetUsePtrData(bool usePtrData); + void SetUseBranchName(string branchName); + Task IsFileValidAsync(string remoteFile, string destinationFile); + } +} \ No newline at end of file diff --git a/SimcProfileParser/Model/DataSync/FileETag.cs b/SimcProfileParser/Model/DataSync/FileETag.cs index 3e7bd68..8f7b0d4 100644 --- a/SimcProfileParser/Model/DataSync/FileETag.cs +++ b/SimcProfileParser/Model/DataSync/FileETag.cs @@ -7,5 +7,6 @@ public class FileETag public string Filename { get; set; } public string ETag { get; set; } public DateTime LastModified { get; set; } + public DateTime LastChecked { get; set; } } } diff --git a/SimcProfileParser/SimcGenerationService.cs b/SimcProfileParser/SimcGenerationService.cs index a1085da..bf209fc 100644 --- a/SimcProfileParser/SimcGenerationService.cs +++ b/SimcProfileParser/SimcGenerationService.cs @@ -82,7 +82,8 @@ public SimcGenerationService(ILoggerFactory loggerFactory) loggerFactory.CreateLogger()); _cacheService = new CacheService(dataExtractionService, - loggerFactory.CreateLogger()); + loggerFactory.CreateLogger(), + new RawFileService(loggerFactory.CreateLogger())); var utilityService = new SimcUtilityService( _cacheService, diff --git a/SimcProfileParser/SimcProfileParser.csproj b/SimcProfileParser/SimcProfileParser.csproj index aa43dd2..eda9379 100644 --- a/SimcProfileParser/SimcProfileParser.csproj +++ b/SimcProfileParser/SimcProfileParser.csproj @@ -3,9 +3,9 @@ net9.0 true - 3.1.2 - 3.1.2 - 3.1.2 + 3.2.0 + 3.2.0 + 3.2.0 Mechanical Priest Mechanical Priest GPL-3.0-only