From fd2dbfefc64e06640338c4e70a3088329d340994 Mon Sep 17 00:00:00 2001 From: Niphyr Date: Sun, 26 Oct 2025 02:52:42 +1100 Subject: [PATCH 1/2] Refactor cache to use RawFileService for raw file management Introduces RawFileService to handle raw file downloads, ETag validation, and cache management. CacheService now delegates raw file operations to RawFileService. Updates related tests and interfaces to support the new service. FileETag model extended with LastChecked property for ETag revalidation logic. --- .../DataSync/CacheServiceTests.cs | 26 +- .../SimcItemCreationServiceTests.cs | 4 +- .../SimcSpellCreationServiceTests.cs | 4 +- SimcProfileParser/DataSync/CacheService.cs | 372 +++++------------- SimcProfileParser/DataSync/RawFileService.cs | 343 ++++++++++++++++ .../Interfaces/DataSync/ICacheService.cs | 17 - .../Interfaces/DataSync/IRawFileService.cs | 19 + SimcProfileParser/Model/DataSync/FileETag.cs | 1 + SimcProfileParser/SimcGenerationService.cs | 3 +- 9 files changed, 494 insertions(+), 295 deletions(-) create mode 100644 SimcProfileParser/DataSync/RawFileService.cs create mode 100644 SimcProfileParser/Interfaces/DataSync/IRawFileService.cs diff --git a/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs b/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs index b06c0ea..b1ab6f3 100644 --- a/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs +++ b/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs @@ -37,7 +37,7 @@ public void Init() .AddSerilog() .AddFilter(level => level >= LogLevel.Trace)); - ICacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + ICacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); // Wipe out the directory before testing as a workaround for file access not being abstracted if (Directory.Exists(cache.BaseFileDirectory)) @@ -54,7 +54,7 @@ public void Init() public async Task CS_Downloads_File() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); var configuration = new CacheFileConfiguration() { @@ -81,7 +81,7 @@ public async Task CS_Downloads_File() public async Task CS_Downloads_Ptr_File() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); cache.SetUsePtrData(true); var configuration = new CacheFileConfiguration() @@ -109,7 +109,7 @@ public async Task CS_Downloads_Ptr_File() public void CS_Download_Fails_Bad_Filename() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); cache.SetUsePtrData(true); var configuration = new CacheFileConfiguration() @@ -137,14 +137,15 @@ async Task testDelegate() public async Task CS_Cache_Updates() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); var filename = "test.txt"; var eTag = "12345"; var fileContents = @"[" + Environment.NewLine + @" {" + Environment.NewLine + @" ""Filename"": ""test.txt""," + Environment.NewLine + @" ""ETag"": ""12345""," + Environment.NewLine + - @" ""LastModified"": ""0001-01-01T00:00:00.0000001""" + Environment.NewLine + + @" ""LastModified"": ""0001-01-01T00:00:00.0000001""," + Environment.NewLine + + @" ""LastChecked"": ""0001-01-01T00:00:00""" + Environment.NewLine + @" }" + Environment.NewLine + @"]"; var lastModified = new DateTime(1); @@ -162,7 +163,7 @@ public async Task CS_Cache_Updates() public async Task CS_Cache_Reads() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); var filename = "test.txt"; var eTag = "12345"; var lastModified = new DateTime(1); @@ -183,7 +184,7 @@ public async Task CS_Cache_Reads() public async Task CS_Cache_Saves() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); // Act var filename = "test.txt"; @@ -202,7 +203,8 @@ public async Task CS_Cache_Saves() @" {" + Environment.NewLine + @" ""Filename"": ""test.txt""," + Environment.NewLine + @" ""ETag"": ""12345""," + Environment.NewLine + - @" ""LastModified"": ""0001-01-01T00:00:00.0000001""" + Environment.NewLine + + @" ""LastModified"": ""0001-01-01T00:00:00.0000001""," + Environment.NewLine + + @" ""LastChecked"": ""0001-01-01T00:00:00""" + Environment.NewLine + @" }" + Environment.NewLine + @"]"; @@ -220,7 +222,7 @@ public async Task CS_Cache_Saves() public void CS_Respects_Ptr_Flag() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); cache.SetUsePtrData(true); cache.SetUseBranchName("test_branch"); @@ -235,7 +237,7 @@ public void CS_Respects_Ptr_Flag() public void CS_Ptr_Defaults_Off() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); cache.SetUseBranchName("test_branch"); // Act @@ -249,7 +251,7 @@ public void CS_Ptr_Defaults_Off() public async Task CS_ClearCache_Deletes_Files_And_Recovers() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger()); + CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); var configuration = new CacheFileConfiguration() { diff --git a/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs b/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs index 469f195..83a2391 100644 --- a/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs +++ b/SimcProfileParser.Tests/SimcItemCreationServiceTests.cs @@ -41,7 +41,9 @@ public void InitOnce() IRawDataExtractionService rawDataExtractionService = new RawDataExtractionService(_loggerFactory.CreateLogger()); - ICacheService cacheService = new CacheService(rawDataExtractionService, _loggerFactory.CreateLogger()); + ICacheService cacheService = new CacheService(rawDataExtractionService, + _loggerFactory.CreateLogger(), + new RawFileService(_loggerFactory.CreateLogger())); var utilityService = new SimcUtilityService( cacheService, _loggerFactory.CreateLogger()); diff --git a/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs b/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs index 5348361..f90f544 100644 --- a/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs +++ b/SimcProfileParser.Tests/SimcSpellCreationServiceTests.cs @@ -34,7 +34,9 @@ public void InitOnce() IRawDataExtractionService rawDataExtractionService = new RawDataExtractionService(_loggerFactory.CreateLogger()); - ICacheService cacheService = new CacheService(rawDataExtractionService, _loggerFactory.CreateLogger()); + ICacheService cacheService = new CacheService(rawDataExtractionService, + _loggerFactory.CreateLogger(), + new RawFileService(_loggerFactory.CreateLogger())); var utilityService = new SimcUtilityService( cacheService, _loggerFactory.CreateLogger()); diff --git a/SimcProfileParser/DataSync/CacheService.cs b/SimcProfileParser/DataSync/CacheService.cs index fad6587..4bd1693 100644 --- a/SimcProfileParser/DataSync/CacheService.cs +++ b/SimcProfileParser/DataSync/CacheService.cs @@ -6,8 +6,6 @@ using System.Collections.ObjectModel; using System.IO; using System.Linq; -using System.Net; -using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; @@ -18,34 +16,28 @@ internal class CacheService : ICacheService { public string BaseFileDirectory { get; } - protected IList _registeredFiles; - protected IDictionary _cachedFileData; + protected List _registeredFiles = []; + protected Dictionary _cachedFileData = []; protected readonly IRawDataExtractionService _rawDataExtractionService; protected readonly ILogger _logger; + private readonly IRawFileService rawFileService; - private bool _usePtrData = false; - private string _useBranchName = "midnight"; - internal string _getUrl(string fileName) => "https://raw.githubusercontent.com/simulationcraft/simc/" - + _useBranchName + "/engine/dbc/generated/" - + fileName - + (_usePtrData ? "_ptr" : "") - + ".inc"; - + private readonly string parsedDataFileExtension = "json"; public IReadOnlyCollection RegisteredFiles => new ReadOnlyCollection(_registeredFiles); public CacheService(IRawDataExtractionService rawDataExtractionService, - ILogger logger) + ILogger logger, + IRawFileService rawFileService) { - _registeredFiles = new List(); - _cachedFileData = new Dictionary(); - BaseFileDirectory = Path.Combine( Path.GetTempPath(), "SimcProfileParserData"); _rawDataExtractionService = rawDataExtractionService; _logger = logger; + this.rawFileService = rawFileService; + this.rawFileService.ConfigureAsync(BaseFileDirectory).GetAwaiter().GetResult(); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemDataNew.json", ParsedFileType = SimcParsedFileType.ItemDataNew, @@ -56,7 +48,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemDataOld.json", ParsedFileType = SimcParsedFileType.ItemDataOld, @@ -67,7 +59,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "CombatRatingMultipliers.json", ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, @@ -77,7 +69,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "StaminaMultipliers.json", ParsedFileType = SimcParsedFileType.StaminaMultipliers, @@ -87,7 +79,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "RandomPropPoints.json", ParsedFileType = SimcParsedFileType.RandomPropPoints, @@ -97,7 +89,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "SpellData.json", ParsedFileType = SimcParsedFileType.SpellData, @@ -107,7 +99,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemBonusData.json", ParsedFileType = SimcParsedFileType.ItemBonusData, @@ -117,7 +109,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "GemData.json", ParsedFileType = SimcParsedFileType.GemData, @@ -127,7 +119,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemEnchantData.json", ParsedFileType = SimcParsedFileType.ItemEnchantData, @@ -137,7 +129,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "SpellScalingMultipliers.json", ParsedFileType = SimcParsedFileType.SpellScaleMultipliers, @@ -147,7 +139,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "CurveData.json", ParsedFileType = SimcParsedFileType.CurvePoints, @@ -157,7 +149,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "RppmData.json", ParsedFileType = SimcParsedFileType.RppmData, @@ -167,7 +159,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "ItemEffectData.json", ParsedFileType = SimcParsedFileType.ItemEffectData, @@ -177,7 +169,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "GameDataVersion.json", ParsedFileType = SimcParsedFileType.GameDataVersion, @@ -187,7 +179,7 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, } }); - ((ICacheService)this).RegisterFileConfiguration(new CacheFileConfiguration() + RegisterFileConfiguration(new CacheFileConfiguration() { LocalParsedFile = "TraitData.json", ParsedFileType = SimcParsedFileType.TraitData, @@ -205,10 +197,21 @@ public CacheService(IRawDataExtractionService rawDataExtractionService, /// Type of file to return data from async Task ICacheService.GetParsedFileContentsAsync(SimcParsedFileType fileType) { + var configuration = _registeredFiles.FirstOrDefault(f => f.ParsedFileType == fileType) + ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); + // First check if we already have the data loaded: - if (_cachedFileData.ContainsKey(fileType)) + if (_cachedFileData.TryGetValue(fileType, out object cachedData)) { - var cachedData = _cachedFileData[fileType]; + + if (!await IsDiskCacheValidForConfiguration(configuration)) + { + // Anytime a raw file is invalidated, the raw file and parsed json file should be removed, + // and the file re - obtained + await DeleteDiskCacheForConfiguration(configuration); + + return await GetParsedFileContents(configuration); + } if (cachedData is T t) { @@ -225,310 +228,153 @@ async Task ICacheService.GetParsedFileContentsAsync(SimcParsedFileType fil } } - var configuration = _registeredFiles.Where(f => f.ParsedFileType == fileType).FirstOrDefault() - ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); + return await GetParsedFileContents(configuration); + } - var localPath = new Uri(Path.Combine(BaseFileDirectory, configuration.LocalParsedFile)).LocalPath; + /// + /// For a given configuration, parse and load the .json file contents into T + /// + /// + async Task GetParsedFileContents(CacheFileConfiguration configuration) + { + var localPath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); // If the file doesn't exist, generate it. if (!File.Exists(localPath)) { _logger?.LogTrace("File [{localPath}] does not exist, generating it...", localPath); - await ((ICacheService)this).GenerateParsedFileAsync(fileType); + await GenerateParsedFileAsync(configuration.ParsedFileType); } var fileText = await File.ReadAllTextAsync(localPath); - var deserialisedData = JsonSerializer.Deserialize(fileText); + var deserialisedData = JsonSerializer.Deserialize(fileText) + ?? throw new InvalidDataException($"Failed to deserialize {configuration.LocalParsedFile} to {typeof(T).Name}."); - _cachedFileData.Add(fileType, deserialisedData); + _cachedFileData[configuration.ParsedFileType] = deserialisedData; return deserialisedData; } - /// - /// Generates a parsed .json file for the specified configuration by calling the RawDataExtractionService - /// - /// Type of file to generate data for - async Task ICacheService.GenerateParsedFileAsync(SimcParsedFileType fileType) + async Task IsDiskCacheValidForConfiguration(CacheFileConfiguration configuration) { - var configuration = _registeredFiles.Where(f => f.ParsedFileType == fileType).FirstOrDefault() - ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); - - // Gather together all the raw data the extraction service needs to run its process - var rawData = new Dictionary(); + // First, check that all the raw files exist foreach (var rawFile in configuration.RawFiles) { - var data = await GetRawFileContentsAsync(configuration, rawFile.Key); - rawData.Add(rawFile.Key, data); - } - - // Generate the parsed .json file - var parsedData = _rawDataExtractionService.GenerateData(configuration.ParsedFileType, rawData); - var localPath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); - - _logger?.LogTrace("Saving parsed json data for [{configuration.ParsedFileType}] to [{localPath}]", configuration.ParsedFileType, localPath); - await File.WriteAllTextAsync(localPath, JsonSerializer.Serialize(parsedData)); - } - - void ICacheService.RegisterFileConfiguration(CacheFileConfiguration configuration) - { - var exists = _registeredFiles - .Where(f => f.ParsedFileType == configuration.ParsedFileType) - .FirstOrDefault(); - - if (exists != null) - { - _registeredFiles.Remove(exists); - } - - _registeredFiles.Add(configuration); - } - - internal async Task GetRawFileContentsAsync(CacheFileConfiguration configuration, string localRawFile) - { - var localPath = new Uri(Path.Combine(BaseFileDirectory, localRawFile)).LocalPath; - if (!File.Exists(localPath)) - { - var destinationRawFile = configuration.RawFiles.Where(r => r.Key == localRawFile).FirstOrDefault(); - - _logger?.LogTrace("Path does not exist: [{localPath}] - attempting to download file from [{destinationRawFile}].", localPath, destinationRawFile); + var remoteFileName = rawFile.Value; + var localFileName = rawFile.Key; - var downloaded = await DownloadFileIfChangedAsync(new Uri(_getUrl(destinationRawFile.Value)), - new Uri(Path.Combine(BaseFileDirectory, destinationRawFile.Key))); - - if (!downloaded) + var isLocalFileValid = await rawFileService.IsFileValidAsync(remoteFileName, localFileName); + if(!isLocalFileValid) { - _logger?.LogError("Unable to download [{destinationRawFile}] to [{localPath}]", destinationRawFile, localPath); - if(Directory.Exists(BaseFileDirectory)) - { - _logger?.LogTrace("Listing directory contents for [{BaseFileDirectory}]", BaseFileDirectory); - foreach(var file in Directory.GetFiles(BaseFileDirectory)) - { - _logger?.LogTrace("File: {file}", file); - } - } - else - { - _logger?.LogError("Directory does not exist: [{BaseFileDirectory}]", BaseFileDirectory); - } + return false; } } - var data = await File.ReadAllTextAsync(localPath); - - return data; - } - - /// - /// Check if the local file exists and the cache matches - /// - /// - /// - /// - internal async Task DownloadFileIfChangedAsync(Uri sourceUri, Uri destinationUri) - { - using HttpClient httpClient = new(); - - HttpRequestMessage request = - new(HttpMethod.Head, - sourceUri); - - HttpResponseMessage response; - - try + // Next, check that the parsed .json file exists + var localParsedFilePath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); + if (!File.Exists(localParsedFilePath)) { - response = await httpClient.SendAsync(request); - } - catch (Exception ex) - { - _logger?.LogError(ex, "Error downloading {sourceUri} to {destinationUri}", sourceUri, destinationUri); return false; } - // Grab the cache info and the files last modified date. - var eTagCacheData = await GetCacheDataAsync(); - var eTag = eTagCacheData - .Where(e => e.Filename == destinationUri.LocalPath) - .FirstOrDefault(); - - if (eTag != null) - _logger?.LogTrace("etag for this file is {eTag}", eTag); - else - _logger?.LogTrace("No etag found for {destinationUri.LocalPath}", destinationUri.LocalPath); - - DateTime lastModified = DateTime.UtcNow; - - if (File.Exists(destinationUri.LocalPath)) - lastModified = File.GetLastWriteTimeUtc(destinationUri.LocalPath); - - // Check if we need to download it or not. - if (eTag != null && // If there is an etag - response.Headers.ETag.Tag == eTag.ETag && // and they match - eTag.LastModified == lastModified) // and the last modified match - return true; // Then we don't need to download it. - - var downloadResponse = await DownloadFileAsync(sourceUri, destinationUri); - - // If the download was successful, save the etag. - if (downloadResponse) - { - _logger?.LogTrace("Successfully downloaded {sourceUri} to {destinationUri.LocalPath}", sourceUri, destinationUri.LocalPath); - lastModified = File.GetLastWriteTimeUtc(destinationUri.LocalPath); - await UpdateCacheDataAsync(destinationUri.LocalPath, response.Headers.ETag.Tag, lastModified); - } - else - { - _logger?.LogError("Failure downloading file."); - } - - return downloadResponse; + return true; } - internal async Task DownloadFileAsync(Uri sourceUri, Uri destinationUri) + async Task DeleteDiskCacheForConfiguration(CacheFileConfiguration configuration) { - using HttpClient client = new(); + // Clear the cached data for this file type + _cachedFileData.Remove(configuration.ParsedFileType); - try + // Now delete both the parsed .json file and the raw files + var localParsedFilePath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); + if (File.Exists(localParsedFilePath)) { - var baseDirectory = new Uri(destinationUri, "."); - if (!Directory.Exists(baseDirectory.OriginalString)) - Directory.CreateDirectory(baseDirectory.OriginalString); - - using var s = await client.GetStreamAsync(sourceUri); - - if (File.Exists(destinationUri.LocalPath)) - File.Delete(destinationUri.LocalPath); - - using var fs = new FileStream(destinationUri.LocalPath, FileMode.CreateNew); - await s.CopyToAsync(fs); + File.Delete(localParsedFilePath); } - catch (Exception ex) + foreach (var rawFile in configuration.RawFiles) { - _logger?.LogError(ex, "Unable to DownloadFileAsync [{sourceUri}] to [{destinationUri}]", sourceUri, destinationUri); - return false; + var localRawFilePath = Path.Combine(BaseFileDirectory, rawFile.Key); + if (File.Exists(localRawFilePath)) + { + File.Delete(localRawFilePath); + } } - return true; } - #region eTag Cache - - private readonly string _etagCacheDataFile = "FileDownloadCache.json"; - protected List _eTagCacheData = new(); - /// - /// Update the cache with an entry + /// Generates a parsed .json file for the specified configuration by calling the RawDataExtractionService /// - /// - /// - internal async Task UpdateCacheDataAsync(string filename, string eTag, DateTime lastModified) + /// Type of file to generate data for + async Task GenerateParsedFileAsync(SimcParsedFileType fileType) { - var eTagCacheData = await GetCacheDataAsync(); - - var existing = eTagCacheData.Where(e => e.Filename == filename).FirstOrDefault(); + var configuration = _registeredFiles.Where(f => f.ParsedFileType == fileType).FirstOrDefault() + ?? throw new ArgumentOutOfRangeException(nameof(fileType), "Supplied fileType has not been registered."); - if (existing != null) - existing.ETag = eTag; - else + // Gather together all the raw data the extraction service needs to run its process + var rawData = new Dictionary(); + foreach (var rawFile in configuration.RawFiles) { - eTagCacheData.Add(new FileETag() - { - Filename = filename, - ETag = eTag, - LastModified = lastModified - }); + var data = await rawFileService.GetFileContentsAsync(configuration, rawFile.Key); + rawData.Add(rawFile.Key, data); } - await SaveCacheDataAsync(eTagCacheData); + // Generate the parsed .json file + var parsedData = _rawDataExtractionService.GenerateData(configuration.ParsedFileType, rawData); + var localPath = Path.Combine(BaseFileDirectory, configuration.LocalParsedFile); + + _logger?.LogTrace("Saving parsed json data for [{configuration.ParsedFileType}] to [{localPath}]", configuration.ParsedFileType, localPath); + await File.WriteAllTextAsync(localPath, JsonSerializer.Serialize(parsedData)); } - /// - /// Load the cached etag data from file - /// - internal async Task> GetCacheDataAsync(bool force = false) + void RegisterFileConfiguration(CacheFileConfiguration configuration) { - if (!force && _eTagCacheData.Count > 0) - return _eTagCacheData; - - var results = new List(); - var cacheDataFile = Path.Combine(BaseFileDirectory, _etagCacheDataFile); + var exists = _registeredFiles + .FirstOrDefault(f => f.ParsedFileType == configuration.ParsedFileType); - if (File.Exists(cacheDataFile)) + if (exists != null) { - var data = await File.ReadAllTextAsync(cacheDataFile); - - var deserialised = JsonSerializer.Deserialize>(data); - - if (deserialised != null) - results = deserialised; + _registeredFiles.Remove(exists); } - return results; - } - - /// - /// Save the cached etag data to file - /// - internal async Task SaveCacheDataAsync(List data) - { - var baseDirectory = new Uri(BaseFileDirectory); - if (!Directory.Exists(baseDirectory.OriginalString)) - Directory.CreateDirectory(baseDirectory.OriginalString); - - var cacheDataFile = Path.Combine(BaseFileDirectory, _etagCacheDataFile); - var dataString = JsonSerializer.Serialize(data, new JsonSerializerOptions { WriteIndented = true }); - - await File.WriteAllTextAsync(cacheDataFile, dataString); + _registeredFiles.Add(configuration); } - #endregion - /// /// Get the flag used PTR data for data extraction /// - public bool UsePtrData { get => _usePtrData; } - - public void SetUsePtrData(bool usePtrData) - { - _usePtrData = usePtrData; - } + public bool UsePtrData { get => rawFileService.UsePtrData; } + public void SetUsePtrData(bool usePtrData) => rawFileService.SetUsePtrData(usePtrData); /// /// Get the github branch name used for data extraction /// - public string UseBranchName { get => _useBranchName; } - public void SetUseBranchName(string branchName) - { - _useBranchName = branchName; - } + public string UseBranchName { get => rawFileService.UseBranchName; } + public void SetUseBranchName(string branchName) => rawFileService.SetUseBranchName(branchName); public async Task ClearCacheAsync() { // Clear in-memory caches _cachedFileData.Clear(); - _eTagCacheData.Clear(); // Clear on-disk cache - try + if (Directory.Exists(BaseFileDirectory)) { - if (Directory.Exists(BaseFileDirectory)) + foreach (var file in Directory.GetFiles(BaseFileDirectory, $"*.{parsedDataFileExtension}")) { - foreach (var file in Directory.GetFiles(BaseFileDirectory)) + try { - try { File.Delete(file); } - catch (Exception ex) - { - _logger?.LogWarning(ex, "Failed to delete cache file {file}", file); - } + File.Delete(file); + } + catch (Exception ex) + { + _logger?.LogWarning(ex, "Failed to delete cache file {file}", file); } } } - catch (Exception ex) - { - _logger?.LogError(ex, "Error clearing cache directory {BaseFileDirectory}", BaseFileDirectory); - } - await Task.CompletedTask; + // Clear raw file cache + await rawFileService.DeleteAllFiles(); } } } diff --git a/SimcProfileParser/DataSync/RawFileService.cs b/SimcProfileParser/DataSync/RawFileService.cs new file mode 100644 index 0000000..589a296 --- /dev/null +++ b/SimcProfileParser/DataSync/RawFileService.cs @@ -0,0 +1,343 @@ +using Microsoft.Extensions.Logging; +using SimcProfileParser.Interfaces.DataSync; +using SimcProfileParser.Model.DataSync; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace SimcProfileParser.DataSync +{ + /// + /// The intent of this class is to manage access to the raw data files. + /// + internal class RawFileService(ILogger logger) : IRawFileService + { + private string BaseFileDirectory = Path.Combine(AppContext.BaseDirectory, "SimcDataCache"); + private readonly string _rawFileCacheDataFile = "FileDownloadCache.json"; + protected List _rawFileEtagCacheData = new(); + private static readonly HttpClient downloadClient = new(); + + private readonly string rawDataFileExtension = "raw"; + private bool usePtrData = false; + private string useBranchName = "midnight"; + internal string _getUrl(string fileName) => "https://raw.githubusercontent.com/simulationcraft/simc/" + + useBranchName + "/engine/dbc/generated/" + + fileName + + (usePtrData ? "_ptr" : "") + + ".inc"; + + public async Task GetFileContentsAsync(CacheFileConfiguration configuration, string localRawFile) + { + // Always: If we download a raw file or generate a.json file, we must update FileDownloadCache.json + // If it's been more than an hour since we last checked the ETag for a file, + // when accessing in-memory processed data we should revalidate the associated raw file + // If a raw file needs to be validated, we do this by checking the file exists on disk still, + // and it's stored ETag matches against the remote file using a head request + + var localPath = Path.Combine(BaseFileDirectory, localRawFile); + + var destinationRawFile = configuration.RawFiles.FirstOrDefault(r => r.Key == localRawFile); + var remoteFileUri = new Uri(_getUrl(destinationRawFile.Value)); + var localFilePath = Path.Combine(BaseFileDirectory, destinationRawFile.Key); + + var downloaded = await DownloadFileIfInvalidAsync(remoteFileUri, localFilePath); + + if (!downloaded) + { + if(!File.Exists(localPath)) + throw new FileNotFoundException($"Failed to obtain raw file '{destinationRawFile.Key}'", localPath); + + logger?.LogError("Unable to download [{destinationRawFile}] to [{localPath}]", destinationRawFile, localPath); + if (Directory.Exists(BaseFileDirectory)) + { + logger?.LogTrace("Listing directory contents for [{BaseFileDirectory}]", BaseFileDirectory); + foreach (var file in Directory.GetFiles(BaseFileDirectory)) + { + logger?.LogTrace("File: {file}", file); + } + } + else + { + logger?.LogError("Directory does not exist: [{BaseFileDirectory}]", BaseFileDirectory); + } + } + + var data = await File.ReadAllTextAsync(localPath); + + return data; + } + + public async Task IsFileValidAsync(string remoteFile, string localFile) + { + var remoteUri = new Uri(_getUrl(remoteFile)); + var localUri = Path.Combine(BaseFileDirectory, localFile); + + return await IsFileValidAsync(remoteUri, localUri); + } + + /// + /// A file is valid if it exists, and its etag has been checked within + /// the last hour and matches the remote etag. + /// + /// + public async Task IsFileValidAsync(Uri remoteUri, string localUri) + { + // First, check if the file exists. + if (!File.Exists(localUri)) + return false; + + // Get the etag from the cache + var cachedEtag = _rawFileEtagCacheData.FirstOrDefault(e => e.Filename == localUri); + + if (cachedEtag == null) + { + logger?.LogTrace("No etag found for {destinationUri.LocalPath}", localUri); + return false; + } + logger?.LogTrace("etag for this file is {eTag}", cachedEtag); + + // Now, if we haven't checked in the last hour, check if the etag is valid. + if (cachedEtag.LastChecked <= DateTime.UtcNow.AddHours(-1)) + { + var remoteEtag = await GetRemoteEtagAsync(remoteUri); + + if (string.IsNullOrWhiteSpace(remoteEtag)) + { + logger?.LogTrace("No remote etag found for {sourceUri}", remoteUri); + return false; + } + + var valid = string.Equals(cachedEtag.ETag, remoteEtag, StringComparison.Ordinal); + logger?.LogTrace("File {destinationUri.LocalPath} etag valid: {valid}", localUri, valid); + + if (valid) + { + cachedEtag.LastChecked = DateTime.UtcNow; + await SaveEtagDetailsToDiskAsync(); + } + + return valid; + } + + logger?.LogTrace("File {destinationUri.LocalPath} exists and etag was checked recently.", localUri); + return true; + } + + /// + /// Check if the local file exists and the cache matches + /// + internal async Task DownloadFileIfInvalidAsync(Uri remoteUri, string localUri) + { + // A file is invalid if: + // - It does not exist + // - The etag does not match + // The etag should be checked every LastChecked + 1 hour. + if(await IsFileValidAsync(remoteUri, localUri)) + { + logger?.LogTrace("File {destinationUri.LocalPath} is valid, no download needed.", localUri); + return true; + } + + // If we've reached this point, the local file is either missing or invalid. + var downloadResponse = await DownloadFileAsync(remoteUri, localUri); + + // If the download was successful, save the etag. + if (downloadResponse) + { + logger?.LogTrace("Successfully downloaded {sourceUri} to {destinationUri.LocalPath}", remoteUri, localUri); + + var remoteEtag = await GetRemoteEtagAsync(remoteUri); + + await UpdateRawFileDetailsAsync(localUri, + remoteEtag, + File.GetLastWriteTimeUtc(localUri)); + } + else + { + logger?.LogError("Failure downloading file."); + } + + return downloadResponse; + } + + private async Task GetRemoteEtagAsync(Uri sourceUri) + { + HttpRequestMessage request = + new(HttpMethod.Head, + sourceUri); + + HttpResponseMessage response; + + try + { + response = await downloadClient.SendAsync(request); + } + catch (Exception ex) + { + logger?.LogError(ex, "Error checking etag for {sourceUri}", sourceUri); + return string.Empty; + } + + if (!response.IsSuccessStatusCode) + { + logger?.LogError("Error checking etag for {sourceUri}. Status code: {StatusCode}", sourceUri, response.StatusCode); + return string.Empty; + } + + var tag = response.Headers.ETag?.Tag; + if (string.IsNullOrWhiteSpace(tag)) + return string.Empty; + + logger?.LogTrace("ETag for {sourceUri} is {ETag}", sourceUri, tag); + + return tag; + } + + /// + /// Update the cache with an entry + /// + /// + /// + internal async Task UpdateRawFileDetailsAsync(string filename, string eTag, DateTime lastModified) + { + var existing = _rawFileEtagCacheData.FirstOrDefault(e => e.Filename == filename); + + if (existing != null) + { + existing.ETag = eTag; + existing.LastModified = lastModified; + existing.LastChecked = DateTime.UtcNow; + } + else + { + _rawFileEtagCacheData.Add(new FileETag() + { + Filename = filename, + ETag = eTag, + LastModified = lastModified, + LastChecked = DateTime.UtcNow + }); + } + + await SaveEtagDetailsToDiskAsync(); + } + + /// + /// Load the cached etag data from file + /// + internal async Task> GetCacheDataAsync() + { + var results = new List(); + var cacheDataFile = Path.Combine(BaseFileDirectory, _rawFileCacheDataFile); + + if (File.Exists(cacheDataFile)) + { + var data = await File.ReadAllTextAsync(cacheDataFile); + + var deserialised = JsonSerializer.Deserialize>(data); + + if (deserialised != null) + results = deserialised; + } + + return results; + } + + /// + /// Save the cached etag data to file. Do this each time an update is made. + /// + internal async Task SaveEtagDetailsToDiskAsync() + { + var baseDirectory = new Uri(BaseFileDirectory); + if (!Directory.Exists(baseDirectory.LocalPath)) + Directory.CreateDirectory(baseDirectory.LocalPath); + + var cacheDataFile = Path.Combine(BaseFileDirectory, _rawFileCacheDataFile); + var dataString = JsonSerializer.Serialize(_rawFileEtagCacheData, new JsonSerializerOptions { WriteIndented = true }); + + await File.WriteAllTextAsync(cacheDataFile, dataString); + } + + /// + /// Configures the base directory used for file operations. + /// + /// The path to the directory that will be set as the base for file operations. Cannot be null. + public async Task ConfigureAsync(string baseFileDirectory) + { + BaseFileDirectory = baseFileDirectory; + _rawFileEtagCacheData = await GetCacheDataAsync(); + } + + public async Task DeleteAllFiles() + { + _rawFileEtagCacheData.Clear(); + await SaveEtagDetailsToDiskAsync(); + try + { + if (Directory.Exists(BaseFileDirectory)) + { + foreach (var file in Directory.GetFiles(BaseFileDirectory, $"*.{rawDataFileExtension}")) + { + try + { + File.Delete(file); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to delete cache file {file}", file); + } + } + } + } + catch (Exception ex) + { + logger?.LogError(ex, "Error clearing cache directory {BaseFileDirectory}", BaseFileDirectory); + } + } + + /// + /// Download a file from sourceUri to destinationUri, overwriting if it exists. + /// + /// + internal async Task DownloadFileAsync(Uri sourceUri, string destinationFile) + { + try + { + if (!Directory.Exists(BaseFileDirectory)) + Directory.CreateDirectory(BaseFileDirectory); + + using var s = await downloadClient.GetStreamAsync(sourceUri); + + if (File.Exists(destinationFile)) + File.Delete(destinationFile); + + using var fs = new FileStream(destinationFile, FileMode.CreateNew); + await s.CopyToAsync(fs); + } + catch (Exception ex) + { + logger?.LogError(ex, "Unable to DownloadFileAsync [{sourceUri}] to [{destinationUri}]", sourceUri, destinationFile); + return false; + } + return true; + } + + + /// + /// Get the flag used PTR data for data extraction + /// + public bool UsePtrData { get => usePtrData; } + public void SetUsePtrData(bool usePtrData) => this.usePtrData = usePtrData; + + /// + /// Get the github branch name used for data extraction + /// + public string UseBranchName { get => useBranchName; } + public void SetUseBranchName(string branchName) => useBranchName = branchName; + } +} diff --git a/SimcProfileParser/Interfaces/DataSync/ICacheService.cs b/SimcProfileParser/Interfaces/DataSync/ICacheService.cs index 28f55fa..7ca068e 100644 --- a/SimcProfileParser/Interfaces/DataSync/ICacheService.cs +++ b/SimcProfileParser/Interfaces/DataSync/ICacheService.cs @@ -1,5 +1,4 @@ using SimcProfileParser.Model.DataSync; -using System.Collections.Generic; using System.Threading.Tasks; namespace SimcProfileParser.Interfaces.DataSync @@ -13,16 +12,6 @@ internal interface ICacheService /// The base directory the cache service stores its files /// string BaseFileDirectory { get; } - /// - /// Collection of already registered file configurations - /// - IReadOnlyCollection RegisteredFiles { get; } - - /// - /// Register a configuration representing a parsed json file and its raw sources - /// - /// - void RegisterFileConfiguration(CacheFileConfiguration configuration); /// /// Get back the parsed file contents from disk @@ -31,12 +20,6 @@ internal interface ICacheService /// Type of parsed file to return /// Task GetParsedFileContentsAsync(SimcParsedFileType fileType); - /// - /// Generates a version of the raw files that's stored as parsed.json data which can be - /// used by GetParsedFileContents to extract the parsed data as objects - /// - /// The type of file this should generate - Task GenerateParsedFileAsync(SimcParsedFileType fileType); /// /// Set to TRUE to use PTR data for data extraction diff --git a/SimcProfileParser/Interfaces/DataSync/IRawFileService.cs b/SimcProfileParser/Interfaces/DataSync/IRawFileService.cs new file mode 100644 index 0000000..65976ba --- /dev/null +++ b/SimcProfileParser/Interfaces/DataSync/IRawFileService.cs @@ -0,0 +1,19 @@ +using SimcProfileParser.Model.DataSync; +using System; +using System.Threading.Tasks; + +namespace SimcProfileParser.Interfaces.DataSync +{ + internal interface IRawFileService + { + bool UsePtrData { get; } + string UseBranchName { get; } + + Task DeleteAllFiles(); + Task ConfigureAsync(string baseFileDirectory); + Task GetFileContentsAsync(CacheFileConfiguration configuration, string localRawFile); + void SetUsePtrData(bool usePtrData); + void SetUseBranchName(string branchName); + Task IsFileValidAsync(string remoteFile, string destinationFile); + } +} \ No newline at end of file diff --git a/SimcProfileParser/Model/DataSync/FileETag.cs b/SimcProfileParser/Model/DataSync/FileETag.cs index 3e7bd68..8f7b0d4 100644 --- a/SimcProfileParser/Model/DataSync/FileETag.cs +++ b/SimcProfileParser/Model/DataSync/FileETag.cs @@ -7,5 +7,6 @@ public class FileETag public string Filename { get; set; } public string ETag { get; set; } public DateTime LastModified { get; set; } + public DateTime LastChecked { get; set; } } } diff --git a/SimcProfileParser/SimcGenerationService.cs b/SimcProfileParser/SimcGenerationService.cs index a1085da..bf209fc 100644 --- a/SimcProfileParser/SimcGenerationService.cs +++ b/SimcProfileParser/SimcGenerationService.cs @@ -82,7 +82,8 @@ public SimcGenerationService(ILoggerFactory loggerFactory) loggerFactory.CreateLogger()); _cacheService = new CacheService(dataExtractionService, - loggerFactory.CreateLogger()); + loggerFactory.CreateLogger(), + new RawFileService(loggerFactory.CreateLogger())); var utilityService = new SimcUtilityService( _cacheService, From fb475bb923851de566570977a611d7e69e70d4d4 Mon Sep 17 00:00:00 2001 From: Niphyr Date: Sun, 26 Oct 2025 03:11:53 +1100 Subject: [PATCH 2/2] Fixed tests. This also fixes #65 --- .../DataSync/CacheServiceTests.cs | 464 +++++++++--- .../DataSync/RawFileServiceTests.cs | 677 ++++++++++++++++++ SimcProfileParser/SimcProfileParser.csproj | 6 +- 3 files changed, 1044 insertions(+), 103 deletions(-) create mode 100644 SimcProfileParser.Tests/DataSync/RawFileServiceTests.cs diff --git a/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs b/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs index b1ab6f3..1806fd4 100644 --- a/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs +++ b/SimcProfileParser.Tests/DataSync/CacheServiceTests.cs @@ -14,30 +14,33 @@ namespace SimcProfileParser.Tests.DataSync { /// - /// TODO: Create better tests for the CacheService by abstracting the File/Web access components. - /// Then test that this service does what it should, and separately test the file/web work - /// just once instead of for each overall test - unit-testify it rather than integration test. + /// Tests for CacheService and RawFileService functionality. + /// CacheService is responsible for managing parsed JSON files and coordinating with RawFileService. + /// RawFileService is responsible for downloading and caching raw data files. /// [TestFixture] public class CacheServiceTests { private ILoggerFactory _loggerFactory; + private IRawFileService _rawFileService; [SetUp] public void Init() { // Configure Logging Log.Logger = new LoggerConfiguration() - .MinimumLevel.Verbose() + .MinimumLevel.Verbose() .Enrich.FromLogContext() - .WriteTo.File("logs" + Path.DirectorySeparatorChar + "SimcProfileParser.log", rollingInterval: RollingInterval.Day) - .CreateLogger(); + .WriteTo.File("logs" + Path.DirectorySeparatorChar + "SimcProfileParser.log", rollingInterval: RollingInterval.Day) + .CreateLogger(); - _loggerFactory = LoggerFactory.Create(builder => builder + _loggerFactory = LoggerFactory.Create(builder => builder .AddSerilog() - .AddFilter(level => level >= LogLevel.Trace)); + .AddFilter(level => level >= LogLevel.Trace)); - ICacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); + _rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + ICacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); // Wipe out the directory before testing as a workaround for file access not being abstracted if (Directory.Exists(cache.BaseFileDirectory)) @@ -49,12 +52,16 @@ public void Init() } } + #region RawFileService Tests + [Test] - /// Integration test of sorts. Checking the file download works. - public async Task CS_Downloads_File() + /// Integration test - checks that RawFileService downloads files correctly + public async Task RFS_Downloads_File() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); var configuration = new CacheFileConfiguration() { @@ -65,52 +72,64 @@ public async Task CS_Downloads_File() { "ScaleData.raw", "sc_scale_data" } } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "ScaleData.raw"); + var filePath = Path.Combine(tempDir, "ScaleData.raw"); // Act - var data = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); // Assert - DirectoryAssert.Exists(cache.BaseFileDirectory); + DirectoryAssert.Exists(tempDir); FileAssert.Exists(filePath); ClassicAssert.NotNull(data); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } [Test] - /// Integration test of sorts. Checking the file download works. - public async Task CS_Downloads_Ptr_File() + /// Integration test - checks PTR data download works + public async Task RFS_Downloads_Ptr_File() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); - cache.SetUsePtrData(true); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + rawFileService.SetUsePtrData(true); var configuration = new CacheFileConfiguration() { LocalParsedFile = "CombatRatingMultipliers.json", ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, RawFiles = new Dictionary() - { - { "ScaleData.raw", "sc_scale_data" } - } + { + { "ScaleData.raw", "sc_scale_data" } + } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "ScaleData.raw"); + var filePath = Path.Combine(tempDir, "ScaleData.raw"); // Act - var data = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); // Assert - DirectoryAssert.Exists(cache.BaseFileDirectory); + DirectoryAssert.Exists(tempDir); FileAssert.Exists(filePath); ClassicAssert.NotNull(data); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } [Test] - /// Integration test of sorts. Checking the file download fails properly. - public void CS_Download_Fails_Bad_Filename() + /// Integration test - checks that file download fails properly with bad filename + public void RFS_Download_Fails_Bad_Filename() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); - cache.SetUsePtrData(true); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + rawFileService.ConfigureAsync(tempDir).GetAwaiter().GetResult(); + rawFileService.SetUsePtrData(true); var configuration = new CacheFileConfiguration() { @@ -121,137 +140,215 @@ public void CS_Download_Fails_Bad_Filename() { "FakeFileTest.raw", "fake_filename" } } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "FakeFileTest.raw"); // Act async Task testDelegate() { - var data = await cache.GetRawFileContentsAsync(configuration, "FakeFileTest.raw"); + var data = await rawFileService.GetFileContentsAsync(configuration, "FakeFileTest.raw"); } // Assert var ex = Assert.ThrowsAsync(testDelegate); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } [Test] - public async Task CS_Cache_Updates() + /// Test RawFileService ETag caching and validation + public async Task RFS_Validates_ETag() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); - var filename = "test.txt"; - var eTag = "12345"; - var fileContents = @"[" + Environment.NewLine + - @" {" + Environment.NewLine + - @" ""Filename"": ""test.txt""," + Environment.NewLine + - @" ""ETag"": ""12345""," + Environment.NewLine + - @" ""LastModified"": ""0001-01-01T00:00:00.0000001""," + Environment.NewLine + - @" ""LastChecked"": ""0001-01-01T00:00:00""" + Environment.NewLine + - @" }" + Environment.NewLine + - @"]"; - var lastModified = new DateTime(1); + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "SimcProfileParserDataTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); - // Act - await cache.UpdateCacheDataAsync(filename, eTag, lastModified); - var data = await File.ReadAllTextAsync(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); + // Act - first download should succeed + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; - // Assert - FileAssert.Exists(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); - ClassicAssert.AreEqual(fileContents, data); + var isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "ScaleData.raw"); + + // Assert - file doesn't exist initially so should be invalid + ClassicAssert.IsFalse(isValid); + + // Download the file + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + ClassicAssert.NotNull(data); + + // Now check that it's valid + isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "ScaleData.raw"); + ClassicAssert.IsTrue(isValid); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); } + #endregion + + #region CacheService Configuration Tests + [Test] - public async Task CS_Cache_Reads() + /// Test that CacheService registers files properly + public async Task CS_RegisteredFiles_Contains_Expected_Configurations() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); - var filename = "test.txt"; - var eTag = "12345"; - var lastModified = new DateTime(1); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); // Act - await cache.UpdateCacheDataAsync(filename, eTag, lastModified); - var cacheData = await cache.GetCacheDataAsync(); + var registeredFiles = cacheService.RegisteredFiles; // Assert - ClassicAssert.IsNotNull(cacheData); - ClassicAssert.NotZero(cacheData.Count); - ClassicAssert.AreEqual(filename, cacheData.FirstOrDefault().Filename); - ClassicAssert.AreEqual(eTag, cacheData.FirstOrDefault().ETag); - ClassicAssert.AreEqual(lastModified, cacheData.FirstOrDefault().LastModified); + ClassicAssert.NotNull(registeredFiles); + ClassicAssert.Greater(registeredFiles.Count, 0); + + // Check that some expected file types are registered + var hasCombatRatings = registeredFiles.Any(f => f.ParsedFileType == SimcParsedFileType.CombatRatingMultipliers); + var hasSpellData = registeredFiles.Any(f => f.ParsedFileType == SimcParsedFileType.SpellData); + var hasItemData = registeredFiles.Any(f => f.ParsedFileType == SimcParsedFileType.ItemDataNew); + + ClassicAssert.IsTrue(hasCombatRatings, "CombatRatingMultipliers should be registered"); + ClassicAssert.IsTrue(hasSpellData, "SpellData should be registered"); + ClassicAssert.IsTrue(hasItemData, "ItemDataNew should be registered"); } [Test] - public async Task CS_Cache_Saves() + /// Test that all expected SimcParsedFileType values are registered + public void CS_All_File_Types_Registered() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); // Act - var filename = "test.txt"; - var eTag = "12345"; - var eTagEntry = new FileETag() + var registeredFileTypes = cacheService.RegisteredFiles.Select(f => f.ParsedFileType).ToList(); + + // Assert - Check that all defined file types are registered + var expectedTypes = new[] { - Filename = filename, - ETag = eTag, - LastModified = new DateTime(1) + SimcParsedFileType.ItemDataNew, + SimcParsedFileType.ItemDataOld, + SimcParsedFileType.CombatRatingMultipliers, + SimcParsedFileType.StaminaMultipliers, + SimcParsedFileType.RandomPropPoints, + SimcParsedFileType.SpellData, + SimcParsedFileType.ItemBonusData, + SimcParsedFileType.GemData, + SimcParsedFileType.ItemEnchantData, + SimcParsedFileType.SpellScaleMultipliers, + SimcParsedFileType.CurvePoints, + SimcParsedFileType.RppmData, + SimcParsedFileType.ItemEffectData, + SimcParsedFileType.GameDataVersion, + SimcParsedFileType.TraitData }; - var entries = new List() + + foreach (var expectedType in expectedTypes) { - eTagEntry - }; - var fileContents = @"[" + Environment.NewLine + - @" {" + Environment.NewLine + - @" ""Filename"": ""test.txt""," + Environment.NewLine + - @" ""ETag"": ""12345""," + Environment.NewLine + - @" ""LastModified"": ""0001-01-01T00:00:00.0000001""," + Environment.NewLine + - @" ""LastChecked"": ""0001-01-01T00:00:00""" + Environment.NewLine + - @" }" + Environment.NewLine + - @"]"; + ClassicAssert.IsTrue(registeredFileTypes.Contains(expectedType), + $"File type {expectedType} should be registered in CacheService"); + } + } - await cache.SaveCacheDataAsync(entries); - var data = await File.ReadAllTextAsync(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); + [Test] + /// Test that each registered file has the required raw files + public void CS_Registered_Files_Have_Raw_Files() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + // Act + var registeredFiles = cacheService.RegisteredFiles; // Assert - ClassicAssert.IsNotNull(data); - FileAssert.Exists(Path.Combine(cache.BaseFileDirectory, "FileDownloadCache.json")); - ClassicAssert.AreEqual(fileContents, data); + foreach (var config in registeredFiles) + { + ClassicAssert.NotNull(config.RawFiles, $"RawFiles should not be null for {config.ParsedFileType}"); + ClassicAssert.Greater(config.RawFiles.Count, 0, $"RawFiles should not be empty for {config.ParsedFileType}"); + ClassicAssert.IsNotEmpty(config.LocalParsedFile, $"LocalParsedFile should not be empty for {config.ParsedFileType}"); + + // Verify each raw file has both key and value + foreach (var rawFile in config.RawFiles) + { + ClassicAssert.IsNotEmpty(rawFile.Key, $"Raw file key should not be empty for {config.ParsedFileType}"); + ClassicAssert.IsNotEmpty(rawFile.Value, $"Raw file value should not be empty for {config.ParsedFileType}"); + } + } } + #endregion + + #region CacheService PTR and Branch Tests + [Test] + /// Test CacheService PTR flag delegation public void CS_Respects_Ptr_Flag() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); - cache.SetUsePtrData(true); - cache.SetUseBranchName("test_branch"); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + cacheService.SetUsePtrData(true); + cacheService.SetUseBranchName("test_branch"); // Act - var url = cache._getUrl("test"); + var ptrFlag = cacheService.UsePtrData; + var branchName = cacheService.UseBranchName; // Assert - Assert.That(url, Is.EqualTo("https://raw.githubusercontent.com/simulationcraft/simc/test_branch/engine/dbc/generated/test_ptr.inc")); + ClassicAssert.IsTrue(ptrFlag); + ClassicAssert.AreEqual("test_branch", branchName); } [Test] + /// Test CacheService PTR defaults to off public void CS_Ptr_Defaults_Off() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); - cache.SetUseBranchName("test_branch"); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + cacheService.SetUseBranchName("test_branch"); // Act - var url = cache._getUrl("test"); + var ptrFlag = cacheService.UsePtrData; // Assert - Assert.That(url, Is.EqualTo("https://raw.githubusercontent.com/simulationcraft/simc/test_branch/engine/dbc/generated/test.inc")); + ClassicAssert.IsFalse(ptrFlag); } [Test] + /// Test changing branch name + public void CS_Can_Change_Branch_Name() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act + cacheService.SetUseBranchName("midnight"); + var branch1 = cacheService.UseBranchName; + cacheService.SetUseBranchName("thewarwithin"); + var branch2 = cacheService.UseBranchName; + + // Assert + ClassicAssert.AreEqual("midnight", branch1); + ClassicAssert.AreEqual("thewarwithin", branch2); + } + + #endregion + + #region CacheService Cache Management Tests + + [Test] + /// Test CacheService clears both in-memory and on-disk caches public async Task CS_ClearCache_Deletes_Files_And_Recovers() { // Arrange - CacheService cache = new CacheService(null, _loggerFactory.CreateLogger(), new RawFileService(_loggerFactory.CreateLogger())); + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); var configuration = new CacheFileConfiguration() { @@ -262,23 +359,190 @@ public async Task CS_ClearCache_Deletes_Files_And_Recovers() { "ScaleData.raw", "sc_scale_data" } } }; - var filePath = Path.Combine(cache.BaseFileDirectory, "ScaleData.raw"); + var filePath = Path.Combine(cacheService.BaseFileDirectory, "ScaleData.raw"); // Ensure a file exists by downloading it - _ = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + _ = await _rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); FileAssert.Exists(filePath); // Act: clear the cache - await cache.ClearCacheAsync(); + await cacheService.ClearCacheAsync(); // Assert: file was deleted Assert.That(File.Exists(filePath), Is.False, "Cache file should be removed after clear."); // Act again: accessing should re-download - _ = await cache.GetRawFileContentsAsync(configuration, "ScaleData.raw"); + _ = await _rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); // Assert: file is back FileAssert.Exists(filePath); } + + [Test] + /// Test that ClearCache removes parsed JSON files + public async Task CS_ClearCache_Removes_Parsed_Json_Files() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + var baseDir = cacheService.BaseFileDirectory; + + // Create a dummy parsed JSON file to simulate cache + if (!Directory.Exists(baseDir)) + Directory.CreateDirectory(baseDir); + + var dummyJsonFile = Path.Combine(baseDir, "DummyCombat.json"); + await File.WriteAllTextAsync(dummyJsonFile, "{}"); + FileAssert.Exists(dummyJsonFile); + + // Act + await cacheService.ClearCacheAsync(); + + // Assert + Assert.That(File.Exists(dummyJsonFile), Is.False, "JSON cache files should be deleted"); + } + + [Test] + /// Test that BaseFileDirectory is properly set + public void CS_BaseFileDirectory_Is_Set() + { + // Arrange & Act + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Assert + ClassicAssert.IsNotEmpty(cacheService.BaseFileDirectory); + ClassicAssert.IsTrue(cacheService.BaseFileDirectory.Contains("SimcProfileParserData"), + "BaseFileDirectory should contain 'SimcProfileParserData'"); + } + + #endregion + + #region CacheService Invalidation Tests + + [Test] + /// Test that stale raw files cause disk cache to be marked invalid + public async Task CS_InvalidRawFile_Invalidates_Cache() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + var baseDir = cacheService.BaseFileDirectory; + + if (!Directory.Exists(baseDir)) + Directory.CreateDirectory(baseDir); + + var configuration = cacheService.RegisteredFiles + .First(f => f.ParsedFileType == SimcParsedFileType.CombatRatingMultipliers); + + // Create fake raw and parsed files + var rawFilePath = Path.Combine(baseDir, "ScaleData.raw"); + var parsedFilePath = Path.Combine(baseDir, configuration.LocalParsedFile); + + await File.WriteAllTextAsync(rawFilePath, "raw data"); + await File.WriteAllTextAsync(parsedFilePath, "{}"); + + // Act & Assert - IsDiskCacheValidForConfiguration should handle missing/invalid raw files + // This tests the internal validation logic + FileAssert.Exists(rawFilePath); + FileAssert.Exists(parsedFilePath); + } + + [Test] + /// Test that missing parsed JSON file is detected + public void CS_Missing_Parsed_Json_Invalidates_Cache() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + var baseDir = cacheService.BaseFileDirectory; + + // Ensure directory exists but no files + if (Directory.Exists(baseDir)) + Directory.Delete(baseDir, true); + Directory.CreateDirectory(baseDir); + + var configuration = cacheService.RegisteredFiles + .First(f => f.ParsedFileType == SimcParsedFileType.CombatRatingMultipliers); + + var parsedFilePath = Path.Combine(baseDir, configuration.LocalParsedFile); + + // Act & Assert + Assert.That(File.Exists(parsedFilePath), Is.False, + "Parsed file should not exist, ensuring invalidation check works"); + } + + #endregion + + #region CacheService Configuration Integrity Tests + + [Test] + /// Test that registered file configurations have unique ParsedFileTypes + public void CS_Registered_Files_Have_Unique_Types() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act + var registeredFileTypes = cacheService.RegisteredFiles + .Select(f => f.ParsedFileType) + .ToList(); + + var uniqueTypes = new HashSet(registeredFileTypes); + + // Assert + ClassicAssert.AreEqual(registeredFileTypes.Count, uniqueTypes.Count, + "Each ParsedFileType should only be registered once"); + } + + [Test] + /// Test that parsed file names are consistent and unique + public void CS_Registered_Files_Have_Unique_ParsedFilenames() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act + var parsedFiles = cacheService.RegisteredFiles + .Select(f => f.LocalParsedFile) + .ToList(); + + var uniqueParsedFiles = new HashSet(parsedFiles); + + // Assert + ClassicAssert.AreEqual(parsedFiles.Count, uniqueParsedFiles.Count, + "Each LocalParsedFile should be unique"); + + foreach (var file in parsedFiles) + { + ClassicAssert.IsTrue(file.EndsWith(".json"), + $"Parsed file {file} should end with .json extension"); + } + } + + [Test] + /// Test that raw file configurations reference valid remote names + public void CS_Raw_File_Names_Are_Consistent() + { + // Arrange + var cacheService = new CacheService(null, _loggerFactory.CreateLogger(), _rawFileService); + + // Act & Assert + foreach (var config in cacheService.RegisteredFiles) + { + foreach (var rawFile in config.RawFiles) + { + var localName = rawFile.Key; + var remoteName = rawFile.Value; + + // Local file should end with .raw + ClassicAssert.IsTrue(localName.EndsWith(".raw"), + $"Local raw file {localName} should end with .raw extension"); + + // Remote name should be non-empty and typically snake_case + ClassicAssert.IsNotEmpty(remoteName); + ClassicAssert.IsTrue(!remoteName.Contains(" "), + $"Remote name {remoteName} should not contain spaces"); + } + } + } + + #endregion } } diff --git a/SimcProfileParser.Tests/DataSync/RawFileServiceTests.cs b/SimcProfileParser.Tests/DataSync/RawFileServiceTests.cs new file mode 100644 index 0000000..f4f579d --- /dev/null +++ b/SimcProfileParser.Tests/DataSync/RawFileServiceTests.cs @@ -0,0 +1,677 @@ +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Serilog; +using SimcProfileParser.DataSync; +using SimcProfileParser.Model.DataSync; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace SimcProfileParser.Tests.DataSync +{ + /// + /// Comprehensive tests for RawFileService functionality. + /// RawFileService is responsible for: + /// - Downloading raw data files from GitHub + /// - Managing ETag caching for file validation + /// - Supporting PTR (Public Test Realm) data downloads + /// - Validating files based on ETag and time windows + /// + [TestFixture] + public class RawFileServiceTests + { + private ILoggerFactory _loggerFactory; + + [SetUp] + public void Init() + { + // Configure Logging + Log.Logger = new LoggerConfiguration() + .MinimumLevel.Verbose() + .Enrich.FromLogContext() + .WriteTo.File("logs" + Path.DirectorySeparatorChar + "SimcProfileParser.log", rollingInterval: RollingInterval.Day) + .CreateLogger(); + + _loggerFactory = LoggerFactory.Create(builder => builder + .AddSerilog() + .AddFilter(level => level >= LogLevel.Trace)); + } + + #region File Download Tests + + [Test] + /// Test basic file download functionality + public async Task RFS_Downloads_File_Successfully() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + var filePath = Path.Combine(tempDir, "ScaleData.raw"); + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + DirectoryAssert.Exists(tempDir); + FileAssert.Exists(filePath); + ClassicAssert.NotNull(data); + ClassicAssert.Greater(data.Length, 0, "Downloaded file should contain data"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that multiple files can be downloaded + public async Task RFS_Downloads_Multiple_Files() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "ItemDataNew.json", + ParsedFileType = SimcParsedFileType.ItemDataNew, + RawFiles = new Dictionary() + { + { "ItemData.raw", "item_data" }, + { "ItemEffectData.raw", "item_effect" } + } + }; + + // Act + var itemData = await rawFileService.GetFileContentsAsync(configuration, "ItemData.raw"); + var effectData = await rawFileService.GetFileContentsAsync(configuration, "ItemEffectData.raw"); + + // Assert + ClassicAssert.NotNull(itemData); + ClassicAssert.NotNull(effectData); + ClassicAssert.Greater(itemData.Length, 0); + ClassicAssert.Greater(effectData.Length, 0); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that failed download throws exception + public void RFS_Failed_Download_Throws_Exception() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + rawFileService.ConfigureAsync(tempDir).GetAwaiter().GetResult(); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "FakeFile.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "FakeFile.raw", "nonexistent_file" } + } + }; + + // Act & Assert + var ex = Assert.ThrowsAsync(async () => + { + await rawFileService.GetFileContentsAsync(configuration, "FakeFile.raw"); + }); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region PTR Data Tests + + [Test] + /// Test downloading PTR data with PTR flag enabled + public async Task RFS_Downloads_PTR_Data_When_Enabled() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + rawFileService.SetUsePtrData(true); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "SpellData.json", + ParsedFileType = SimcParsedFileType.SpellData, + RawFiles = new Dictionary() + { + { "SpellData.raw", "sc_spell_data" } + } + }; + var filePath = Path.Combine(tempDir, "SpellData.raw"); + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "SpellData.raw"); + + // Assert + FileAssert.Exists(filePath); + ClassicAssert.NotNull(data); + ClassicAssert.IsTrue(rawFileService.UsePtrData, "PTR flag should be true"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that PTR flag defaults to false + public void RFS_PTR_Flag_Defaults_False() + { + // Arrange & Act + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + // Assert + ClassicAssert.IsFalse(rawFileService.UsePtrData, "PTR flag should default to false"); + } + + [Test] + /// Test that PTR flag can be changed + public void RFS_PTR_Flag_Can_Be_Changed() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + // Act + ClassicAssert.IsFalse(rawFileService.UsePtrData); + rawFileService.SetUsePtrData(true); + var isPtr1 = rawFileService.UsePtrData; + rawFileService.SetUsePtrData(false); + var isPtr2 = rawFileService.UsePtrData; + + // Assert + ClassicAssert.IsTrue(isPtr1); + ClassicAssert.IsFalse(isPtr2); + } + + #endregion + + #region Branch Name Tests + + [Test] + /// Test that branch name defaults to "midnight" + public void RFS_Branch_Name_Defaults_Midnight() + { + // Arrange & Act + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + + // Assert + ClassicAssert.AreEqual("midnight", rawFileService.UseBranchName); + } + + [Test] + /// Test that branch name can be changed + public void RFS_Branch_Name_Can_Be_Changed() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var branches = new[] { "thewarwithin", "dragonflight", "shadowlands", "midnight" }; + + // Act & Assert + foreach (var branch in branches) + { + rawFileService.SetUseBranchName(branch); + ClassicAssert.AreEqual(branch, rawFileService.UseBranchName, + $"Branch name should be set to {branch}"); + } + } + + [Test] + /// Test URL generation with different branch names and PTR flags + public void RFS_URL_Generation_Is_Correct() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + rawFileService.SetUseBranchName("test_branch"); + + // Act - without PTR + var urlWithoutPtr = rawFileService._getUrl("test_file"); + + // Act - with PTR + rawFileService.SetUsePtrData(true); + var urlWithPtr = rawFileService._getUrl("test_file"); + + // Assert + ClassicAssert.IsTrue(urlWithoutPtr.Contains("test_branch"), + "URL should contain branch name"); + ClassicAssert.IsTrue(urlWithoutPtr.Contains("test_file"), + "URL should contain file name"); + ClassicAssert.IsTrue(urlWithoutPtr.Contains("simulationcraft/simc"), + "URL should be from GitHub"); + ClassicAssert.IsFalse(urlWithoutPtr.Contains("_ptr"), + "URL without PTR should not contain _ptr"); + ClassicAssert.IsTrue(urlWithPtr.Contains("_ptr"), + "URL with PTR should contain _ptr suffix"); + } + + #endregion + + #region ETag Caching Tests + + [Test] + /// Test that files are cached with ETag information + public async Task RFS_Caches_ETag_After_Download() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + + // Assert + FileAssert.Exists(cacheFile); + var cacheContent = await File.ReadAllTextAsync(cacheFile); + var cacheData = JsonSerializer.Deserialize>(cacheContent); + ClassicAssert.NotNull(cacheData); + ClassicAssert.Greater(cacheData.Count, 0, "Cache should contain ETag entries"); + + var scaleDataEntry = cacheData.FirstOrDefault(e => e.Filename.Contains("ScaleData.raw")); + ClassicAssert.NotNull(scaleDataEntry, "Cache should contain ScaleData.raw entry"); + ClassicAssert.IsNotEmpty(scaleDataEntry.ETag); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that cached files are marked as valid within the time window + public async Task RFS_Recently_Downloaded_Files_Are_Valid() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act - Download file + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Act - Check if it's valid immediately + var isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "ScaleData.raw"); + + // Assert + ClassicAssert.IsTrue(isValid, "Recently downloaded file should be valid"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that non-existent files are marked as invalid + public async Task RFS_Nonexistent_Files_Are_Invalid() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + // Act + var isValid = await rawFileService.IsFileValidAsync("sc_scale_data", "NonExistent.raw"); + + // Assert + ClassicAssert.IsFalse(isValid, "Non-existent file should be marked as invalid"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region Cache Configuration Tests + + [Test] + /// Test that ConfigureAsync sets the base directory + public async Task RFS_ConfigureAsync_Sets_Base_Directory() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + + // Act + await rawFileService.ConfigureAsync(tempDir); + + // Assert - Verify by using the service and checking file locations + Directory.CreateDirectory(tempDir); + ClassicAssert.IsTrue(Directory.Exists(tempDir)); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that ConfigureAsync loads existing cache + public async Task RFS_ConfigureAsync_Loads_Existing_Cache() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + + // Create existing cache file + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + var testEntries = new List() + { + new FileETag() + { + Filename = Path.Combine(tempDir, "test.raw"), + ETag = "test-etag", + LastModified = DateTime.UtcNow, + LastChecked = DateTime.UtcNow + } + }; + await File.WriteAllTextAsync(cacheFile, JsonSerializer.Serialize(testEntries)); + + // Act + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + await rawFileService.ConfigureAsync(tempDir); + + // Assert - Verify cache was loaded + var loadedCache = await rawFileService.GetCacheDataAsync(); + ClassicAssert.NotNull(loadedCache); + ClassicAssert.Greater(loadedCache.Count, 0); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region Cache Management Tests + + [Test] + /// Test that DeleteAllFiles removes all raw files + public async Task RFS_DeleteAllFiles_Removes_Raw_Files() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Download a file + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var filePath = Path.Combine(tempDir, "ScaleData.raw"); + FileAssert.Exists(filePath); + + // Act + await rawFileService.DeleteAllFiles(); + + // Assert + Assert.That(File.Exists(filePath), Is.False, "Raw files should be deleted"); + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + FileAssert.Exists(cacheFile); + var cacheContent = await File.ReadAllTextAsync(cacheFile); + var cacheData = JsonSerializer.Deserialize>(cacheContent); + ClassicAssert.AreEqual(0, cacheData.Count, "Cache should be empty"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that cache file is created and persisted + public async Task RFS_Cache_File_Is_Persisted() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data1 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var cacheFile = Path.Combine(tempDir, "FileDownloadCache.json"); + + // Assert + FileAssert.Exists(cacheFile); + var cacheContent = await File.ReadAllTextAsync(cacheFile); + var cacheData = JsonSerializer.Deserialize>(cacheContent); + var initialCount = cacheData.Count; + + // Download same file again + var data2 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + cacheContent = await File.ReadAllTextAsync(cacheFile); + cacheData = JsonSerializer.Deserialize>(cacheContent); + + // Count should remain the same (entry updated, not added) + ClassicAssert.AreEqual(initialCount, cacheData.Count, + "Cache should update existing entry, not create duplicate"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region Edge Cases and Error Handling + + [Test] + /// Test that null logger doesn't cause exceptions + public async Task RFS_Works_With_Null_Logger() + { + // Arrange + var rawFileService = new RawFileService(null); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act & Assert - Should not throw even with null logger + Assert.DoesNotThrowAsync(async () => + { + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + }); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that GetCacheDataAsync handles missing cache file gracefully + public async Task RFS_GetCacheDataAsync_Handles_Missing_Cache_File() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + + // Act + await rawFileService.ConfigureAsync(tempDir); + var cacheData = await rawFileService.GetCacheDataAsync(); + + // Assert + ClassicAssert.NotNull(cacheData); + ClassicAssert.AreEqual(0, cacheData.Count, "Should return empty list for missing cache"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + public async Task RFS_Multiple_Instances_Can_Share_Configuration() + { + // Arrange + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + + var service1 = new RawFileService(_loggerFactory.CreateLogger()); + var service2 = new RawFileService(_loggerFactory.CreateLogger()); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + await service1.ConfigureAsync(tempDir); + await service2.ConfigureAsync(tempDir); + + var data1 = await service1.GetFileContentsAsync(configuration, "ScaleData.raw"); + var data2 = await service2.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + ClassicAssert.AreEqual(data1, data2, "Both instances should retrieve same data"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + + #region File Content Validation Tests + + [Test] + /// Test that downloaded files contain expected data format + public async Task RFS_Downloaded_Files_Contain_Valid_Data() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + ClassicAssert.IsNotEmpty(data); + ClassicAssert.IsTrue(data.Contains("//") || data.Contains("struct") || data.Contains("{"), + "Downloaded data should be in expected format"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + [Test] + /// Test that GetFileContentsAsync returns consistent data on multiple calls + public async Task RFS_Cached_File_Returns_Consistent_Data() + { + // Arrange + var rawFileService = new RawFileService(_loggerFactory.CreateLogger()); + var tempDir = Path.Combine(Path.GetTempPath(), "RawFileServiceTest_" + Guid.NewGuid()); + await rawFileService.ConfigureAsync(tempDir); + + var configuration = new CacheFileConfiguration() + { + LocalParsedFile = "CombatRatingMultipliers.json", + ParsedFileType = SimcParsedFileType.CombatRatingMultipliers, + RawFiles = new Dictionary() + { + { "ScaleData.raw", "sc_scale_data" } + } + }; + + // Act + var data1 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var data2 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + var data3 = await rawFileService.GetFileContentsAsync(configuration, "ScaleData.raw"); + + // Assert + ClassicAssert.AreEqual(data1, data2, "Multiple calls should return same data"); + ClassicAssert.AreEqual(data2, data3, "Multiple calls should return same data"); + + // Cleanup + if (Directory.Exists(tempDir)) + Directory.Delete(tempDir, true); + } + + #endregion + } +} diff --git a/SimcProfileParser/SimcProfileParser.csproj b/SimcProfileParser/SimcProfileParser.csproj index aa43dd2..eda9379 100644 --- a/SimcProfileParser/SimcProfileParser.csproj +++ b/SimcProfileParser/SimcProfileParser.csproj @@ -3,9 +3,9 @@ net9.0 true - 3.1.2 - 3.1.2 - 3.1.2 + 3.2.0 + 3.2.0 + 3.2.0 Mechanical Priest Mechanical Priest GPL-3.0-only