Skip to content

Add check for empty files #685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Suterusu1337 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add check for empty files #685

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 33 additions & 15 deletions PlayerSync/PlayerData/Pairs/VisibleUserDataDistributor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class VisibleUserDataDistributor : DisposableMediatorSubscriberBase
private readonly HashSet<UserData> _usersToPushDataTo = [];
private readonly SemaphoreSlim _pushDataSemaphore = new(1, 1);
private readonly CancellationTokenSource _runtimeCts = new();
private readonly HashSet<string> _filesTooLargeHashes = new HashSet<string>();
private readonly HashSet<string> _filesTooLargeOrEmptyHasBeenNotified = new HashSet<string>();


public VisibleUserDataDistributor(ILogger<VisibleUserDataDistributor> logger, ApiController apiController, DalamudUtilService dalamudUtil,
Expand Down Expand Up @@ -114,7 +114,7 @@ private void PushCharacterData(bool forced = false)
if (_fileUploadTask == null || (_fileUploadTask?.IsCompleted ?? false) || forced)
{
_uploadingCharacterData = _lastCreatedData.DeepClone();
RemoveFilesThatAreTooLarge(); // psync allows for 200 MiB to the server, 300 MiB once decompressed
RemoveFilesThatAreTooLargeOrEmpty(); // psync allows for 200 MiB to the server, 300 MiB once decompressed
Logger.LogDebug("Starting UploadTask for {hash}, Reason: TaskIsNull: {task}, TaskIsCompleted: {taskCpl}, Forced: {frc}",
_lastCreatedData.DataHash, _fileUploadTask == null, _fileUploadTask?.IsCompleted ?? false, forced);
_fileUploadTask = _fileTransferManager.UploadFiles(_uploadingCharacterData, [.. _usersToPushDataTo]);
Expand All @@ -139,41 +139,59 @@ private void PushCharacterData(bool forced = false)
});
}

private void RemoveFilesThatAreTooLarge()
private void RemoveFilesThatAreTooLargeOrEmpty()
{
try
{
// check for files that are larger than what the server allows for
Dictionary<ObjectKind, List<FileReplacementData>> filesTooLarge = new();
Dictionary<ObjectKind, List<FileReplacementData>> filesTooLargeorEmpty = new();
foreach (var fileReplacements in _uploadingCharacterData!.FileReplacements)
{
foreach (var replacement in fileReplacements.Value)
{
bool mustRemoveReplacementData = false;

var cache = _fileCacheManager.GetFileCacheByHash(replacement.Hash);
if (cache == null)
{
continue;
}

if ((cache.Size != null && cache.Size > 300 * 1024 * 1024) || (cache.CompressedSize != null && cache.CompressedSize > 200 * 1024 * 1024))
// check for empty files
if (cache.Size != null && cache.Size == 0)
{
mustRemoveReplacementData = true;
}

// check for files that are larger than what the server allows for
else if ((cache.Size != null && cache.Size > 300 * 1024 * 1024) || (cache.CompressedSize != null && cache.CompressedSize > 200 * 1024 * 1024))
{
Logger.LogWarning("File {file} exceeds the size limit to sync and will not be sent. (300 MiB, or 200 MiB compressed)", cache.ResolvedFilepath);
if (!filesTooLarge.Keys.Contains(fileReplacements.Key))
{
filesTooLarge[fileReplacements.Key] = new();
}
filesTooLarge[fileReplacements.Key].Add(replacement);

if (!_filesTooLargeHashes.Contains(cache.Hash) && _configService.Current.ShowFileUnableToSyncNotification)
if (!_filesTooLargeOrEmptyHasBeenNotified.Contains(cache.Hash) && _configService.Current.ShowFileUnableToSyncNotification)
{
var isTexFile = cache.ResolvedFilepath.EndsWith(".tex", StringComparison.OrdinalIgnoreCase);
var texInfo = isTexFile ? " Ensure it is compressed and consider reducing its resolution." : "";
Mediator.Publish(new NotificationMessage("File Size Error", $"The file {cache.ResolvedFilepath} exceeds the size limit and will not sync. (300 MiB, or 200 MiB compressed)" + texInfo, MareConfiguration.Models.NotificationType.Warning));
_filesTooLargeOrEmptyHasBeenNotified.Add(cache.Hash);
}

mustRemoveReplacementData = true;
}

_filesTooLargeHashes.Add(cache.Hash);
if (mustRemoveReplacementData)
{
if (!filesTooLargeorEmpty.ContainsKey(fileReplacements.Key))
{
// create an empty replacements list for this ObjectKind if ObjectKind not already in the dictionary
filesTooLargeorEmpty[fileReplacements.Key] = [];
}
filesTooLargeorEmpty[fileReplacements.Key].Add(replacement);
}
}
}
foreach (var replacements in filesTooLarge)

// check each ObjectKind that is present for any items in its replacement list to remove
foreach (var replacements in filesTooLargeorEmpty)
{
foreach (var replacement in replacements.Value)
{
Expand All @@ -183,7 +201,7 @@ private void RemoveFilesThatAreTooLarge()
}
catch (Exception ex)
{
Logger.LogError(ex, "Failed to properly detect if files to upload are too large.");
Logger.LogError(ex, "Failed to properly detect if files to upload are too large or empty.");
}
}
}