From 4fa63e9a39dff79261283d4e4db9c248cc43f3cc Mon Sep 17 00:00:00 2001 From: MattEqualsCoder Date: Fri, 23 Aug 2024 17:50:27 -0400 Subject: [PATCH] Fix sprite downloads --- .../Options/GeneralOptions.cs | 5 - .../Services/GitHubSpriteDownloaderService.cs | 96 +++++++++++++------ .../IGitHubSpriteDownloaderService.cs | 3 +- .../Services/OptionsWindowService.cs | 4 +- .../Services/MainWindowService.cs | 8 -- 5 files changed, 71 insertions(+), 45 deletions(-) diff --git a/src/TrackerCouncil.Smz3.Data/Options/GeneralOptions.cs b/src/TrackerCouncil.Smz3.Data/Options/GeneralOptions.cs index 5ef033898..420b670bc 100644 --- a/src/TrackerCouncil.Smz3.Data/Options/GeneralOptions.cs +++ b/src/TrackerCouncil.Smz3.Data/Options/GeneralOptions.cs @@ -217,11 +217,6 @@ public string? TwitchId /// public List ConfigSources { get; set; } = new(); - /// - /// Dictionary of previously downloaded hashes - /// - public Dictionary SpriteHashes { get; set; } = new(); - /// /// Device to be used for push to talk mode /// diff --git a/src/TrackerCouncil.Smz3.Data/Services/GitHubSpriteDownloaderService.cs b/src/TrackerCouncil.Smz3.Data/Services/GitHubSpriteDownloaderService.cs index 45ee34c05..ebae06f0b 100644 --- a/src/TrackerCouncil.Smz3.Data/Services/GitHubSpriteDownloaderService.cs +++ b/src/TrackerCouncil.Smz3.Data/Services/GitHubSpriteDownloaderService.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using TrackerCouncil.Smz3.Data.Options; +using YamlDotNet.Serialization; namespace TrackerCouncil.Smz3.Data.Services; @@ -19,13 +20,11 @@ public class GitHubSpriteDownloaderService : IGitHubSpriteDownloaderService { private ILogger _logger; private string _spriteFolder; - private OptionsFactory _optionsFactory; private CancellationTokenSource _cts = new(); - public GitHubSpriteDownloaderService(ILogger logger, OptionsFactory optionsFactory) + public GitHubSpriteDownloaderService(ILogger logger) { _logger = logger; - _optionsFactory = optionsFactory; _spriteFolder = Sprite.SpritePath; _logger.LogInformation("Sprite path: {Path} | {OtherPath}", Sprite.SpritePath, AppContext.BaseDirectory); } @@ -34,7 +33,7 @@ public GitHubSpriteDownloaderService(ILogger logg public event EventHandler? SpriteDownloadUpdate; - public async Task?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null) + public async Task?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null, bool ignoreNoPreviousHashes = false) { var sprites = await GetGitHubSpritesAsync(owner, repo, timeout); if (sprites == null) @@ -43,6 +42,13 @@ public GitHubSpriteDownloaderService(ILogger logg } var previousHashes = GetPreviousSpriteHashes(); + + // If there are no previous hashes, don't download any sprites + if (!ignoreNoPreviousHashes && previousHashes.Count == 0) + { + return new Dictionary(); + } + var toDownload = new ConcurrentDictionary(); Parallel.ForEach(sprites, parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = 4 }, @@ -66,7 +72,7 @@ public GitHubSpriteDownloaderService(ILogger logg public async Task DownloadSpritesAsync(string owner, string repo, IDictionary? spritesToDownload = null, TimeSpan? timeout = null) { spritesToDownload ??= await GetSpritesToDownloadAsync(owner, repo, timeout); - if (spritesToDownload == null) + if (spritesToDownload == null || spritesToDownload.Count == 0) { return; } @@ -78,13 +84,13 @@ public async Task DownloadSpritesAsync(string owner, string repo, IDictionary(); + var spriteHashes = GetPreviousSpriteHashes(); + var addedHashes = new ConcurrentDictionary(); var total = spritesToDownload.Count; var completed = 0; - if (spritesToDownload?.Any() == true) + if (spritesToDownload.Any()) { await Parallel.ForEachAsync(spritesToDownload, parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = 4, CancellationToken = _cts.Token}, async (spriteData, _) => @@ -96,45 +102,70 @@ public async Task DownloadSpritesAsync(string owner, string repo, IDictionary GetPreviousSpriteHashes() { - if (File.Exists(Path.Combine(_spriteFolder, "sprites.json"))) + if (File.Exists(GitHubSpriteJsonFilePath)) { - var spriteJson = File.ReadAllText(Path.Combine(_spriteFolder, "sprites.json")); + var spriteJson = File.ReadAllText(GitHubSpriteJsonFilePath); var tree = JsonSerializer.Deserialize(spriteJson); - if (tree?.tree?.Any() == true) + if (tree?.tree == null || tree.tree.Count == 0) { - _logger.LogInformation("Loading previous sprite hashes from {Path}", Path.Combine(_spriteFolder, "sprites.json")); - return tree.tree - .Where(IsValidSpriteFile) - .ToDictionary(x => ConvertGitHubPath(x.path), x => x.sha); + File.Delete(GitHubSpriteJsonFilePath); + return []; + } + + _logger.LogInformation("Loading previous sprite hashes from {Path}", GitHubSpriteJsonFilePath); + + var toReturn = tree.tree + .Where(IsValidSpriteFile) + .ToDictionary(x => ConvertGitHubPath(x.path), x => x.sha); + + SaveSpriteHashYaml(toReturn); + File.Delete(GitHubSpriteJsonFilePath); + + return toReturn; + } + else if (File.Exists(SpriteHashYamlFilePath)) + { + var yamlText = File.ReadAllText(SpriteHashYamlFilePath); + var serializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .Build(); + try + { + return serializer.Deserialize>(yamlText); + } + catch (Exception e) + { + _logger.LogError(e, "Error deserializing sprite hash yaml file {Path}", SpriteHashYamlFilePath); } } - return _optionsFactory.Create().GeneralOptions.SpriteHashes; + return []; + } + + private void SaveSpriteHashYaml(Dictionary hashes) + { + var serializer = new Serializer(); + var yamlText = serializer.Serialize(hashes); + File.WriteAllText(SpriteHashYamlFilePath, yamlText); } private async Task DownloadFileAsync(string destination, string url, int attempts = 2) @@ -197,7 +228,6 @@ private async Task DownloadFileAsync(string destination, string url, int a _logger.LogInformation("Retrieved {Count} file data from GitHub", tree.tree.Count); - return tree.tree .Where(IsValidSpriteFile) .ToDictionary(x => x.path, x => x.sha); @@ -227,7 +257,7 @@ private class GitHubTree public List? tree { get; set; } } - public class GitHubFile + private class GitHubFile { public required string path { get; set; } public required string mode { get; set; } @@ -235,4 +265,12 @@ public class GitHubFile public required string sha { get; set; } public required string url { get; set; } } + +#if DEBUG + private string SpriteHashYamlFilePath => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SMZ3CasRandomizer", "sprite-hashes-debug.yml"); +#else + private string SpriteHashYamlFilePath => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SMZ3CasRandomizer", "sprite-hashes.yml"); +#endif + + private string GitHubSpriteJsonFilePath => Path.Combine(_spriteFolder, "sprites.json"); } diff --git a/src/TrackerCouncil.Smz3.Data/Services/IGitHubSpriteDownloaderService.cs b/src/TrackerCouncil.Smz3.Data/Services/IGitHubSpriteDownloaderService.cs index 3cd551477..6da252330 100644 --- a/src/TrackerCouncil.Smz3.Data/Services/IGitHubSpriteDownloaderService.cs +++ b/src/TrackerCouncil.Smz3.Data/Services/IGitHubSpriteDownloaderService.cs @@ -15,8 +15,9 @@ public interface IGitHubSpriteDownloaderService /// The GitHub repository owner /// The GitHub repository to download /// How long to timeout from the api call + /// If downloads should be triggered, even if no previous hashes exist /// A dictionary of paths on GitHub and their git hashes - public Task?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null); + public Task?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null, bool ignoreNoPreviousHashes = false); /// /// Downloads sprites from GitHub to the folder diff --git a/src/TrackerCouncil.Smz3.Data/Services/OptionsWindowService.cs b/src/TrackerCouncil.Smz3.Data/Services/OptionsWindowService.cs index 8b455c7c8..9031550c8 100644 --- a/src/TrackerCouncil.Smz3.Data/Services/OptionsWindowService.cs +++ b/src/TrackerCouncil.Smz3.Data/Services/OptionsWindowService.cs @@ -192,12 +192,12 @@ private async Task UpdateConfigsAsync() private async Task UpdateSpritesAsync() { - var sprites = await gitHubSpriteDownloaderService.GetSpritesToDownloadAsync("TheTrackerCouncil", "SMZ3CasSprites"); + var sprites = await gitHubSpriteDownloaderService.GetSpritesToDownloadAsync("TheTrackerCouncil", "SMZ3CasSprites", null, true); if (sprites?.Any() == true) { SpriteDownloadStarted?.Invoke(this, EventArgs.Empty); - await gitHubSpriteDownloaderService.DownloadSpritesAsync("TheTrackerCouncil", "SMZ3CasSprites"); + await gitHubSpriteDownloaderService.DownloadSpritesAsync("TheTrackerCouncil", "SMZ3CasSprites", sprites); SpriteDownloadEnded?.Invoke(this, EventArgs.Empty); } } diff --git a/src/TrackerCouncil.Smz3.UI/Services/MainWindowService.cs b/src/TrackerCouncil.Smz3.UI/Services/MainWindowService.cs index 61fb32cbb..8a02f2679 100644 --- a/src/TrackerCouncil.Smz3.UI/Services/MainWindowService.cs +++ b/src/TrackerCouncil.Smz3.UI/Services/MainWindowService.cs @@ -113,14 +113,6 @@ public async Task DownloadSpritesAsync() return; } - #if DEBUG - // Skip sprite download when debugging if the folder already exists - if (Directory.Exists(Sprite.SpritePath)) - { - return; - } - #endif - var toDownload = await gitHubSpriteDownloaderService.GetSpritesToDownloadAsync("TheTrackerCouncil", "SMZ3CasSprites"); if (toDownload is not { Count: > 4 })