Merge pull request #560 from TheTrackerCouncil/tracker-speech-window

Add tracker speech window

src/TrackerCouncil.Smz3.Data/Options/GeneralOptions.cs

@@ -47,10 +47,14 @@ public class GeneralOptions : INotifyPropertyChanged
     [Range(0.0, 1.0)]
     public float TrackerConfidenceSassThreshold { get; set; } = 0.92f;
 
-    public byte[] TrackerBGColor { get; set; } = { 0xFF, 0x21, 0x21, 0x21 };
+    public byte[] TrackerBGColor { get; set; } = [0xFF, 0x21, 0x21, 0x21];
 
     public bool TrackerShadows { get; set; } = true;
 
+    public byte[] TrackerSpeechBGColor { get; set; } = [0xFF, 0x48, 0x3D, 0x8B];
+
+    public bool TrackerSpeechEnableBounce { get; set; } = true;
+
     [YamlIgnore]
     public int LaunchButton { get; set; } = (int)LaunchButtonOptions.PlayAndTrack;
 
@@ -172,6 +176,11 @@ public string? TwitchId
     /// </summary>
     public bool DisplayMsuTrackWindow { get; set; }
 
+    /// <summary>
+    /// If the tracker speech window should open by default
+    /// </summary>
+    public bool DisplayTrackerSpeechWindow { get; set; }
+
     /// <summary>
     /// Check for new releases on GitHub on startup
     /// </summary>

src/TrackerCouncil.Smz3.Data/ViewModels/OptionsWindowTrackerOptions.cs

@@ -18,6 +18,12 @@ public class OptionsWindowTrackerOptions
     [DynamicFormFieldCheckBox(checkBoxText: "Render shadows", alignment: DynamicFormAlignment.Right)]
     public bool TrackerShadows { get; set; } = true;
 
+    [DynamicFormFieldColorPicker(label: "Tracker speech window color:")]
+    public byte[] TrackerSpeechBGColor { get; set; } = [0xFF, 0x48, 0x3D, 0x8B];
+
+    [DynamicFormFieldCheckBox(checkBoxText: "Enable speech bounce animation", alignment: DynamicFormAlignment.Right)]
+    public bool TrackerSpeechEnableBounce { get; set; } = true;
+
     [DynamicFormFieldSlider(minimumValue: 0, maximumValue:100, decimalPlaces:1, incrementAmount:.1, suffix:"%", label: "Tracker recognition threshold:", platforms: DynamicFormPlatform.Windows)]
     public float TrackerRecognitionThreshold { get; set; }
 

src/TrackerCouncil.Smz3.Data/ViewModels/OptionsWindowViewModel.cs

@@ -34,6 +34,8 @@ public OptionsWindowViewModel(GeneralOptions options, Dictionary<string, string>
 
         TrackerOptions.TrackerBGColor = options.TrackerBGColor;
         TrackerOptions.TrackerShadows = options.TrackerShadows;
+        TrackerOptions.TrackerSpeechBGColor = options.TrackerSpeechBGColor;
+        TrackerOptions.TrackerSpeechEnableBounce = options.TrackerSpeechEnableBounce;
         TrackerOptions.TrackerRecognitionThreshold = options.TrackerRecognitionThreshold * 100;
         TrackerOptions.TrackerConfidenceThreshold = options.TrackerConfidenceThreshold * 100;
         TrackerOptions.TrackerConfidenceSassThreshold = options.TrackerConfidenceSassThreshold * 100;
@@ -84,6 +86,8 @@ public void UpdateOptions(GeneralOptions options)
 
         options.TrackerBGColor = TrackerOptions.TrackerBGColor;
         options.TrackerShadows = TrackerOptions.TrackerShadows;
+        options.TrackerSpeechBGColor = TrackerOptions.TrackerSpeechBGColor;
+        options.TrackerSpeechEnableBounce = TrackerOptions.TrackerSpeechEnableBounce;
         options.TrackerRecognitionThreshold = TrackerOptions.TrackerRecognitionThreshold / 100;
         options.TrackerConfidenceThreshold = TrackerOptions.TrackerConfidenceThreshold / 100;
         options.TrackerConfidenceSassThreshold = TrackerOptions.TrackerConfidenceSassThreshold / 100;

src/TrackerCouncil.Smz3.Tracking/Services/ICommunicator.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Speech.Synthesis;
 
 namespace TrackerCouncil.Smz3.Tracking.Services;
 
@@ -69,10 +70,20 @@ public void SlowDown() { }
     /// </summary>
     public bool IsSpeaking { get;  }
 
+    /// <summary>
+    /// Event for when the communicator has started speaking
+    /// </summary>
+    public event EventHandler SpeakStarted;
+
     /// <summary>
     /// Event for when the communicator has finished speaking
     /// </summary>
     public event EventHandler<SpeakCompletedEventArgs> SpeakCompleted;
 
+    /// <summary>
+    /// Event for when the communicator has reached a new viseme
+    /// </summary>
+    public event EventHandler<VisemeReachedEventArgs> VisemeReached;
+
 
 }

src/TrackerCouncil.Smz3.Tracking/Services/IUIService.cs

@@ -74,4 +74,12 @@ public interface IUIService
     /// <param name="basePath">The base path of the desired sprite</param>
     /// <returns>The full path of the sprite or null if it's not found</returns>
     public string? GetSpritePath(string category, string imageFileName, out string? profilePath, string? basePath = null);
+
+    /// <summary>
+    /// Gets the images for tracker talking
+    /// </summary>
+    /// <param name="profilePath">The selected profile</param>
+    /// <param name="basePath">The base path of the folder used</param>
+    /// <returns>A dictionary of all of the available tracker speech images</returns>
+    public Dictionary<string, TrackerSpeechImages> GetTrackerSpeechSprites(out string? profilePath, string? basePath = null);
 }

src/TrackerCouncil.Smz3.Tracking/Services/TextToSpeechCommunicator.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Speech.Synthesis;
 using Microsoft.Extensions.Logging;
 using TrackerCouncil.Smz3.Data.Options;
@@ -34,6 +35,7 @@ public TextToSpeechCommunicator(TrackerOptionsAccessor trackerOptionsAccessor, I
             if (IsSpeaking) return;
             _startSpeakingTime = DateTime.Now;
             IsSpeaking = true;
+            SpeakStarted?.Invoke(this, EventArgs.Empty);
         };
 
         _tts.SpeakCompleted += (sender, args) =>
@@ -44,6 +46,12 @@ public TextToSpeechCommunicator(TrackerOptionsAccessor trackerOptionsAccessor, I
             SpeakCompleted?.Invoke(this, new SpeakCompletedEventArgs(duration));
         };
 
+        _tts.VisemeReached += (sender, args) =>
+        {
+            if (!OperatingSystem.IsWindows()) return;
+            VisemeReached?.Invoke(this, args);
+        };
+
         _canSpeak = trackerOptionsAccessor.Options?.VoiceFrequency != Shared.Enums.TrackerVoiceFrequency.Disabled;
     }
 
@@ -143,8 +151,12 @@ public void SlowDown()
 
     public bool IsSpeaking { get; private set; }
 
+    public event EventHandler? SpeakStarted;
+
     public event EventHandler<SpeakCompletedEventArgs>? SpeakCompleted;
 
+    public event EventHandler<VisemeReachedEventArgs>? VisemeReached;
+
     /// <inheritdoc/>
     public void Dispose()
     {

src/TrackerCouncil.Smz3.Tracking/Services/UIService.cs

@@ -165,4 +165,80 @@ UIConfig uiConfig
         profilePath = null;
         return null;
     }
+
+    /// <summary>
+    /// Gets the images for tracker talking
+    /// </summary>
+    /// <param name="profilePath">The selected profile</param>
+    /// <param name="basePath">The base path of the folder used</param>
+    /// <returns>A dictionary of all of the available tracker speech images</returns>
+    public Dictionary<string, TrackerSpeechImages> GetTrackerSpeechSprites(out string? profilePath, string? basePath = null)
+    {
+        var toReturn = new Dictionary<string, TrackerSpeechImages>();
+
+        foreach (var idleSprite in GetCategorySprites("Tracker", out profilePath, basePath).Where(x => x.EndsWith("_idle.png")))
+        {
+            var file = new FileInfo(idleSprite);
+
+            var reaction = file.Name.Replace("_idle.png", "").ToLower();
+            var talkSprite = idleSprite.Replace("_idle.png", "_talk.png");
+
+            if (File.Exists(talkSprite))
+            {
+                toReturn.Add(reaction, new TrackerSpeechImages()
+                {
+                    ReactionName = reaction,
+                    IdleImage = idleSprite,
+                    TalkingImage = talkSprite,
+                });
+            }
+        }
+
+        return toReturn;
+    }
+
+    /// <summary>
+    /// Returns all of the sprites for a category
+    /// </summary>
+    /// <param name="category">The category of sprite</param>
+    /// <param name="profilePath">The path of the selected profile</param>
+    /// <param name="basePath">The base path of the desired sprite</param>
+    /// <returns>The full path of the sprite or null if it's not found</returns>
+    public List<string> GetCategorySprites(string category, out string? profilePath, string? basePath = null)
+    {
+        var toReturn = new List<string>();
+
+        if (!string.IsNullOrEmpty(basePath))
+        {
+            var path = Path.Combine(basePath, category);
+            if (Directory.Exists(path))
+            {
+                foreach (var spritePath in Directory.EnumerateFiles(path, "*.png"))
+                {
+                    toReturn.Add(spritePath);
+                }
+                profilePath = basePath;
+                return toReturn;
+            }
+        }
+        else
+        {
+            foreach (var profile in _iconPaths)
+            {
+                var path = Path.Combine(profile, category);
+                if (Directory.Exists(path))
+                {
+                    foreach (var spritePath in Directory.EnumerateFiles(path, "*.png", new EnumerationOptions() { MatchCasing = MatchCasing.CaseInsensitive }))
+                    {
+                        toReturn.Add(spritePath);
+                    }
+                    profilePath = profile;
+                    return toReturn;
+                }
+            }
+        }
+
+        profilePath = null;
+        return toReturn;
+    }
 }

src/TrackerCouncil.Smz3.Tracking/TrackerSpeechImages.cs

@@ -0,0 +1,8 @@
+namespace TrackerCouncil.Smz3.Tracking;
+
+public class TrackerSpeechImages
+{
+    public required string ReactionName { get; set; }
+    public required string IdleImage { get; set; }
+    public required string TalkingImage { get; set; }
+}

src/TrackerCouncil.Smz3.UI/Services/TrackerSpeechWindowService.cs

@@ -0,0 +1,146 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Avalonia;
+using Avalonia.Media;
+using Avalonia.Threading;
+using AvaloniaControls.ControlServices;
+using TrackerCouncil.Smz3.Data.Options;
+using TrackerCouncil.Smz3.Tracking;
+using TrackerCouncil.Smz3.Tracking.Services;
+using TrackerCouncil.Smz3.UI.ViewModels;
+
+namespace TrackerCouncil.Smz3.UI.Services;
+
+public class TrackerSpeechWindowService(ICommunicator communicator, IUIService uiService, OptionsFactory optionsFactory) : ControlService
+{
+    TrackerSpeechWindowViewModel _model = new();
+
+    private DispatcherTimer _dispatcherTimer = new()
+    {
+        Interval = TimeSpan.FromSeconds(1.0 / 60),
+    };
+
+    private TrackerSpeechImages? _currentSpeechImages;
+    private Dictionary<string, TrackerSpeechImages> _availableSpeechImages = [];
+    private int _tickCount;
+    private readonly int _maxTicks = 12;
+    private readonly double _bounceHeight = 6;
+    private int _prevViseme;
+    private bool _enableBounce;
+
+    public TrackerSpeechWindowViewModel GetViewModel()
+    {
+        _availableSpeechImages = uiService.GetTrackerSpeechSprites(out _);
+        SetSpeechImages("default");
+
+        var options = optionsFactory.Create();
+        var bytes = options.GeneralOptions.TrackerSpeechBGColor;
+        _enableBounce = options.GeneralOptions.TrackerSpeechEnableBounce;
+        _model.Background = new SolidColorBrush(Color.FromArgb(bytes[0], bytes[1], bytes[2], bytes[3]));
+
+        if (_currentSpeechImages == null)
+        {
+            return new TrackerSpeechWindowViewModel();
+        }
+
+        _model.TrackerImage = _currentSpeechImages.IdleImage;
+
+        if (_enableBounce)
+        {
+            _model.AnimationMargin = new Thickness(0, 0, 0, -1 * _bounceHeight);
+
+            _dispatcherTimer.Tick += (sender, args) =>
+            {
+                _tickCount++;
+                var fraction = Math.Clamp(1.0 * _tickCount / _maxTicks, 0, 1);
+
+                if (fraction < 0.5)
+                {
+                    _model.AnimationMargin = new Thickness(0, 0, 0, -1 * _bounceHeight + fraction * 2 * _bounceHeight);
+                }
+                else
+                {
+                    _model.AnimationMargin = new Thickness(0, 0, 0, (fraction - 0.5) * 2 * -1 * _bounceHeight);
+                }
+
+                if (fraction >= 1)
+                {
+                    _dispatcherTimer.Stop();
+                }
+            };
+        }
+
+        SaveOpenStatus(true);
+
+        communicator.SpeakCompleted += Communicator_SpeakCompleted;
+        communicator.VisemeReached += Communicator_VisemeReached;
+        return _model;
+    }
+
+    public void StopTimer()
+    {
+        _dispatcherTimer.Stop();
+    }
+
+    public void SaveOpenStatus(bool isOpen)
+    {
+        var options = optionsFactory.Create();
+        if (options.GeneralOptions.DisplayTrackerSpeechWindow == isOpen)
+        {
+            return;
+        }
+        options.GeneralOptions.DisplayTrackerSpeechWindow = isOpen;
+        options.Save();
+    }
+
+    private void Communicator_VisemeReached(object? sender, System.Speech.Synthesis.VisemeReachedEventArgs e)
+    {
+        if (!OperatingSystem.IsWindows()) return;
+
+        if (e.Viseme == 0)
+        {
+            _model.TrackerImage = _currentSpeechImages?.IdleImage;
+        }
+        else
+        {
+            if (_enableBounce && _prevViseme == 0 && !_dispatcherTimer.IsEnabled)
+            {
+                _tickCount = 0;
+                _dispatcherTimer.Start();
+            }
+            _model.TrackerImage = _currentSpeechImages?.TalkingImage;
+        }
+
+        _prevViseme = e.Viseme;
+    }
+
+    private void Communicator_SpeakCompleted(object? sender, SpeakCompletedEventArgs e)
+    {
+        _model.TrackerImage = _currentSpeechImages?.IdleImage;
+        _prevViseme = 0;
+    }
+
+    private void SetSpeechImages(string reaction)
+    {
+        if (_availableSpeechImages.TryGetValue(reaction.ToLower(), out var requestedSpeechImage))
+        {
+            _currentSpeechImages = requestedSpeechImage;
+        }
+        else if (_availableSpeechImages.TryGetValue("default", out var defaultSpeechImage))
+        {
+            _currentSpeechImages = defaultSpeechImage;
+        }
+        else
+        {
+            _currentSpeechImages = _availableSpeechImages.Values.FirstOrDefault();
+        }
+    }
+
+    public string GetBackgroundHex()
+    {
+        var color = _model.Background.Color;
+        return "#" + BitConverter.ToString([color.R, color.G, color.B]).Replace("-", string.Empty);
+    }
+}
+

src/TrackerCouncil.Smz3.UI/Services/TrackerWindowService.cs

@@ -60,6 +60,7 @@ public TrackerWindowViewModel GetViewModel(TrackerWindow parent)
         var bytes = Options.GeneralOptions.TrackerBGColor;
         _model.Background = new SolidColorBrush(Color.FromArgb(bytes[0], bytes[1], bytes[2], bytes[3]));
         _model.OpenTrackWindow = Options.GeneralOptions.DisplayMsuTrackWindow;
+        _model.OpenSpeechWindow = Options.GeneralOptions.DisplayTrackerSpeechWindow;
         _model.AddShadows = Options.GeneralOptions.TrackerShadows;
         _model.DisplayTimer = Options.GeneralOptions.TrackerTimerEnabled;
 
@@ -520,6 +521,13 @@ public void OpenTrackerHelpWindow()
         _trackerHelpWindow.Closed += (_, _) => _trackerHelpWindow = null;
     }
 
+    public TrackerSpeechWindow OpenTrackerSpeechWindow()
+    {
+        var window = serviceProvider.GetRequiredService<TrackerSpeechWindow>();
+        window.Show(_window);
+        return window;
+    }
+
     private TrackerWindowPanelViewModel? GetPanelViewModel(UIGridLocation? gridLocation)
     {
         return gridLocation?.Type switch