Implement voice commands

sample-by-jsakamoto · Dec 14, 2023 · 53a803e · 53a803e
1 parent e7e2b07
commit 53a803e
Show file tree

Hide file tree

Showing 6 changed files with 94 additions and 7 deletions.
diff --git a/App.razor b/App.razor
@@ -1,8 +1,9 @@
-@using Toolbelt.Blazor.HotKeys2
-@inject HotKeys HotKeys
-@inject Toolbelt.Blazor.Gamepad.GamepadList GamepadList
 @implements IDisposable
 
+@inject HotKeys HotKeys
+@inject GamepadList GamepadList
+@inject SpeechRecognition SpeechRecognition
+
 <div
     style="width: @(Context.GameAreaSize.Width)px; height: @(Context.GameAreaSize.Height)px; background-color: black;position: relative;">
 
@@ -19,18 +20,36 @@
     <img src="assets/snowman.svg"
         style="left:@(Context.SnowManRect.X)px; top:@(Context.SnowManRect.Y)px; width:@(Context.SnowManRect.Width)px; height:@(Context.SnowManRect.Height); position:absolute;" />
 </div>
+<div
+    style="width: @(Context.GameAreaSize.Width)px; height: 72px; background-color: black; display:flex; align-items:center; justify-content: space-evenly;">
+    <button @onclick="StartStopListening"
+        style="opacity:@(_voiceCommandState == VoiceCommand.None ? "0.5": "1"); width:52px; height:52px; background: transparent; outline: none; border: solid 3px #fff; border-radius: 26px;">
+        <img src="assets/microphone.svg" style="width:32px; height:32px;" />
+    </button>
+</div>
 
 @code {
     private GameContext Context = new();
 
     private HotKeysContext HotKeysContext;
 
+    private enum VoiceCommand { None, Stop, Left, Right }
+
+    private VoiceCommand _voiceCommandState = VoiceCommand.None;
+
     protected override void OnInitialized()
     {
         this.Context.GameLoopTimer.Elapsed += GameLoopTimer_Elapsed;
-        this.HotKeysContext = this.HotKeys.CreateContext();
-        this.HotKeysContext.Add(Code.ArrowLeft, this.Context.MoveSnowManToLeft);
-        this.HotKeysContext.Add(Code.ArrowRight, this.Context.MoveSnowManToRight);
+
+        this.HotKeysContext = this.HotKeys.CreateContext()
+        .Add(Code.ArrowLeft, this.Context.MoveSnowManToLeft)
+        .Add(Code.ArrowRight, this.Context.MoveSnowManToRight);
+
+        this.SpeechRecognition.Lang = "en-US";
+        this.SpeechRecognition.InterimResults = true;
+        this.SpeechRecognition.Continuous = true;
+        this.SpeechRecognition.Result += OnSpeechRecognized;
+        this.SpeechRecognition.End += OnEndSpeechRecognition;
     }
 
     private async void GameLoopTimer_Elapsed(object sender, EventArgs e)
@@ -43,11 +62,61 @@
             if (gamepad.Axes[0] > 0.8) this.Context.MoveSnowManToRight();
         }
 
+        if (_voiceCommandState == VoiceCommand.Left) this.Context.MoveSnowManToLeft();
+        if (_voiceCommandState == VoiceCommand.Right) this.Context.MoveSnowManToRight();
+
+        this.StateHasChanged();
+    }
+
+    private async Task StartStopListening()
+    {
+        if (_voiceCommandState == VoiceCommand.None)
+        {
+            _voiceCommandState = VoiceCommand.Stop;
+            await this.SpeechRecognition.StartAsync();
+            this.Context.Interval = GameContext.Speed.Slow;
+        }
+        else
+        {
+            _voiceCommandState = VoiceCommand.None;
+            await this.SpeechRecognition.StopAsync();
+            this.Context.Interval = GameContext.Speed.Normal;
+        }
+    }
+
+    private void OnSpeechRecognized(object sender, SpeechRecognitionEventArgs args)
+    {
+        var result = args.Results[args.ResultIndex];
+        var item = result.Items.LastOrDefault();
+        if (item == null) return;
+
+        var terms = item.Transcript.Split(new[] { ' ', ',', '.' }, StringSplitOptions.RemoveEmptyEntries);
+        foreach (var term in terms)
+        {
+            _voiceCommandState = term.ToLower() switch
+            {
+                "stop" => VoiceCommand.Stop,
+                "left" => VoiceCommand.Left,
+                "lift" => VoiceCommand.Left,
+                "right" => VoiceCommand.Right,
+                "just" => VoiceCommand.Right,
+                _ => _voiceCommandState
+            };
+            Console.WriteLine($"{term} => {_voiceCommandState}");
+        }
+    }
+
+    private void OnEndSpeechRecognition(object sender, EventArgs args)
+    {
+        _voiceCommandState = VoiceCommand.None;
+        this.Context.Interval = GameContext.Speed.Normal;
         this.StateHasChanged();
     }
 
     public void Dispose()
     {
+        this.SpeechRecognition.Result -= OnSpeechRecognized;
+        this.SpeechRecognition.End -= OnEndSpeechRecognition;
         this.Context.GameLoopTimer.Elapsed -= GameLoopTimer_Elapsed;
         this.HotKeysContext.Dispose();
     }

diff --git a/GameContext.cs b/GameContext.cs
@@ -2,18 +2,30 @@
 
 public class GameContext
 {
+    public static class Speed
+    {
+        public const double Slow = 100;
+        public const double Normal = 30;
+    }
+
     public Size GameAreaSize = new(width: 320, height: 480);
 
     public Rectangle SnowFlakeRect = new(x: 0, y: 0, width: 48, height: 48);
 
     public Rectangle SnowManRect = new(x: 0, y: 0, width: 64, height: 64);
 
-    public System.Timers.Timer GameLoopTimer = new(interval: 30);
+    public System.Timers.Timer GameLoopTimer = new(interval: Speed.Normal);
 
     public int Score;
 
     private Random Random = new();
 
+    public double Interval
+    {
+        get => this.GameLoopTimer.Interval;
+        set => this.GameLoopTimer.Interval = value;
+    }
+
     public GameContext()
     {
         this.ResetSnowFlakePos();

diff --git a/Program.cs b/Program.cs
@@ -8,5 +8,6 @@
 builder.Services.AddScoped(sp => new HttpClient { BaseAddress = new Uri(builder.HostEnvironment.BaseAddress) });
 builder.Services.AddHotKeys2();
 builder.Services.AddGamepadList();
+builder.Services.AddSpeechRecognition();
 
 await builder.Build().RunAsync();
diff --git a/SnowCatch.csproj b/SnowCatch.csproj
@@ -15,6 +15,7 @@
     <PackageReference Include="System.Drawing.Primitives" Version="4.3.0" />
     <PackageReference Include="Toolbelt.Blazor.Gamepad" Version="9.0.0" />
     <PackageReference Include="Toolbelt.Blazor.HotKeys2" Version="3.2.1" />
+    <PackageReference Include="Toolbelt.Blazor.SpeechRecognition" Version="1.0.0" />
   </ItemGroup>
 
 </Project>
diff --git a/_Imports.razor b/_Imports.razor
@@ -7,3 +7,6 @@
 @using Microsoft.AspNetCore.Components.WebAssembly.Http
 @using Microsoft.JSInterop
 @using SnowCatch
+@using Toolbelt.Blazor.HotKeys2
+@using Toolbelt.Blazor.SpeechRecognition
+@using Toolbelt.Blazor.Gamepad
diff --git a/wwwroot/assets/microphone.svg b/wwwroot/assets/microphone.svg