Detect when USB is (dis)connected, update displayed devices

src/Baballonia.Desktop/Calibration/ICalibrationRoutine.cs

@@ -304,14 +304,9 @@ public override Frame AddFrame(Mat[] images)
     }
 }
 
-public class CommandDispatchStep : ICalibrationStep
+public class CommandDispatchStep(string name) : ICalibrationStep
 {
-    public string Name { get; }
-
-    public CommandDispatchStep(string name)
-    {
-        Name = name;
-    }
+    public string Name { get; } = name;
 
     public Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
     {
@@ -320,27 +315,21 @@ public Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken
     }
 }
 
-public class TrainerCalibrationStep : ICalibrationStep
+public class TrainerCalibrationStep(ITrainerService overlayTrainer) : ICalibrationStep
 {
-    public string Name { get; }
-    private readonly ITrainerService _trainer;
-
-    public TrainerCalibrationStep(ITrainerService overlayTrainer)
-    {
-        _trainer = overlayTrainer;
-        Name = "trainer";
-    }
+    public string Name => "trainer";
+    private readonly ITrainerService _trainer = overlayTrainer;
 
     public async Task ExecuteAsync(OverlayMessageDispatcher dispatcher, CancellationToken ct)
     {
         dispatcher.Dispatch(new RunVariableLenghtRoutinePacket(Name, TimeSpan.FromSeconds(120)));
-        var onProgresHandler = (TrainerProgressReportPacket packet) => { dispatcher.Dispatch(packet); };
-        _trainer.OnProgress += onProgresHandler;
+        var onProgressHandler = (TrainerProgressReportPacket packet) => { dispatcher.Dispatch(packet); };
+        _trainer.OnProgress += onProgressHandler;
         _trainer.RunTraining(Path.Combine(Utils.ModelDataDirectory, "user_cal.bin"),
             Path.Combine(Utils.ModelDataDirectory, "tuned_temporal_eye_tracking_latest.onnx"));
         await _trainer.WaitAsync();
 
-        _trainer.OnProgress -= onProgresHandler;
+        _trainer.OnProgress -= onProgressHandler;
     }
 }
 
@@ -365,7 +354,7 @@ public BaseEyeCaptureStep Create(string name, uint flags, TimeSpan time,
 
 public class MergeBinsStep : ICalibrationStep
 {
-    public string Name { get; } = "bin_merger";
+    public string Name => "bin_merger";
     private string[] _binNames;
 
     public MergeBinsStep(params string[] binNames)
@@ -414,36 +403,12 @@ public IEnumerable<ICalibrationStep> BasicAllCalibration()
                 CaptureFlags.FLAG_VERSION_BIT1,
                 TimeSpan.FromSeconds(20), lid: 0
             ),
-            // steps.Add(new BaseTutorialStep("dilationtutorial"));
-            // steps.Add(_eyeCaptureStepFactory.Create("dilation",
-            //     CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_DILATION_BLACK));
-
-            new BaseTutorialStep("widentutorial", TimeSpan.FromSeconds(10)),
-            _eyeCaptureStepFactory.Create("widen",
-                CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20), widen: 1, lid: 1),
-
-            new BaseTutorialStep("squinttutorial", TimeSpan.FromSeconds(10)),
-            _eyeCaptureStepFactory.Create("squint",
-                CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20), squint: 1, lid: 1),
-
-            new BaseTutorialStep("browtutorial", TimeSpan.FromSeconds(10)),
-            _eyeCaptureStepFactory.Create("brow",
-                CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20), browAngry: 1, lid: 1),
-            // steps.Add(new BaseTutorialStep("covergencetutorial"));
-            // steps.Add(_eyeCaptureStepFactory.Create("covergence",
-            //     CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_WHATEVER_NOT_IMPLEMENTED));
-
-            new MergeBinsStep("gaze.bin", "blink.bin", "widen.bin", "squint.bin", "brow.bin"),
+
+            new MergeBinsStep("gaze.bin", "blink.bin"),
             new TrainerCalibrationStep(_trainer),
             new CommandDispatchStep("close")
 
         ];
-        // steps.Add(new BaseTutorialStep("dilationtutorial"));
-        // steps.Add(_eyeCaptureStepFactory.Create("dilation",
-        //     CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_DILATION_BLACK));
-        // steps.Add(new BaseTutorialStep("covergencetutorial"));
-        // steps.Add(_eyeCaptureStepFactory.Create("covergence",
-        //     CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_WHATEVER_NOT_IMPLEMENTED));
 
         return steps;
     }
@@ -462,19 +427,7 @@ public IEnumerable<ICalibrationStep> BasicAllCalibrationQuick()
                 TimeSpan.FromSeconds(20)
             ),
 
-            new BaseTutorialStep("widentutorial", TimeSpan.FromSeconds(4)),
-            _eyeCaptureStepFactory.Create("widen",
-                CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20)),
-
-            new BaseTutorialStep("squinttutorial", TimeSpan.FromSeconds(4)),
-            _eyeCaptureStepFactory.Create("squint",
-                CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20)),
-
-            new BaseTutorialStep("browtutorial", TimeSpan.FromSeconds(4)),
-            _eyeCaptureStepFactory.Create("brow",
-                CaptureFlags.FLAG_GOOD_DATA | CaptureFlags.FLAG_VERSION_BIT1, TimeSpan.FromSeconds(20)),
-
-            new MergeBinsStep("gaze.bin", "blink.bin", "widen.bin", "squint.bin", "brow.bin"),
+            new MergeBinsStep("gaze.bin", "blink.bin"),
             new TrainerCalibrationStep(_trainer),
             new CommandDispatchStep("close")
 

src/Baballonia.Desktop/main.nsi

@@ -11,7 +11,7 @@
   !define NAME "Baballonia"
   !define APPFILE "Baballonia.Desktop.exe"
   !define PUBLISHER "dfgHiatus - Paradigm Reality Enhancement Laboratories"
-  !define VERSION "1.1.0.8"
+  !define VERSION "1.1.0.9"
   !define SLUG "${NAME} v${VERSION}"
 
 ;--------------------------------

src/Baballonia/App.axaml.cs

@@ -147,6 +147,7 @@ public override void OnFrameworkInitializationCompleted()
             {
                 services.AddSingleton<ICommandSenderFactory, CommandSenderFactory>();
                 services.AddSingleton<ICommandSender, SerialCommandSender>();
+                services.AddSingleton<IUsbService, UsbService>();
                 services.AddSingleton<OpenVRService>();
                 services.AddTransient<VrcViewModel>();
                 services.AddTransient<VrcView>();

src/Baballonia/Baballonia.csproj

@@ -7,8 +7,8 @@
         <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
 	<NoWarn>CS8618;CS8509;CS0067;CS8613;CS8622;MVVMTK0034</NoWarn>
 	<Company>Paradigm Reality Enhancement Labs</Company>
-	<AssemblyVersion>1.1.0.8</AssemblyVersion>
-	<FileVersion>1.1.0.8</FileVersion>
+	<AssemblyVersion>1.1.0.9</AssemblyVersion>
+	<FileVersion>1.1.0.9</FileVersion>
     </PropertyGroup>
 
     <ItemGroup>
@@ -65,6 +65,7 @@
         <PackageReference Include="SoundFlow" Version="1.2.1" />
         <PackageReference Include="System.IO.Ports" Version="9.0.8" />
         <PackageReference Include="System.Management" Version="9.0.8" />
+        <PackageReference Include="Usb.Events" Version="11.1.1.1" />
         <PackageReference Include="VRChat.OSCQuery" Version="0.0.7" />
     </ItemGroup>
 

src/Baballonia/Contracts/IUsbService.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace Baballonia.Contracts;
+
+public interface IUsbService
+{
+    public event Action<string> OnUsbConnected;
+    public event Action<string> OnUsbDisconnected;
+}

src/Baballonia/Controls/DropOverlay.axaml

@@ -36,14 +36,17 @@
             TextWrapping="Wrap"
             Text="{x:Static assets:Resources.DropOverlay_Important_Header}" />
 
-          <TextBlock 
+          <TextBlock
             TextWrapping="Wrap"
-            Text="{x:Static assets:Resources.DropOverlay_Message}" />
+            Text="Project Babble now has a data sharing program to help us improve our eye and face tracking models. You can opt in or out anytime by visiting the settings page." />
 
           <Grid>
-            <CheckBox x:Name="WarningCheckbox" IsCheckedChanged="Changed" Content="{x:Static assets:Resources.DropOverlay_DontShowAgain}" />
+            <CheckBox
+              x:Name="EyeDataCheckbox"
+              IsCheckedChanged="Changed"
+              Content="Opt in" />
             <Button
-              Click="SecondWarningUnderstood"
+              Click="EyeDataOptInRead"
               Content="{x:Static assets:Resources.DropOverlay_Dismiss_Button}"
               CornerRadius="4"
               HorizontalAlignment="Right"

src/Baballonia/Controls/DropOverlay.axaml.cs

@@ -29,11 +29,12 @@ public bool IsOverlayVisible
 
     private void Changed(object? sender, RoutedEventArgs e)
     {
-        _localSettingsService.SaveSetting("SecondsWarningRead", WarningCheckbox.IsChecked);
+        _localSettingsService.SaveSetting("AppSettings_ShareEyeData", EyeDataCheckbox.IsChecked);
     }
 
-    private void SecondWarningUnderstood(object? sender, RoutedEventArgs e)
+    private void EyeDataOptInRead(object? sender, RoutedEventArgs e)
     {
+        _localSettingsService.SaveSetting("EyeDataOptInRead", true);
         IsOverlayVisible = false;
     }
 }

src/Baballonia/LocalSettings.json

@@ -371,5 +371,7 @@
   "AppSettings_SteamVRAutoStart": false,
   "AppSettings_UseGPU": true,
   "AppSettings_CheckForUpdates": true,
+  "AppSettings_ShareEyeData": false,
+  "EyeDataOptInRead": false,
   "AppSettings_LogLevel": "Information"
 }

src/Baballonia/Services/DefaultInferenceRunner.cs

@@ -200,7 +200,7 @@ public float[] Run()
 
         using var results = _session.Run(inputs);
 
-        float[] output = OrganizeOutputShapes(results);
+        var output = results[0].AsEnumerable<float>().ToArray();
         OutputSize = output.Length;
         return output;
     }
@@ -220,50 +220,6 @@ public float[] Run()
         { "rightBrow", 9 }
     };
 
-    // Guarantee an expression's index by using named parameter values
-    private float[] OrganizeOutputShapes(IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results)
-    {
-        // Is this model the face model?
-        var candidate = results[0].AsEnumerable<float>().ToArray();
-        if (candidate.Length == Utils.FaceRawExpressions)
-        {
-            return candidate;
-        }
-
-        // Else, is this an older eye model?
-        if (_isOldEyeModel)
-        {
-            return [];
-        }
-
-        // Else, order eye information using cached metadata
-        // Start by flattening new eye ONNX output
-        List<Tuple<string, float>> arKitExpressions = [];
-        int counter = 0;
-        foreach (var result in results)
-        {
-            var exps = result.AsEnumerable<float>().ToArray();
-            foreach (var exp in exps)
-            {
-                arKitExpressions.Add(new Tuple<string, float>(_outputExpressionNames[counter], exp));
-                counter++;
-            }
-        }
-
-        float[] output = new float[arKitExpressions.Count];
-        foreach (var expression in arKitExpressions)
-        {
-            var name = expression.Item1;
-            var value = expression.Item2;
-
-            if (OutputIndexMap.TryGetValue(name, out var index))
-            {
-                output[index] = value;
-            }
-        }
-
-        return output;
-    }
 
     public DenseTensor<float> GetInputTensor()
     {

src/Baballonia/Services/EyePipelineManager.cs

@@ -64,33 +64,13 @@ private DefaultInferenceRunner CreateInference()
         var eyeModelName = _localSettings.ReadSetting<string>("EyeHome_EyeModel", defaultEyeModelName);
         var eyeModelPath = Path.Combine(AppContext.BaseDirectory, eyeModelName);
 
-        if (File.Exists(eyeModelPath))
-        {
-            var candidate = _inferenceFactory.Create(eyeModelPath);
-
-            // We need to set up the model and run it once to get the model's output size
-            candidate.Setup(eyeModelName);
-            candidate.Run();
-
-            // Get the output size of the model. If is different from our expected eye expression count,
-            // It is likely an old(er) model and will require a re-calibration
-            if (Utils.EyeRawExpressions == candidate.OutputSize)
-                return candidate;
-
-            _logger.LogInformation("Eye model output size {CandidateOutputSize} was different than expected {EyeRawExpressions}.",
-                candidate.OutputSize, Utils.EyeRawExpressions);
-            _logger.LogInformation("You likely have an old(er) model that only predicts gaze and blinking.");
-            _logger.LogInformation("Please re-calibrate to train a newer model on eyebrow angry, squint and wide!");
-        }
-        else
-        {
-            _logger.LogError("{} Does not exist, Loading default...", eyeModelPath);
-        }
+        if (File.Exists(eyeModelPath)) return _inferenceFactory.Create(eyeModelPath);
+        _logger.LogError("{} Does not exists, Loading default...", eyeModelPath);
 
         eyeModelName = defaultEyeModelName;
         eyeModelPath = Path.Combine(AppContext.BaseDirectory, eyeModelName);
 
-        return _inferenceFactory.Create(eyeModelPath);;
+        return _inferenceFactory.Create(eyeModelPath);
     }
 
 

src/Baballonia/Services/Inference/EyeProcessingPipeline.cs

@@ -48,8 +48,10 @@ public EyeProcessingPipeline(IEyePipelineEventBus eyePipelineEventBus)
         if(inferenceResult == null)
             return null;
 
-        if(Filter != null)
+        if (Filter != null)
+        {
             inferenceResult = Filter.Filter(inferenceResult);
+        }
 
         ProcessExpressions(ref inferenceResult);
 
@@ -73,9 +75,9 @@ private bool ProcessExpressions(ref float[] arKitExpressions)
         var leftYaw = arKitExpressions[1] * mulV - mulV / 2;
         var leftLid = 1 - arKitExpressions[2];
 
-        var rightPitch = arKitExpressions[5] * mulY - mulY / 2;
-        var rightYaw = arKitExpressions[6] * mulV - mulV / 2;
-        var rightLid = 1 - arKitExpressions[7];
+        var rightPitch = arKitExpressions[3] * mulY - mulY / 2;
+        var rightYaw = arKitExpressions[4] * mulV - mulV / 2;
+        var rightLid = 1 - arKitExpressions[5];
 
         var eyeY = (leftPitch * leftLid + rightPitch * rightLid) / (leftLid + rightLid);
 
@@ -99,13 +101,9 @@ private bool ProcessExpressions(ref float[] arKitExpressions)
         convertedExpressions[0] = rightEyeYawCorrected; // left pitch
         convertedExpressions[1] = eyeY;                   // left yaw
         convertedExpressions[2] = rightLid;               // left lid
-        convertedExpressions[3] = arKitExpressions[3];
-        convertedExpressions[4] = arKitExpressions[4];
-        convertedExpressions[5] = leftEyeYawCorrected;  // right pitch
-        convertedExpressions[6] = eyeY;                   // right yaw
-        convertedExpressions[7] = leftLid;                // right lid
-        convertedExpressions[8] = arKitExpressions[8];
-        convertedExpressions[9] = arKitExpressions[9];
+        convertedExpressions[3] = leftEyeYawCorrected;  // right pitch
+        convertedExpressions[4] = eyeY;                   // right yaw
+        convertedExpressions[5] = leftLid;                // right lid
 
         arKitExpressions = convertedExpressions;
 

src/Baballonia/Services/Inference/ImageCollector.cs

@@ -30,22 +30,17 @@ public class ImageCollector : IImageTransformer
         // feed the most recent matrix here at the start
         var last4 = ImageQueue.Skip(ImageQueue.Count - 4).Take(4).Reverse().ToArray();
 
-        var leftChannels = new List<Mat>();
-        var rightChannels = new List<Mat>();
+        var channels = new List<Mat>();
         foreach (var m in last4)
         {
             Mat[] splitChannels = Cv2.Split(m);
-            leftChannels.Add(splitChannels[0]);
-            rightChannels.Add(splitChannels[1]);
+            channels.AddRange(splitChannels);
         }
 
         Mat octoMatrix = new Mat();
-        Cv2.Merge(leftChannels.Concat(rightChannels).ToArray(), octoMatrix);
+        Cv2.Merge(channels.ToArray(), octoMatrix);
 
-        foreach (var channel in leftChannels)
-            channel.Dispose();
-
-        foreach (var channel in rightChannels)
+        foreach (var channel in channels)
             channel.Dispose();
 
         return octoMatrix;