Release 0.16.92.1

Author: danbohus

Directory.Build.props

@@ -6,7 +6,7 @@
     <Company>Microsoft Corporation</Company>
     <Owners>microsoft,psi</Owners>
     <Authors>Microsoft</Authors>
-    <AssemblyVersion>0.15.49.1</AssemblyVersion>
+    <AssemblyVersion>0.16.92.1</AssemblyVersion>
     <FileVersion>$(AssemblyVersion)</FileVersion>
     <Version>$(AssemblyVersion)-beta</Version>
     <SignAssembly>false</SignAssembly>

Psi.sln

@@ -198,6 +198,22 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Filters", "Filters", "{E0E7
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Filters", "Sources\Filters\Microsoft.Psi.Filters\Microsoft.Psi.Filters.csproj", "{E0621435-AF35-4CFA-BE9E-3781AF6E161F}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Spatial", "Spatial", "{F36DEF23-4FFF-4237-9104-03CF19036C70}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Spatial.Euclidean", "Sources\Spatial\Microsoft.Psi.Spatial.Euclidean\Microsoft.Psi.Spatial.Euclidean.csproj", "{C3114338-AD22-4EBC-85C3-EE06045CDD78}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Spatial.Euclidean.Visualization.Windows", "Sources\Spatial\Microsoft.Psi.Spatial.Euclidean.Visualization.Windows\Microsoft.Psi.Spatial.Euclidean.Visualization.Windows.csproj", "{A1429F96-C7F8-49D8-ADB8-73A1A4DAA70F}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "MixedReality", "MixedReality", "{32023088-0392-4B48-B2CF-3754B55C6DE9}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "HoloLens2ResearchMode", "Sources\MixedReality\HoloLens2ResearchMode\HoloLens2ResearchMode.vcxproj", "{F50194C0-9561-40C7-B9CB-B977E3B3D76D}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.MixedReality", "Sources\MixedReality\Microsoft.Psi.MixedReality\Microsoft.Psi.MixedReality.csproj", "{3434D5B2-B06F-4356-9E9B-90171CEF482B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Psi.MixedReality.UniversalWindows", "Sources\MixedReality\Microsoft.Psi.MixedReality.UniversalWindows\Microsoft.Psi.MixedReality.UniversalWindows.csproj", "{ECD9E150-8104-4DA3-B807-A6A4392A67C6}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.MixedReality.Visualization.Windows", "Sources\MixedReality\Microsoft.Psi.MixedReality.Visualization.Windows\Microsoft.Psi.MixedReality.Visualization.Windows.csproj", "{BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -436,6 +452,30 @@ Global
 		{E0621435-AF35-4CFA-BE9E-3781AF6E161F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E0621435-AF35-4CFA-BE9E-3781AF6E161F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{E0621435-AF35-4CFA-BE9E-3781AF6E161F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C3114338-AD22-4EBC-85C3-EE06045CDD78}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C3114338-AD22-4EBC-85C3-EE06045CDD78}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C3114338-AD22-4EBC-85C3-EE06045CDD78}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C3114338-AD22-4EBC-85C3-EE06045CDD78}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A1429F96-C7F8-49D8-ADB8-73A1A4DAA70F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A1429F96-C7F8-49D8-ADB8-73A1A4DAA70F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A1429F96-C7F8-49D8-ADB8-73A1A4DAA70F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A1429F96-C7F8-49D8-ADB8-73A1A4DAA70F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Debug|Any CPU.ActiveCfg = Debug|ARM
+		{F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Debug|Any CPU.Build.0 = Debug|ARM
+		{F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Release|Any CPU.ActiveCfg = Release|ARM
+		{F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Release|Any CPU.Build.0 = Release|ARM
+		{3434D5B2-B06F-4356-9E9B-90171CEF482B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3434D5B2-B06F-4356-9E9B-90171CEF482B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3434D5B2-B06F-4356-9E9B-90171CEF482B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3434D5B2-B06F-4356-9E9B-90171CEF482B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Debug|Any CPU.ActiveCfg = Debug|ARM
+		{ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Debug|Any CPU.Build.0 = Debug|ARM
+		{ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Release|Any CPU.ActiveCfg = Release|ARM
+		{ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Release|Any CPU.Build.0 = Release|ARM
+		{BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -522,6 +562,14 @@ Global
 		{A0677BEA-ADB1-4950-89E6-89483D621A52} = {EE4035A8-CEFE-4E3A-9CD9-4AE7E88DA2C4}
 		{E0E7957E-731F-4FDD-83FE-634FFE24862F} = {A0856299-D28A-4513-B964-3FA5290FF160}
 		{E0621435-AF35-4CFA-BE9E-3781AF6E161F} = {E0E7957E-731F-4FDD-83FE-634FFE24862F}
+		{F36DEF23-4FFF-4237-9104-03CF19036C70} = {A0856299-D28A-4513-B964-3FA5290FF160}
+		{C3114338-AD22-4EBC-85C3-EE06045CDD78} = {F36DEF23-4FFF-4237-9104-03CF19036C70}
+		{A1429F96-C7F8-49D8-ADB8-73A1A4DAA70F} = {F36DEF23-4FFF-4237-9104-03CF19036C70}
+		{32023088-0392-4B48-B2CF-3754B55C6DE9} = {A0856299-D28A-4513-B964-3FA5290FF160}
+		{F50194C0-9561-40C7-B9CB-B977E3B3D76D} = {32023088-0392-4B48-B2CF-3754B55C6DE9}
+		{3434D5B2-B06F-4356-9E9B-90171CEF482B} = {32023088-0392-4B48-B2CF-3754B55C6DE9}
+		{ECD9E150-8104-4DA3-B807-A6A4392A67C6} = {32023088-0392-4B48-B2CF-3754B55C6DE9}
+		{BE95524A-F9C2-4D0D-8F7E-1C7019B5A114} = {32023088-0392-4B48-B2CF-3754B55C6DE9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {EAF15EE9-DCC5-411B-A9E5-7C2F3D132331}

Sources/Audio/Microsoft.Psi.Audio.Linux/AudioCapture.cs

@@ -158,7 +158,7 @@ public void Start(Action<DateTime> notifyCompletionTime)
                     Array.Copy(buf, this.buffer.Data, length);
 
                     // use the end of the last sample in the packet as the originating time
-                    DateTime originatingTime = this.pipeline.GetCurrentTime().AddSeconds(length / format.AvgBytesPerSec);
+                    DateTime originatingTime = this.pipeline.GetCurrentTime().AddSeconds((double)length / format.AvgBytesPerSec);
 
                     // post the data to the output stream
                     this.audioBuffers.Post(this.buffer, originatingTime);

Sources/Audio/Microsoft.Psi.Audio.Windows/MFResampler.cs

@@ -91,16 +91,32 @@ public void Initialize(int targetLatencyInMs, WaveFormat inFormat, WaveFormat ou
             this.inputBufferSize = (int)(this.bufferLengthInMs * inFormat.AvgBytesPerSec / 1000);
             this.outputBufferSize = (int)(this.bufferLengthInMs * outFormat.AvgBytesPerSec / 1000);
 
+            Exception taskException = null;
+
             // Activate native Media Foundation COM objects on a thread-pool thread to ensure that they are in an MTA
             Task.Run(() =>
             {
-                DeviceUtil.CreateResamplerBuffer(this.inputBufferSize, out this.inputSample, out this.inputBuffer);
-                DeviceUtil.CreateResamplerBuffer(this.outputBufferSize, out this.outputSample, out this.outputBuffer);
+                try
+                {
+                    DeviceUtil.CreateResamplerBuffer(this.inputBufferSize, out this.inputSample, out this.inputBuffer);
+                    DeviceUtil.CreateResamplerBuffer(this.outputBufferSize, out this.outputSample, out this.outputBuffer);
 
-                // Create resampler object
-                this.resampler = DeviceUtil.CreateResampler(inFormat, outFormat);
+                    // Create resampler object
+                    this.resampler = DeviceUtil.CreateResampler(inFormat, outFormat);
+                }
+                catch (Exception e)
+                {
+                    taskException = e;
+                }
             }).Wait();
 
+            // do error checking on the main thread
+            if (taskException != null)
+            {
+                // rethrow exception
+                throw taskException;
+            }
+
             // Set the callback function
             this.dataAvailableCallback = callback;
         }

Sources/Audio/Microsoft.Psi.Audio.Windows/Operators.cs

@@ -3,8 +3,6 @@
 
 namespace Microsoft.Psi.Audio
 {
-    using Microsoft.Psi.Audio;
-
     /// <summary>
     /// Stream operators and extension methods for Microsoft.Psi.Audio.Windows.
     /// </summary>

Sources/Audio/Microsoft.Psi.Audio.Windows/WasapiCapture.cs

@@ -143,33 +143,47 @@ public void Dispose()
         /// </param>
         public void Initialize(string deviceDescription)
         {
+            Exception taskException = null;
+
             // Activate native audio COM objects on a thread-pool thread to ensure that they are in an MTA
             Task.Run(() =>
             {
-                if (string.IsNullOrEmpty(deviceDescription))
+                try
                 {
-                    // use the default console device
-                    this.audioDevice = DeviceUtil.GetDefaultDevice(EDataFlow.Capture, ERole.Console);
+                    if (string.IsNullOrEmpty(deviceDescription))
+                    {
+                        // use the default console device
+                        this.audioDevice = DeviceUtil.GetDefaultDevice(EDataFlow.Capture, ERole.Console);
+                    }
+                    else
+                    {
+                        this.audioDevice = DeviceUtil.GetDeviceByName(EDataFlow.Capture, deviceDescription);
+                    }
+
+                    if (this.audioDevice != null)
+                    {
+                        // Try to get the volume control
+                        object obj = this.audioDevice.Activate(new Guid(Guids.IAudioEndpointVolumeIIDString), ClsCtx.ALL, IntPtr.Zero);
+                        this.volume = (IAudioEndpointVolume)obj;
+
+                        // Now create an IAudioEndpointVolumeCallback object that wraps the callback and register it with the endpoint.
+                        this.volumeCallback = new AudioEndpointVolumeCallback(this.AudioVolumeCallback);
+                        this.volume.RegisterControlChangeNotify(this.volumeCallback);
+                    }
                 }
-                else
+                catch (Exception e)
                 {
-                    this.audioDevice = DeviceUtil.GetDeviceByName(EDataFlow.Capture, deviceDescription);
-                }
-
-                if (this.audioDevice != null)
-                {
-                    // Try to get the volume control
-                    object obj = this.audioDevice.Activate(new Guid(Guids.IAudioEndpointVolumeIIDString), ClsCtx.ALL, IntPtr.Zero);
-                    this.volume = (IAudioEndpointVolume)obj;
-
-                    // Now create an IAudioEndpointVolumeCallback object that wraps the callback and register it with the endpoint.
-                    this.volumeCallback = new AudioEndpointVolumeCallback(this.AudioVolumeCallback);
-                    this.volume.RegisterControlChangeNotify(this.volumeCallback);
+                    taskException = e;
                 }
             }).Wait();
 
             // do error checking on the main thread
-            if (this.audioDevice == null)
+            if (taskException != null)
+            {
+                // rethrow exception
+                throw taskException;
+            }
+            else if (this.audioDevice == null)
             {
                 throw new IOException(string.IsNullOrEmpty(deviceDescription) ?
                     "No default audio capture device found." :

Sources/Audio/Microsoft.Psi.Audio.Windows/WasapiRender.cs

@@ -4,6 +4,7 @@
 namespace Microsoft.Psi.Audio
 {
     using System;
+    using System.IO;
     using System.Runtime.InteropServices;
     using System.Threading.Tasks;
     using Microsoft.Psi.Audio.ComInterop;
@@ -138,30 +139,52 @@ public void Dispose()
         /// </param>
         public void Initialize(string deviceDescription)
         {
+            Exception taskException = null;
+
             // Activate native audio COM objects on a thread-pool thread to ensure that they are in an MTA
             Task.Run(() =>
             {
-                if (string.IsNullOrEmpty(deviceDescription))
+                try
                 {
-                    // use the default console device
-                    this.audioDevice = DeviceUtil.GetDefaultDevice(EDataFlow.Render, ERole.Console);
+                    if (string.IsNullOrEmpty(deviceDescription))
+                    {
+                        // use the default console device
+                        this.audioDevice = DeviceUtil.GetDefaultDevice(EDataFlow.Render, ERole.Console);
+                    }
+                    else
+                    {
+                        this.audioDevice = DeviceUtil.GetDeviceByName(EDataFlow.Render, deviceDescription);
+                    }
+
+                    if (this.audioDevice != null)
+                    {
+                        // Try to get the volume control
+                        object obj = this.audioDevice.Activate(new Guid(Guids.IAudioEndpointVolumeIIDString), ClsCtx.ALL, IntPtr.Zero);
+                        this.volume = (IAudioEndpointVolume)obj;
+
+                        // Now create an IAudioEndpointVolumeCallback object that wraps the callback and register it with the endpoint.
+                        this.volumeCallback = new AudioEndpointVolumeCallback(this.AudioVolumeCallback);
+                        this.volume.RegisterControlChangeNotify(this.volumeCallback);
+                    }
                 }
-                else
+                catch (Exception e)
                 {
-                    this.audioDevice = DeviceUtil.GetDeviceByName(EDataFlow.Render, deviceDescription);
-                }
-
-                if (this.audioDevice != null)
-                {
-                    // Try to get the volume control
-                    object obj = this.audioDevice.Activate(new Guid(Guids.IAudioEndpointVolumeIIDString), ClsCtx.ALL, IntPtr.Zero);
-                    this.volume = (IAudioEndpointVolume)obj;
-
-                    // Now create an IAudioEndpointVolumeCallback object that wraps the callback and register it with the endpoint.
-                    this.volumeCallback = new AudioEndpointVolumeCallback(this.AudioVolumeCallback);
-                    this.volume.RegisterControlChangeNotify(this.volumeCallback);
+                    taskException = e;
                 }
             }).Wait();
+
+            // do error checking on the main thread
+            if (taskException != null)
+            {
+                // rethrow exception
+                throw taskException;
+            }
+            else if (this.audioDevice == null)
+            {
+                throw new IOException(string.IsNullOrEmpty(deviceDescription) ?
+                    "No default audio playback device found." :
+                    $"Audio playback device {deviceDescription} not found.");
+            }
         }
 
         /// <summary>

Sources/Audio/Microsoft.Psi.Audio/WaveFileHelper.cs

@@ -63,8 +63,18 @@ public static WaveFormat ReadWaveFileHeader(string filename)
         /// <returns>The number of byte of wave data that follow.</returns>
         public static long ReadWaveDataLength(BinaryReader br)
         {
-            if (Encoding.UTF8.GetString(BitConverter.GetBytes(br.ReadInt32())) != "data")
+            var name = Encoding.UTF8.GetString(BitConverter.GetBytes(br.ReadInt32()));
+            if (name != "data")
             {
+                if (name == "fact" || name == "LIST")
+                {
+                    // Some formats (e.g. IEEE float) contain fact and LIST chunks (which we skip).
+                    // see fhe "fact Chunk" section of the spec: http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html
+                    // "IEEE float data (introduced after the Rev. 3 documention) need a fact"
+                    br.ReadBytes((int)br.ReadUInt32()); // skip
+                    return ReadWaveDataLength(br);
+                }
+
                 throw new FormatException("Data header missing");
             }
 

Sources/Audio/Microsoft.Psi.Audio/WaveStreamSampleSource.cs

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Audio
+{
+    using System;
+    using System.IO;
+    using Microsoft.Psi.Components;
+
+    /// <summary>
+    /// Component that produces on-demand an audio sample specified by a <see cref="System.IO.Stream"/>.
+    /// </summary>
+    /// <remarks>
+    /// This is meant for relatively short sound effects cached in memory.
+    /// We consume the stream given at construction time; breaking it into
+    /// audio buffers which are "played" upon receiving a true input signal.
+    /// </remarks>
+    public class WaveStreamSampleSource : IConsumerProducer<bool, AudioBuffer>
+    {
+        private readonly Pipeline pipeline;
+        private readonly AudioBuffer[] audioData;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="WaveStreamSampleSource"/> class.
+        /// </summary>
+        /// <param name="pipeline">The pipeline to add the component to.</param>
+        /// <param name="stream">Audio stream in WAVE format (48KHz, 1-channel, IEEE Float).</param>
+        public WaveStreamSampleSource(Pipeline pipeline, Stream stream)
+        {
+            this.pipeline = pipeline;
+            this.In = pipeline.CreateReceiver<bool>(this, this.Play, nameof(this.In));
+            this.Out = pipeline.CreateEmitter<AudioBuffer>(this, nameof(this.Out));
+
+            var reader = new BinaryReader(stream);
+            var inputFormat = WaveFileHelper.ReadWaveFileHeader(reader);
+
+            // we don't do resampling or conversion (must be 1-channel, 48kHz, float32).
+            // convert offline if needed: ffmpeg -i foo.wav -f wav -acodec pcm_f32le -ar 48000 -ac 1 bar.wav
+            if (inputFormat.Channels != 1 ||
+                inputFormat.SamplesPerSec != 48000 ||
+                (inputFormat.FormatTag != WaveFormatTag.WAVE_FORMAT_IEEE_FLOAT &&
+                 inputFormat.FormatTag != WaveFormatTag.WAVE_FORMAT_EXTENSIBLE) ||
+                inputFormat.BitsPerSample != 32)
+            {
+                throw new ArgumentException("Expected 1-channel, 48kHz, float32 audio format.");
+            }
+
+            // break into 1 second audio buffers
+            var outputFormat = WaveFormat.CreateIeeeFloat(48000, 1);
+            var dataLength = WaveFileHelper.ReadWaveDataLength(reader);
+
+            // stepping over this line computing frames (e.g. F10) in the debugger will throw - still trying to understand why
+            var frames = (int)Math.Ceiling((double)dataLength / (double)outputFormat.AvgBytesPerSec);
+            this.audioData = new AudioBuffer[frames];
+            for (var i = 0; dataLength > 0; i++)
+            {
+                var count = (int)Math.Min(dataLength, outputFormat.AvgBytesPerSec);
+                var bytes = reader.ReadBytes(count);
+                this.audioData[i] = new AudioBuffer(bytes, outputFormat);
+                dataLength -= count;
+            }
+        }
+
+        /// <summary>
+        /// Gets the receiver of a signal indicating whether to play a sound.
+        /// </summary>
+        public Receiver<bool> In { get; private set; }
+
+        /// <summary>
+        /// Gets the stream of sound output.
+        /// </summary>
+        public Emitter<AudioBuffer> Out { get; private set; }
+
+        private void Play(bool play)
+        {
+            if (play)
+            {
+                var now = this.pipeline.GetCurrentTime();
+                if (now < this.Out.LastEnvelope.OriginatingTime)
+                {
+                    // overlapping with last time played (play after)
+                    now = this.Out.LastEnvelope.OriginatingTime.AddTicks(1);
+                }
+
+                for (var i = 0; i < this.audioData.Length; i++)
+                {
+                    this.Out.Post(this.audioData[i], now + TimeSpan.FromSeconds(i));
+                }
+            }
+        }
+    }
+}