From 805e3db166ed90f7be104f4d71a78309eacc5585 Mon Sep 17 00:00:00 2001 From: janezdu Date: Tue, 6 Jul 2021 11:22:16 -0400 Subject: [PATCH 1/9] New HSV colouring, implemented in cs --- .../AudioPreprocessing/MainWindow.xaml.cs | 2 + .../Model/PreprocessModel.cs | 175 +++++++++++++++++- .../ViewModel/PreprocessViewModel.cs | 7 - .../AudioPreprocessing/tmp/colorize_me.onnx | Bin 0 -> 1805 bytes 4 files changed, 167 insertions(+), 17 deletions(-) create mode 100644 Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/tmp/colorize_me.onnx diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs index 544d3109..ef553871 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs @@ -6,6 +6,7 @@ using System.Runtime.InteropServices; using System.Threading.Tasks; using Windows.Graphics.Imaging; +using Windows.Media; using Windows.Storage; using Windows.Storage.Pickers; @@ -40,6 +41,7 @@ private async void OnOpenClick(object sender, RoutedEventArgs e) string wavPath = await GetFilePath(); PreprocessModel melSpectrogram = new PreprocessModel(); var softwareBitmap = melSpectrogram.GenerateMelSpectrogram(wavPath, ColorMelSpectrogramCheckBox.IsChecked ?? false); + //PreprocessModel.Colorize(VideoFrame.CreateWithSoftwareBitmap(softwareBitmap), 0.5f, 0.3f); ViewModel.AudioPath = wavPath; ViewModel.MelSpectrogramImage = softwareBitmap; diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs index 3e353917..2bd40e85 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs @@ -115,7 +115,17 @@ static SoftwareBitmap GetMelspectrogramFromSignal( .Operators.Add(new Operator("Reshape") .SetInput("data", "mel_spectrogram") .SetConstant("shape", TensorInt64Bit.CreateFromArray(new List() { 4 }, melSpectrogramShape)) - .SetOutput("reshaped", "Output.MelSpectrogram")); + .SetOutput("reshaped", "Output.MelSpectrogram")) + ; + + ////colouring with LearningModelBuilder + //builder.Operators.Add(new Operator("Slice") + // .SetInput("data", "bw_mel_spectrogram") + // .SetConstant("starts", TensorInt64Bit.CreateFromArray(new List() { 3 }, new long[] { 0, 0, 0 })) + // .SetInput("ends", TensorInt64Bit.CreateFromArray(new List() { 3 }, new long[] { 0, , 0 })) + // .SetConstant("shape", TensorInt64Bit.CreateFromArray(new List() { 4 }, melSpectrogramShape)) + // .SetOutput("reshaped", "hue")) + //; var model = builder.CreateModel(); @@ -142,6 +152,111 @@ static SoftwareBitmap GetMelspectrogramFromSignal( return outputImage.SoftwareBitmap; } + public static void Colorize(Windows.Media.VideoFrame image, float saturation, float value) + { + long width = image.SoftwareBitmap.PixelWidth; + long height = image.SoftwareBitmap.PixelHeight; + long channels = image.SoftwareBitmap.BitmapPixelFormat == Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8 ? 4 : 1; + + long batch_size = 1; + + var c = saturation * value; + var m = value - saturation; + + + var builder = LearningModelBuilder.Create(13) + .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Input", TensorKind.Float, new long[] { batch_size, channels, height, width })) + .Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output", TensorKind.Float, new long[] { batch_size, channels, height, width })) + .Operators.Add(new LearningModelOperator("Slice") + .SetInput("data", "Input") + .SetConstant("starts", TensorInt64Bit.CreateFromIterable(new long[] { 4 }, new long[] { 0, 0, 0, 0 })) + .SetConstant("ends", TensorInt64Bit.CreateFromIterable(new long[] { 4 }, new long[] { long.MaxValue, 1, long.MaxValue, long.MaxValue })) + .SetConstant("axes", TensorInt64Bit.CreateFromIterable(new long[] { 4 }, new long[] { 0, 1, 2, 3 })) + .SetConstant("steps", TensorInt64Bit.CreateFromIterable(new long[] { 4 }, new long[] { 1, 1, 1, 1 })) + .SetOutput("output", "hue")) + .Operators.Add(new LearningModelOperator("Div") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 60 })) + .SetOutput("C", "div_output")) + .Operators.Add(new LearningModelOperator("Mod") + .SetInput("A", "div_output") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 2 })) + .SetOutput("C", "mod_output")) + .Operators.Add(new LearningModelOperator("Sub") + .SetInput("A", "mod_output") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 })) + .SetOutput("C", "sub1_output")) + .Operators.Add(new LearningModelOperator("Abs") + .SetInput("X", "sub1_output") + .SetOutput("Y", "abs_output")) + .Operators.Add(new LearningModelOperator("Sub") + .SetConstant("A", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 1 })) + .SetInput("B", "abs_output") + .SetOutput("C", "sub2_output")) + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "sub2_output") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { c })) + .SetOutput("C", "mul1_output")) + // generate 6 masks + .Operators.Add(new LearningModelOperator("Less") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 60 })) + .SetOutput("C", "mask1")) + + .Operators.Add(new LearningModelOperator("GreaterOrEqual") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 60 })) + .SetOutput("C", "greater2_output")) + .Operators.Add(new LearningModelOperator("Less") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 120 })) + .SetOutput("C", "less2_output")) + .Operators.Add(new LearningModelOperator("Less") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 120 })) + .SetOutput("C", "less2_output")) + + .Operators.Add(new LearningModelOperator("GreaterOrEqual") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 120 })) + .SetOutput("C", "greater3_output")) + .Operators.Add(new LearningModelOperator("Less") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 180 })) + .SetOutput("C", "less3_output")) + + .Operators.Add(new LearningModelOperator("GreaterOrEqual") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 180 })) + .SetOutput("C", "greater4_output")) + .Operators.Add(new LearningModelOperator("Less") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 240 })) + .SetOutput("C", "less4_output")) + + .Operators.Add(new LearningModelOperator("GreaterOrEqual") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 180 })) + .SetOutput("C", "greater4_output")) + .Operators.Add(new LearningModelOperator("Less") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 240 })) + .SetOutput("C", "less4_output")) + + .Operators.Add(new LearningModelOperator("GreaterOrEqual") + .SetInput("A", "hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 300 })) + .SetOutput("C", "mask6")) + ; + + builder.Save(@"C:\Users\t-janedu\source\repos\Windows-Machine-Learning\Samples\AudioPreprocessing\WinUI\AudioPreprocessing\AudioPreprocessing\tmp\colorize_me.onnx"); + + var model = builder.CreateModel(); + var session = new LearningModelSession(model); + var binding = new LearningModelBinding(session); + } + + static unsafe SoftwareBitmap ColorizeMelspectrogram(SoftwareBitmap bwSpectrogram) { using (BitmapBuffer buffer = bwSpectrogram.LockBuffer(BitmapBufferAccessMode.Write)) @@ -150,6 +265,12 @@ static unsafe SoftwareBitmap ColorizeMelspectrogram(SoftwareBitmap bwSpectrogram IMemoryBufferByteAccess memoryBuffer = reference.As(); memoryBuffer.GetBuffer(out byte* dataInBytes, out uint capacity); + //HSV conversion constants + float value = 0.5f; + float saturation = 0.7f; + byte c = (byte)(value * saturation * 255); + byte m = (byte)((value - saturation) * 255); + // Edit the BGRA Plane BitmapPlaneDescription bufferLayout = buffer.GetPlaneDescription(0); for (int i = 0; i < bufferLayout.Height; i++) @@ -157,15 +278,49 @@ static unsafe SoftwareBitmap ColorizeMelspectrogram(SoftwareBitmap bwSpectrogram for (int j = 0; j < bufferLayout.Width; j++) { int pixel = bufferLayout.StartIndex + bufferLayout.Stride * i + 4 * j; - //Lines below can be tweaked for different custom color filters - //Blue - dataInBytes[pixel + 0] = (byte)((255 - dataInBytes[pixel + 0]) / 2); - //Green - dataInBytes[pixel + 1] = (byte)(dataInBytes[pixel + 1] / 2); - //Red - //dataInBytes[pixel + 2] = (byte)(dataInBytes[pixel + 2]); - //Alpha - must leave each pixel at max - dataInBytes[bufferLayout.StartIndex + bufferLayout.Stride * i + 4 * j + 3] = (byte)255; + int hue = dataInBytes[pixel]; + byte x = (byte)(c * (1 - Math.Abs((hue / 60) / 2 - 1))); + + int b = pixel + 0; + int g = pixel + 1; + int r = pixel + 2; + int a = pixel + 3; //Alpha Layer is always at 255 for full opacity + switch (hue) + { + case int n when (n < 60): + dataInBytes[r] = c; + dataInBytes[g] = x; + dataInBytes[b] = 0; + break; + case int n when (60 <= n && n < 120): + dataInBytes[r] = x; + dataInBytes[g] = c; + dataInBytes[b] = 0; + break; + case int n when (120 <= n && n < 180): + dataInBytes[r] = 0; + dataInBytes[g] = c; + dataInBytes[b] = x; + break; + case int n when (180 <= n && n < 240): + dataInBytes[r] = 0; + dataInBytes[g] = x; + dataInBytes[b] = c; + break; + case int n when (240 <= n && n < 300): + dataInBytes[r] = x; + dataInBytes[g] = 0; + dataInBytes[b] = c; + break; + case int n when (300 <= n): + dataInBytes[r] = c; + dataInBytes[g] = 0; + dataInBytes[b] = x; + break; + } + // For the conversion, add m to all channels + for (int k = 0; k < 3; k++) dataInBytes[pixel + k] += m; + dataInBytes[a] = 255; } } } diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs index 9932abf0..a4bacee7 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs @@ -97,13 +97,6 @@ private async void SaveSoftwareBitmapToFile(SoftwareBitmap softwareBitmap, Stora // Set the software bitmap encoder.SetSoftwareBitmap(softwareBitmap); - // Set additional encoding parameters, if needed - encoder.BitmapTransform.ScaledWidth = 320; - encoder.BitmapTransform.ScaledHeight = 240; - encoder.BitmapTransform.Rotation = Windows.Graphics.Imaging.BitmapRotation.Clockwise90Degrees; - encoder.BitmapTransform.InterpolationMode = BitmapInterpolationMode.Fant; - encoder.IsThumbnailGenerated = true; - try { await encoder.FlushAsync(); diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/tmp/colorize_me.onnx b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/tmp/colorize_me.onnx new file mode 100644 index 0000000000000000000000000000000000000000..e48b22aac5f5a52f2dc9bd09a7983103f9af2b12 GIT binary patch literal 1805 zcma)7O>fgc5RGG}F+&?d3I}q4$l?H&Nb7Zyrab_5%2xy`QhR}m`P%$EfKxz~8<9Os*7E-Q<@yL>#1U6KR%usr_0sbhI znGUWkVyGCIdZK8z9Z01ybs{-T-4^OLP<643|014vR)<=e#*>Am1jfG+PnackM%P{k zA=d9=c4g{^pR8t;BZfv7nE5t2&FvTSz>k9Y(frxx*vASzfTDQw=1P`P%R;EQ zlPL1qd#JNH!CF#;TiBBpvV?vfp)Uz}o@qA-&$c+u70>%co(b&{=2?m)p2=>J1wwm8 zSm@+gAXcFTA}mM|Zh;h9u&-pnF0?>Js;mLc8M!e&R;>Qz^c z209FetrC9}9?LwcJT9~7KpD3FCeJs)67M3KJyvUJEG?>W@s(h98817chAmMhFPhuB zQde9ttcF^tG;_^c43qR`rRe3842p9ji;7_|)PDcE^Vq!2bQiatm%GQ{ zbyu7xWO#UEDg4LH%Hh4_aE=__zBN30nJC;DnCp4;%J77=4T3j-!n0f+_W3#$O*nK;9M5%51}~wkYoO}iAL7rYK5E0p5&O(sE#EEI GxBda7Y2Q`= literal 0 HcmV?d00001 From 77085b5d322effbcc0239a0f3483d2600cff83f9 Mon Sep 17 00:00:00 2001 From: janezdu Date: Tue, 6 Jul 2021 11:50:05 -0400 Subject: [PATCH 2/9] Compare hsv to rgb --- .../AudioPreprocessing/MainWindow.xaml | 3 +- .../AudioPreprocessing/MainWindow.xaml.cs | 22 +- .../Model/PreprocessModel.cs | 317 +++++++++++++++--- 3 files changed, 286 insertions(+), 56 deletions(-) diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml index 1b5f9a10..4a94ad7a 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml @@ -24,7 +24,8 @@ HorizontalAlignment="Center" VerticalAlignment="Center" /> - + + diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs index ef553871..7cc80161 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs @@ -32,7 +32,8 @@ public MainWindow() { InitializeComponent(); ViewModel = new PreprocessViewModel(); - spectrogram.Source = new SoftwareBitmapSource(); + spectrogramComputational.Source = new SoftwareBitmapSource(); + spectrogramBitmapEdited.Source = new SoftwareBitmapSource(); } public PreprocessViewModel ViewModel { get; set; } @@ -40,9 +41,19 @@ private async void OnOpenClick(object sender, RoutedEventArgs e) { string wavPath = await GetFilePath(); PreprocessModel melSpectrogram = new PreprocessModel(); - var softwareBitmap = melSpectrogram.GenerateMelSpectrogram(wavPath, ColorMelSpectrogramCheckBox.IsChecked ?? false); - //PreprocessModel.Colorize(VideoFrame.CreateWithSoftwareBitmap(softwareBitmap), 0.5f, 0.3f); - + bool colorize = ColorMelSpectrogramCheckBox.IsChecked ?? false; + // Use bitmap editing, pixel by pixel, to colorize image, if box is checked + SoftwareBitmap softwareBitmap = melSpectrogram.GenerateMelSpectrogram(wavPath, colorize); + // Use computational graph to colorize image, if box is checked + SoftwareBitmap computationalSoftwareBitmap; + if (colorize) + { + computationalSoftwareBitmap = PreprocessModel.ColorizeWithComputationalGraph(VideoFrame.CreateWithSoftwareBitmap(softwareBitmap), 0.7f, 0.5f); + } + else + { + computationalSoftwareBitmap = softwareBitmap; + } ViewModel.AudioPath = wavPath; ViewModel.MelSpectrogramImage = softwareBitmap; @@ -56,7 +67,8 @@ private async void OnOpenClick(object sender, RoutedEventArgs e) WavFilePath.Text = ViewModel.AudioPath; - await ((SoftwareBitmapSource)spectrogram.Source).SetBitmapAsync(softwareBitmap); + await ((SoftwareBitmapSource)spectrogramComputational.Source).SetBitmapAsync(softwareBitmap); + await ((SoftwareBitmapSource)spectrogramBitmapEdited.Source).SetBitmapAsync(computationalSoftwareBitmap); } private async Task GetFilePath() diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs index 2bd40e85..56bf1322 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs @@ -33,7 +33,7 @@ public SoftwareBitmap GenerateMelSpectrogram(string audioPath, bool color = fals { var signal = GetSignalFromFile(audioPath); var softwareBitmap = GetMelspectrogramFromSignal(signal); - if (color) softwareBitmap = ColorizeMelspectrogram(softwareBitmap); + if (color) softwareBitmap = ColorizeWithBitmapEditing(softwareBitmap); return softwareBitmap; } @@ -152,16 +152,18 @@ static SoftwareBitmap GetMelspectrogramFromSignal( return outputImage.SoftwareBitmap; } - public static void Colorize(Windows.Media.VideoFrame image, float saturation, float value) + public static SoftwareBitmap ColorizeWithComputationalGraph(Windows.Media.VideoFrame image, float saturation, float value) { long width = image.SoftwareBitmap.PixelWidth; long height = image.SoftwareBitmap.PixelHeight; - long channels = image.SoftwareBitmap.BitmapPixelFormat == Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8 ? 4 : 1; + long channels = image.SoftwareBitmap.BitmapPixelFormat == Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8 ? 3 : 1; long batch_size = 1; - var c = saturation * value; - var m = value - saturation; + var saturation_value = saturation * value; + var saturation_value_difference = value - saturation; + + Stopwatch stopWatch = new Stopwatch(); var builder = LearningModelBuilder.Create(13) @@ -173,14 +175,15 @@ public static void Colorize(Windows.Media.VideoFrame image, float saturation, fl .SetConstant("ends", TensorInt64Bit.CreateFromIterable(new long[] { 4 }, new long[] { long.MaxValue, 1, long.MaxValue, long.MaxValue })) .SetConstant("axes", TensorInt64Bit.CreateFromIterable(new long[] { 4 }, new long[] { 0, 1, 2, 3 })) .SetConstant("steps", TensorInt64Bit.CreateFromIterable(new long[] { 4 }, new long[] { 1, 1, 1, 1 })) - .SetOutput("output", "hue")) + .SetOutput("output", "Hue")) .Operators.Add(new LearningModelOperator("Div") - .SetInput("A", "hue") + .SetInput("A", "Hue") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 60 })) .SetOutput("C", "div_output")) .Operators.Add(new LearningModelOperator("Mod") .SetInput("A", "div_output") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 2 })) + .SetAttribute("fmod", TensorInt64Bit.CreateFromIterable(new long[] { }, new long[] { 1 })) .SetOutput("C", "mod_output")) .Operators.Add(new LearningModelOperator("Sub") .SetInput("A", "mod_output") @@ -195,69 +198,283 @@ public static void Colorize(Windows.Media.VideoFrame image, float saturation, fl .SetOutput("C", "sub2_output")) .Operators.Add(new LearningModelOperator("Mul") .SetInput("A", "sub2_output") - .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { c })) - .SetOutput("C", "mul1_output")) - // generate 6 masks + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value })) + .SetOutput("C", "X")) + // LESS THAN MASKS .Operators.Add(new LearningModelOperator("Less") - .SetInput("A", "hue") - .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 60 })) - .SetOutput("C", "mask1")) - - .Operators.Add(new LearningModelOperator("GreaterOrEqual") - .SetInput("A", "hue") + .SetInput("A", "Hue") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 60 })) - .SetOutput("C", "greater2_output")) - .Operators.Add(new LearningModelOperator("Less") - .SetInput("A", "hue") - .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 120 })) - .SetOutput("C", "less2_output")) + .SetOutput("C", "lt60_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "lt60_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "lt60_mask")) .Operators.Add(new LearningModelOperator("Less") - .SetInput("A", "hue") + .SetInput("A", "Hue") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 120 })) - .SetOutput("C", "less2_output")) + .SetOutput("C", "lt120_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "lt120_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "lt120_mask")) - .Operators.Add(new LearningModelOperator("GreaterOrEqual") - .SetInput("A", "hue") - .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 120 })) - .SetOutput("C", "greater3_output")) .Operators.Add(new LearningModelOperator("Less") - .SetInput("A", "hue") + .SetInput("A", "Hue") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 180 })) - .SetOutput("C", "less3_output")) + .SetOutput("C", "lt180_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "lt180_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "lt180_mask")) - .Operators.Add(new LearningModelOperator("GreaterOrEqual") - .SetInput("A", "hue") - .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 180 })) - .SetOutput("C", "greater4_output")) .Operators.Add(new LearningModelOperator("Less") - .SetInput("A", "hue") + .SetInput("A", "Hue") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 240 })) - .SetOutput("C", "less4_output")) + .SetOutput("C", "lt240_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "lt240_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "lt240_mask")) - .Operators.Add(new LearningModelOperator("GreaterOrEqual") - .SetInput("A", "hue") - .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 180 })) - .SetOutput("C", "greater4_output")) .Operators.Add(new LearningModelOperator("Less") - .SetInput("A", "hue") + .SetInput("A", "Hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 300 })) + .SetOutput("C", "lt300_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "lt300_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "lt300_mask")) + // GREATER THAN MASKS + .Operators.Add(new LearningModelOperator("Greater") + .SetInput("A", "Hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 60 })) + .SetOutput("C", "gt60_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "gt60_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "gt60_mask")) + + .Operators.Add(new LearningModelOperator("Greater") + .SetInput("A", "Hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 120 })) + .SetOutput("C", "gt120_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "gt120_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "gt120_mask")) + + .Operators.Add(new LearningModelOperator("Greater") + .SetInput("A", "Hue") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 180 })) + .SetOutput("C", "gt180_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "gt180_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "gt180_mask")) + + .Operators.Add(new LearningModelOperator("Greater") + .SetInput("A", "Hue") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 240 })) - .SetOutput("C", "less4_output")) - - .Operators.Add(new LearningModelOperator("GreaterOrEqual") - .SetInput("A", "hue") + .SetOutput("C", "gt240_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "gt240_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "gt240_mask")) + + .Operators.Add(new LearningModelOperator("Greater") + .SetInput("A", "Hue") .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 300 })) - .SetOutput("C", "mask6")) - ; + .SetOutput("C", "gt300_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "gt300_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "gt300_mask")) + + // CASE 0 => 60 + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "lt60_mask") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value })) + .SetOutput("C", "0_to_60_R")) + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "lt60_mask") + .SetInput("B", "X") + .SetOutput("C", "0_to_60_G")) + // CASE 60 => 120 + .Operators.Add(new LearningModelOperator("And") + .SetInput("A", "gt60_mask_b") + .SetInput("B", "lt120_mask_b") + .SetOutput("C", "60_to_120_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "60_to_120_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "60_to_120_mask")) + + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "60_to_120_mask") + .SetInput("B", "X") + .SetOutput("C", "60_to_120_R")) + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "60_to_120_mask") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value })) + .SetOutput("C", "60_to_120_G")) + + // CASE 120 => 180 + .Operators.Add(new LearningModelOperator("And") + .SetInput("A", "gt120_mask_b") + .SetInput("B", "lt180_mask_b") + .SetOutput("C", "120_to_180_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "120_to_180_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "120_to_180_mask")) + + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "120_to_180_mask") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value })) + .SetOutput("C", "120_to_180_G")) + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "120_to_180_mask") + .SetInput("B", "X") + .SetOutput("C", "120_to_180_B")) + // CASE 180 => 240 + .Operators.Add(new LearningModelOperator("And") + .SetInput("A", "gt180_mask_b") + .SetInput("B", "lt240_mask_b") + .SetOutput("C", "180_to_240_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "180_to_240_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "180_to_240_mask")) + + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "180_to_240_mask") + .SetInput("B", "X") + .SetOutput("C", "180_to_240_G")) + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "180_to_240_mask") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value })) + .SetOutput("C", "180_to_240_B")) + // CASE 240 => 300 + .Operators.Add(new LearningModelOperator("And") + .SetInput("A", "gt240_mask_b") + .SetInput("B", "lt300_mask_b") + .SetOutput("C", "240_to_300_mask_b")) + .Operators.Add(new LearningModelOperator("Cast") + .SetInput("input", "240_to_300_mask_b") + .SetAttribute("to", TensorInt32Bit.CreateFromIterable(new long[] { }, new int[] { 1 })) + .SetOutput("output", "240_to_300_mask")) + + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "240_to_300_mask") + .SetInput("B", "X") + .SetOutput("C", "240_to_300_R")) + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "240_to_300_mask") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value })) + .SetOutput("C", "240_to_300_B")) + // CASE 300 => 360 + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "gt300_mask") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value })) + .SetOutput("C", "300_to_360_R")) + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "gt300_mask") + .SetInput("B", "X") + .SetOutput("C", "300_to_360_B")) + // SUM UP R + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "300_to_360_R") + .SetInput("B", "240_to_300_R") + .SetOutput("C", "red1")) + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "red1") + .SetInput("B", "60_to_120_R") + .SetOutput("C", "red2")) + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "red2") + .SetInput("B", "0_to_60_R") + .SetOutput("C", "red")) + + // SUM UP G + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "180_to_240_G") + .SetInput("B", "120_to_180_G") + .SetOutput("C", "green1")) + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "green1") + .SetInput("B", "60_to_120_G") + .SetOutput("C", "green2")) + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "green2") + .SetInput("B", "0_to_60_G") + .SetOutput("C", "green")) + // SUM UP B + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "120_to_180_B") + .SetInput("B", "180_to_240_B") + .SetOutput("C", "blue1")) + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "blue1") + .SetInput("B", "240_to_300_B") + .SetOutput("C", "blue2")) + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "blue2") + .SetInput("B", "300_to_360_B") + .SetOutput("C", "blue")) + + // Create Output + .Operators.Add(new LearningModelOperator("SequenceConstruct") + .SetInput("inputs", "red") + .SetOutput("output_sequence", "sequence_1")) + .Operators.Add(new LearningModelOperator("SequenceInsert") + .SetInput("input_sequence", "sequence_1") + .SetInput("tensor", "green") + .SetOutput("output_sequence", "sequence_2")) + .Operators.Add(new LearningModelOperator("SequenceInsert") + .SetInput("input_sequence", "sequence_2") + .SetInput("tensor", "blue") + .SetOutput("output_sequence", "sequence_image")) + .Operators.Add(new LearningModelOperator("ConcatFromSequence") + .SetInput("input_sequence", "sequence_image") + .SetAttribute("axis", TensorInt64Bit.CreateFromIterable(new long[] { 1 }, new long[] { 1 })) + .SetOutput("concat_result", "concat_output")) + + .Operators.Add(new LearningModelOperator("Add") + .SetInput("A", "concat_output") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { saturation_value_difference })) + .SetOutput("C", "0_1_offset")) + + .Operators.Add(new LearningModelOperator("Mul") + .SetInput("A", "0_1_offset") + .SetConstant("B", TensorFloat.CreateFromIterable(new long[] { 1 }, new float[] { 255 })) + .SetOutput("C", "Output")); + ; + builder.Save("colorize_me.onnx"); - builder.Save(@"C:\Users\t-janedu\source\repos\Windows-Machine-Learning\Samples\AudioPreprocessing\WinUI\AudioPreprocessing\AudioPreprocessing\tmp\colorize_me.onnx"); + var output_image = new VideoFrame(Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8, (int)width, (int)height); + stopWatch.Start(); var model = builder.CreateModel(); - var session = new LearningModelSession(model); + var device = new LearningModelDevice(LearningModelDeviceKind.Default); + var session = new LearningModelSession(model, device); var binding = new LearningModelBinding(session); - } + binding.Bind("Input", image); + binding.Bind("Output", output_image); + + session.Evaluate(binding, ""); + stopWatch.Stop(); + + TimeSpan ts = stopWatch.Elapsed; + // Format and display the TimeSpan value. + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", + ts.Hours, ts.Minutes, ts.Seconds, + ts.Milliseconds / 10); + Console.WriteLine("RunTime " + elapsedTime); + return output_image.SoftwareBitmap; + } - static unsafe SoftwareBitmap ColorizeMelspectrogram(SoftwareBitmap bwSpectrogram) + private static unsafe SoftwareBitmap ColorizeWithBitmapEditing(SoftwareBitmap bwSpectrogram) { using (BitmapBuffer buffer = bwSpectrogram.LockBuffer(BitmapBufferAccessMode.Write)) { From c45f2d66ab21e6ead9066d789374b0588b41e0fd Mon Sep 17 00:00:00 2001 From: janezdu Date: Tue, 6 Jul 2021 12:29:54 -0400 Subject: [PATCH 3/9] Refactor colorizing code --- .../AudioPreprocessing/MainWindow.xaml | 3 +- .../AudioPreprocessing/MainWindow.xaml.cs | 31 ++----------------- .../Model/PreprocessModel.cs | 25 +++++++-------- .../ViewModel/PreprocessViewModel.cs | 27 ++++++++++++++-- 4 files changed, 40 insertions(+), 46 deletions(-) diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml index 4a94ad7a..1b5f9a10 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml @@ -24,8 +24,7 @@ HorizontalAlignment="Center" VerticalAlignment="Center" /> - - + diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs index 7cc80161..99aee3cc 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml.cs @@ -32,43 +32,18 @@ public MainWindow() { InitializeComponent(); ViewModel = new PreprocessViewModel(); - spectrogramComputational.Source = new SoftwareBitmapSource(); - spectrogramBitmapEdited.Source = new SoftwareBitmapSource(); + spectrogram.Source = new SoftwareBitmapSource(); } public PreprocessViewModel ViewModel { get; set; } private async void OnOpenClick(object sender, RoutedEventArgs e) { string wavPath = await GetFilePath(); - PreprocessModel melSpectrogram = new PreprocessModel(); - bool colorize = ColorMelSpectrogramCheckBox.IsChecked ?? false; - // Use bitmap editing, pixel by pixel, to colorize image, if box is checked - SoftwareBitmap softwareBitmap = melSpectrogram.GenerateMelSpectrogram(wavPath, colorize); - // Use computational graph to colorize image, if box is checked - SoftwareBitmap computationalSoftwareBitmap; - if (colorize) - { - computationalSoftwareBitmap = PreprocessModel.ColorizeWithComputationalGraph(VideoFrame.CreateWithSoftwareBitmap(softwareBitmap), 0.7f, 0.5f); - } - else - { - computationalSoftwareBitmap = softwareBitmap; - } - ViewModel.AudioPath = wavPath; - ViewModel.MelSpectrogramImage = softwareBitmap; - - //Image control only accepts BGRA8 encoding and Premultiplied/no alpha channel. This checks and converts - //the SoftwareBitmap we want to bind. - if (softwareBitmap.BitmapPixelFormat != BitmapPixelFormat.Bgra8 || - softwareBitmap.BitmapAlphaMode != BitmapAlphaMode.Premultiplied) - { - softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); - } + ViewModel.GenerateMelSpectrograms(wavPath, ColorMelSpectrogramCheckBox.IsChecked ?? false); WavFilePath.Text = ViewModel.AudioPath; - await ((SoftwareBitmapSource)spectrogramComputational.Source).SetBitmapAsync(softwareBitmap); - await ((SoftwareBitmapSource)spectrogramBitmapEdited.Source).SetBitmapAsync(computationalSoftwareBitmap); + await ((SoftwareBitmapSource)spectrogram.Source).SetBitmapAsync(ViewModel.MelSpectrogramImage); } private async Task GetFilePath() diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs index 56bf1322..fb635bdb 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/PreprocessModel.cs @@ -29,11 +29,11 @@ public class PreprocessModel public string MelSpecImagePath { get; set; } - public SoftwareBitmap GenerateMelSpectrogram(string audioPath, bool color = false) + public SoftwareBitmap GenerateMelSpectrogram(string audioPath) { var signal = GetSignalFromFile(audioPath); var softwareBitmap = GetMelspectrogramFromSignal(signal); - if (color) softwareBitmap = ColorizeWithBitmapEditing(softwareBitmap); + //if (color) softwareBitmap = ColorizeWithBitmapEditing(softwareBitmap); return softwareBitmap; } @@ -152,11 +152,11 @@ static SoftwareBitmap GetMelspectrogramFromSignal( return outputImage.SoftwareBitmap; } - public static SoftwareBitmap ColorizeWithComputationalGraph(Windows.Media.VideoFrame image, float saturation, float value) + public static SoftwareBitmap ColorizeWithComputationalGraph(SoftwareBitmap image, float saturation = 0.7f, float value = 0.5f) { - long width = image.SoftwareBitmap.PixelWidth; - long height = image.SoftwareBitmap.PixelHeight; - long channels = image.SoftwareBitmap.BitmapPixelFormat == Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8 ? 3 : 1; + long width = image.PixelWidth; + long height = image.PixelHeight; + long channels = image.BitmapPixelFormat == Windows.Graphics.Imaging.BitmapPixelFormat.Bgra8 ? 3 : 1; long batch_size = 1; @@ -459,7 +459,7 @@ public static SoftwareBitmap ColorizeWithComputationalGraph(Windows.Media.VideoF var session = new LearningModelSession(model, device); var binding = new LearningModelBinding(session); - binding.Bind("Input", image); + binding.Bind("Input", VideoFrame.CreateWithSoftwareBitmap(image)); binding.Bind("Output", output_image); session.Evaluate(binding, ""); @@ -474,7 +474,7 @@ public static SoftwareBitmap ColorizeWithComputationalGraph(Windows.Media.VideoF return output_image.SoftwareBitmap; } - private static unsafe SoftwareBitmap ColorizeWithBitmapEditing(SoftwareBitmap bwSpectrogram) + public static unsafe void ColorizeWithBitmapEditing(SoftwareBitmap bwSpectrogram, float value = 0.5f, float saturation = 0.7f) { using (BitmapBuffer buffer = bwSpectrogram.LockBuffer(BitmapBufferAccessMode.Write)) { @@ -483,8 +483,6 @@ private static unsafe SoftwareBitmap ColorizeWithBitmapEditing(SoftwareBitmap bw memoryBuffer.GetBuffer(out byte* dataInBytes, out uint capacity); //HSV conversion constants - float value = 0.5f; - float saturation = 0.7f; byte c = (byte)(value * saturation * 255); byte m = (byte)((value - saturation) * 255); @@ -497,7 +495,7 @@ private static unsafe SoftwareBitmap ColorizeWithBitmapEditing(SoftwareBitmap bw int pixel = bufferLayout.StartIndex + bufferLayout.Stride * i + 4 * j; int hue = dataInBytes[pixel]; byte x = (byte)(c * (1 - Math.Abs((hue / 60) / 2 - 1))); - + int b = pixel + 0; int g = pixel + 1; int r = pixel + 2; @@ -513,7 +511,7 @@ private static unsafe SoftwareBitmap ColorizeWithBitmapEditing(SoftwareBitmap bw dataInBytes[r] = x; dataInBytes[g] = c; dataInBytes[b] = 0; - break; + break; case int n when (120 <= n && n < 180): dataInBytes[r] = 0; dataInBytes[g] = c; @@ -523,7 +521,7 @@ private static unsafe SoftwareBitmap ColorizeWithBitmapEditing(SoftwareBitmap bw dataInBytes[r] = 0; dataInBytes[g] = x; dataInBytes[b] = c; - break; + break; case int n when (240 <= n && n < 300): dataInBytes[r] = x; dataInBytes[g] = 0; @@ -541,7 +539,6 @@ private static unsafe SoftwareBitmap ColorizeWithBitmapEditing(SoftwareBitmap bw } } } - return bwSpectrogram; } } } diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs index a4bacee7..549b6b56 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/ViewModel/PreprocessViewModel.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using System.Windows.Input; using Windows.Graphics.Imaging; +using Windows.Media; using Windows.Storage; using Windows.Storage.Pickers; using Windows.Storage.Streams; @@ -17,11 +18,12 @@ public class PreprocessViewModel : INotifyPropertyChanged { private string audioPath; private string imagePath; + private PreprocessModel preprocessModel; private SoftwareBitmap melSpectrogramImage; public PreprocessViewModel() { - PreprocessModel preprocessModel = new PreprocessModel(); + preprocessModel = new PreprocessModel(); audioPath = preprocessModel.AudioPath; imagePath = preprocessModel.MelSpecImagePath; } @@ -61,13 +63,34 @@ public string ImagePath get { return imagePath; } set { imagePath = value; OnPropertyChanged(); } } - public SoftwareBitmap MelSpectrogramImage { get { return melSpectrogramImage; } set { melSpectrogramImage = value; OnPropertyChanged(); } } + public void GenerateMelSpectrograms(string wavPath, bool colorize) + { + MelSpectrogramImage = preprocessModel.GenerateMelSpectrogram(wavPath); + if (colorize) + { + // Use computational graph to colorize image, if box is checked + //MelSpectrogramImage = PreprocessModel.ColorizeWithComputationalGraph(MelSpectrogramImage); + // Use bitmap editing, pixel by pixel, to colorize image, if box is checked. In place editing. + PreprocessModel.ColorizeWithBitmapEditing(MelSpectrogramImage); + } + AudioPath = wavPath; + + //Image control only accepts BGRA8 encoding and Premultiplied/no alpha channel. This checks and converts + //the SoftwareBitmap we want to bind. + if (MelSpectrogramImage.BitmapPixelFormat != BitmapPixelFormat.Bgra8 || + MelSpectrogramImage.BitmapAlphaMode != BitmapAlphaMode.Premultiplied) + { + MelSpectrogramImage = SoftwareBitmap.Convert(MelSpectrogramImage, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied); + } + } + + public ICommand SaveFileCommand => new AsyncRelayCommand(p => SaveFile()); private async Task SaveFile() From 2b89b27f43cd278383e21cff2b543ad1f8041b44 Mon Sep 17 00:00:00 2001 From: janezdu Date: Tue, 6 Jul 2021 12:58:31 -0400 Subject: [PATCH 4/9] Add toggles to UI --- .../AudioPreprocessing/MainWindow.xaml | 39 ++++++++++++++----- .../AudioPreprocessing/MainWindow.xaml.cs | 8 ++++ .../AudioPreprocessing/Model/ModelSetting.cs | 28 +++++++++++++ .../Model/PreprocessModel.cs | 4 +- .../ViewModel/PreprocessViewModel.cs | 4 +- 5 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/Model/ModelSetting.cs diff --git a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml index 1b5f9a10..ea7d8ff8 100644 --- a/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml +++ b/Samples/AudioPreprocessing/WinUI/AudioPreprocessing/AudioPreprocessing/MainWindow.xaml @@ -10,20 +10,39 @@ xmlns:model="using:AudioPreprocessing.Model" xmlns:muxc="using:Microsoft.UI.Xaml.Controls" mc:Ignorable="d"> - - -