Create a project

<?xml version="1.0" encoding="utf-8" ?>
<ContentPage
    x:Class="AIApp.MainPage"
    xmlns="https://xamarin.com/schemas/2014/forms"
    xmlns:x="https://schemas.microsoft.com/winfx/2009/xaml"
    xmlns:ios="clr-namespace:Xamarin.Forms.PlatformConfiguration.iOSSpecific;assembly=Xamarin.Forms.Core"
    xmlns:local="clr-namespace:AIApp"
    Title="Safe Area"
    ios:Page.UseSafeArea="True">

    <StackLayout>
        <Image
            x:Name="picture"
            Aspect="AspectFill"
            VerticalOptions="FillAndExpand" />
        <Label x:Name="output" HorizontalOptions="CenterAndExpand" />
        <StackLayout Orientation="Horizontal">
            <Button
                Clicked="PickPhotoButton_Clicked"
                HorizontalOptions="FillAndExpand"
                Text="Pick a picture" />
            <Button
                Clicked="TakePhotoButton_Clicked"
                HorizontalOptions="FillAndExpand"
                Text="Take a picture" />
        </StackLayout>
    </StackLayout>

</ContentPage>

using Plugin.Media;
using Plugin.Media.Abstractions;
using System;
using System.Threading.Tasks;
using Xamarin.Forms;

namespace AIApp
{
    public partial class MainPage : ContentPage
    {
        public MainPage()
        {
            InitializeComponent();
        }

        private async void TakePhotoButton_Clicked(object sender, EventArgs e)
        {
            await ProcessPhotoAsync(true);
        }

        private async void PickPhotoButton_Clicked(object sender, EventArgs e)
        {
            await ProcessPhotoAsync(false);
        }

        private async Task ProcessPhotoAsync(bool useCamera)
        {
            await CrossMedia.Current.Initialize();
            if (useCamera ? !CrossMedia.Current.IsTakePhotoSupported : !CrossMedia.Current.IsPickPhotoSupported)
            {
                await DisplayAlert("Info", "Your phone doesn't support photo feature.", "OK");
                return;
            }

            var photo = useCamera ? 
                await CrossMedia.Current.TakePhotoAsync(new StoreCameraMediaOptions()) : 
                await CrossMedia.Current.PickPhotoAsync();
            if (photo == null)
            {
                picture.Source = null;
                return;
            }

            picture.Source = ImageSource.FromFile(photo.Path);

            var service = DependencyService.Get<IPhotoDetector>();
            if (service == null)
            {
                await DisplayAlert("Info", "Not implemented the feature on your device.", "OK");
                return;
            }

            using (var s = photo.GetStream())
            {
                var result = await service.DetectAsync(s);
                output.Text = $"It looks like a {result}";
            }
        }
    }
}

using System.IO;
using System.Threading.Tasks;

namespace AIApp
{
    public interface IPhotoDetector
    {
        Task<FriesOrNotFriesTag> DetectAsync(Stream photo);
    }

    public enum FriesOrNotFriesTag
    {
        None,
        Fries,
        NotFries,
    }
}

Create ML models

Add Windows 10 implementation

using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Windows.AI.MachineLearning;
using Windows.Graphics.Imaging;
using Windows.Media;
using Windows.Storage;
using Xamarin.Forms;

[assembly: Dependency(typeof(AIApp.UWP.PhotoDetector))]
namespace AIApp.UWP
{
    public class PhotoDetector : IPhotoDetector
    {
        private FriesOrNotFriesModel _model;
        public async Task DetectAsync(Stream photo)
        {
            await InitializeModelAsync();
            var bitmapDecoder = await BitmapDecoder.CreateAsync(photo.AsRandomAccessStream());
            var output = await _model.EvaluateAsync(new FriesOrNotFriesInput
            {
                data = ImageFeatureValue.CreateFromVideoFrame(VideoFrame.CreateWithSoftwareBitmap(await bitmapDecoder.GetSoftwareBitmapAsync())),
            });
            var label = output.classLabel.GetAsVectorView().FirstOrDefault();
            return Enum.Parse(label);
        }

        private async Task InitializeModelAsync()
        {
            if (_model != null)
            {
                return;
            }

            var onnx = await StorageFile.GetFileFromApplicationUriAsync(new Uri("ms-appx:///Assets/FriesOrNotFries.onnx"));
            _model = await FriesOrNotFriesModel.CreateFromStreamAsync(onnx);
        }
    }
}

Add Android implementation

using Android.Graphics;
using Org.Tensorflow.Contrib.Android;
using Plugin.CurrentActivity;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Xamarin.Forms;

[assembly: Dependency(typeof(AIApp.Droid.PhotoDetector))]
namespace AIApp.Droid
{
    public class PhotoDetector : IPhotoDetector
    {
        private static readonly string ModelFile = "model.pb";
        private static readonly string LabelFile = "labels.txt";
        private static readonly string InputName = "Placeholder";
        private static readonly string OutputName = "loss";
        private static readonly int InputSize = 227;
        private readonly TensorFlowInferenceInterface _inferenceInterface;
        private readonly string[] _labels;

        public PhotoDetector()
        {
            _inferenceInterface = new TensorFlowInferenceInterface(CrossCurrentActivity.Current.Activity.Assets, ModelFile);
            using (var sr = new StreamReader(CrossCurrentActivity.Current.Activity.Assets.Open(LabelFile)))
            {
                _labels = sr.ReadToEnd().Split('\n').Select(x => x.Trim()).Where(x => !string.IsNullOrEmpty(x)).ToArray();
            }
        }

        public async Task DetectAsync(Stream photo)
        {
            var bitmap = await BitmapFactory.DecodeStreamAsync(photo);
            var floatValues = GetBitmapPixels(bitmap);
            var outputs = new float[_labels.Length];
            _inferenceInterface.Feed(InputName, floatValues, 1, InputSize, InputSize, 3);
            _inferenceInterface.Run(new[] { OutputName });
            _inferenceInterface.Fetch(OutputName, outputs);
            var index = Array.IndexOf(outputs, outputs.Max());
            return (FriesOrNotFriesTag)Enum.Parse(typeof(FriesOrNotFriesTag), _labels[index]);
        }

        private async Task LoadByteArrayFromAssetsAsync(string name)
        {
            using (var s = CrossCurrentActivity.Current.Activity.Assets.Open(name))
            using (var ms = new MemoryStream())
            {
                await s.CopyToAsync(ms);
                ms.Seek(0, SeekOrigin.Begin);
                return ms.ToArray();
            }
        }

        private static float[] GetBitmapPixels(Bitmap bitmap)
        {
            var floatValues = new float[InputSize * InputSize * 3];
            using (var scaledBitmap = Bitmap.CreateScaledBitmap(bitmap, InputSize, InputSize, false))
            {
                using (var resizedBitmap = scaledBitmap.Copy(Bitmap.Config.Argb8888, false))
                {
                    var intValues = new int[InputSize * InputSize];
                    resizedBitmap.GetPixels(intValues, 0, resizedBitmap.Width, 0, 0, resizedBitmap.Width, resizedBitmap.Height);
                    for (int i = 0; i > 8) & 0xFF) - 117);
                        floatValues[i * 3 + 2] = (((val >> 16) & 0xFF) - 123);
                    }
                    resizedBitmap.Recycle();
                }
                scaledBitmap.Recycle();
            }

            return floatValues;
        }
    }
}

Add iOS implementation

using CoreFoundation;
using CoreImage;
using CoreML;
using Foundation;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Vision;
using Xamarin.Forms;

[assembly: Dependency(typeof(AIApp.iOS.PhotoDetector))]
namespace AIApp.iOS
{
    public class PhotoDetector : IPhotoDetector
    {
        private readonly MLModel _mlModel;
        private readonly VNCoreMLModel _model;

        public PhotoDetector()
        {
            var assetPath = NSBundle.MainBundle.GetUrlForResource("FriesOrNotFries", "mlmodelc");
            _mlModel = MLModel.Create(assetPath, out var _);
            _model = VNCoreMLModel.FromMLModel(_mlModel, out var __);
        }

        public Task DetectAsync(Stream photo)
        {
            var taskCompletionSource = new TaskCompletionSource();
            void handleClassification(VNRequest request, NSError error)
            {
                var observations = request.GetResults();
                if (observations == null)
                {
                    taskCompletionSource.SetException(new Exception("Unexpected result type from VNCoreMLRequest"));
                    return;
                }

                if (!observations.Any())
                {
                    taskCompletionSource.SetResult(FriesOrNotFriesTag.None);
                    return;
                }

                var best = observations.First();
                taskCompletionSource.SetResult((FriesOrNotFriesTag)Enum.Parse(typeof(FriesOrNotFriesTag), best.Identifier));
            }

            using (var data = NSData.FromStream(photo))
            {
                var ciImage = new CIImage(data);
                var handler = new VNImageRequestHandler(ciImage, new VNImageOptions());
                DispatchQueue.DefaultGlobalQueue.DispatchAsync(() =>
                {
                    handler.Perform(new VNRequest[] { new VNCoreMLRequest(_model, handleClassification) }, out var _);
                });
            }

            return taskCompletionSource.Task;
        }
    }
}

How does it run?

Conclusion

<Project Sdk="Microsoft.NET.Sdk">
  <PropertyGroup>
    <TargetFramework>netcoreapp3.0</TargetFramework>
  </PropertyGroup>
</Project>

<Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">
  <PropertyGroup>
    <TargetFramework>netcoreapp3.0</TargetFramework>
    <UseWPF>true</UseWPF>
  </PropertyGroup>
</Project>

path to apiport.exe folder\ApiPort.exe analyze -f .

Publish to NuGet

Conclusion

Can use .NET Framework libraries?

Conclusion

Using the Microsoft Graph in HoloLens and Mixed Reality

• The Windows Community toolkit controls for Graph
• The High level classes from the .NET SDK for Graph
• Under the hood, the Microsoft Authentication Library (MSAL) is used to authenticate to the Azure Active Directory for using Microsoft Graph

1. I went through the tutorial - https://docs.microsoft.com/en-us/graph/training/uwp-tutorial. I had the full working project and looked at the dependencies.

2. Looking deeper at the XAML and the code, I first wanted to remove the Windows Community Toolkit dependencies because I cannot use them in a Unity 3D project.

3. Lastly, the code of the tutorial uses the .NET SDK for Graph. Because all the SDK for the Microsoft are wrappers on the HTTP Rest API, I decided to not take an extra dll in the Unity project but just do HTTP calls using the UWP HttpClient. ➡ Explanations and code in Step 4 below ⬅

Here are now the real steps for using Microsoft Graph in your HoloLens or Mixed Reality apps.

Step 1. Register an entry 🆔 for your app in the Azure Active Directory

Step 2. Create the Unity 🎮 App

Project setup 📏

• Windows version 1809 for the PC and HoloLens
• Visual Studio 2017 version 15.9.0
• Windows SDK version 10.0.17763.0
• Unity 2018.2.18f1

Note: If you are familiar with Unity and Mixed Reality, please refer to Create a Windows Mixed Reality App from 0 – The Unity project.

• File menu / Build Settings ➡ Universal Windows Platform selected as the Build platform
• Edit menu / Project Settings / Player ➡ Virtual Reality Supported checked
• Edit menu / Project Settings / Player ➡ Other Settings / .NET as Scripting Backend. Please also use the Experimental (.NET 4.6 Equivalent) for the Scripting Runtime Version.
• Put the camera at the origin (0,0,0) and use Solid Color = Black.
• Add an empty game object named ScriptManager with a script Startup where we will place all the source code for simplicity.

Field of view feedback

Plugins

Note: This folder exists and would be populated if you have used this component at least one time for one project (This will be the case if you go through the Graph tutorial that I mentioned)

Note: Using the Plugins\WSA automatically flag the dll to be used only for Windows:

Step 3. ✅ Authenticate against the Azure Active Directory

private async Task GoGraph()
{
    #if UNITY_WSA && !UNITY_EDITOR
    // Get the authentication token using Microsoft.Identity.Client
    bool authenticated = false;
    PublicClientApplication client = new PublicClientApplication("<<YOUR_APP_ID>>"); // (1)
    AuthenticationResult authResult = null;
    IEnumerable<string> Scopes = new List<string> { "User.Read", "User.ReadBasic.All" }; // (2)
    try
    {
        authResult = await client.AcquireTokenAsync(Scopes);
        // Display some text like ($"UniqueId={authResult.UniqueId}, AccessToken={authResult.AccessToken}");
        authenticated = true;
    }
    catch (MsalException msaEx)
    {
        // Display some text like ($"Failed to acquire token: {msaEx.Message}");
    }
    catch (Exception ex)
    {
        // Display some text like ($"Failed to acquire token: {ex.Message}");
    }
    // To be continued...
}
#endif

public class Startup : MonoBehaviour
{
    // Use this for initialization
    async void Start()
    {
        await GoGraph();
    }

Note: I will give you the needed using statements later in this article.

Step 4. 🔍 Query Microsoft Graph

if (authenticated)
{
    // Query MS Graph using a HTTP GET
    string GraphUriAndQuery = @"https://graph.microsoft.com/v1.0/users?$filter=startswith(Displayname,'Marty')&$select=id";
    var uri = new System.Uri(GraphUriAndQuery);
    using (HttpClient httpClient = new HttpClient())
    {
        try
        {
            httpClient.DefaultRequestHeaders.Authorization =
                new AuthenticationHeaderValue("bearer", authResult.AccessToken); // <-- Authentication token got from the MSAL code
            string result = await httpClient.GetStringAsync(uri);

            Rootobject root = new Rootobject(); // JSON pasted as a class :-). See below
            MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(result));
            DataContractJsonSerializer ser = new DataContractJsonSerializer(root.GetType());
            root = ser.ReadObject(ms) as Rootobject;
            // Display some text like ($"id={root.value[0].id}"); // root.value[0].id is the Object ID of the first person found
        }
        catch (Exception ex)
        {
            // Display some text like ($"Failed to query MS Graph: {ex.Message}");
        }
    }
}

public class Rootobject
{
    public string odatacontext { get; set; }
    public Value[] value { get; set; }
}

public class Value
{
    public string id { get; set; }
}

using System.Collections;
using System.Collections.Generic;
using System.Threading.Tasks;
using UnityEngine;
using System;
#if UNITY_WSA && !UNITY_EDITOR
using System.Net.Http;
using System.Net.Http.Headers;
using System.IO;
using System.Runtime.Serialization.Json;
using System.Text;
using Microsoft.Identity.Client;
#endif

The sample code 📜

using System.Collections;
using System.Collections.Generic;
using System.Threading.Tasks;
using UnityEngine;
using System;
#if UNITY_WSA && !UNITY_EDITOR
using System.Net.Http;
using System.Net.Http.Headers;
using System.IO;
using System.Runtime.Serialization.Json;
using System.Text;
using Microsoft.Identity.Client;
#endif

public class Startup : MonoBehaviour
{
    // Use this for initialization
    async void Start()
    {
        await GoGraph();
    }

    private async Task GoGraph()
    {
#if UNITY_WSA && !UNITY_EDITOR
        // Get the authentication token using Microsoft.Identity.Client
        bool authenticated = false;
        PublicClientApplication client = new PublicClientApplication("06644049-fd27-4366-9dfb-c9f1bd99d35a");
        AuthenticationResult authResult = null;
        IEnumerable<string> Scopes = new List<string> { "User.Read", "User.ReadBasic.All" };
        try
        {
            authResult = await client.AcquireTokenAsync(Scopes);
            // FOVFeedback.instance.ModifyText($"UniqueId={authResult.UniqueId}, AccessToken={authResult.AccessToken}");
            authenticated = true;
        }
        catch (MsalException msaEx)
        {
            FOVFeedback.instance.ModifyText($"Failed to acquire token: {msaEx.Message}");
        }
        catch (Exception ex)
        {
            FOVFeedback.instance.ModifyText($"Failed to acquire token: {ex.Message}");
        }


        if (authenticated)
        {
            // Query MS Graph using a HTTP GET
            string GraphUriAndQuery = @"https://graph.microsoft.com/v1.0/users?$filter=startswith(Displayname,'Marty')&$select=id";
            var uri = new System.Uri(GraphUriAndQuery);
            using (HttpClient httpClient = new HttpClient())
            {
                try
                {
                    httpClient.DefaultRequestHeaders.Authorization =
                        new AuthenticationHeaderValue("bearer", authResult.AccessToken);
                    string result = await httpClient.GetStringAsync(uri);

                    Rootobject root = new Rootobject();
                    MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(result));
                    DataContractJsonSerializer ser = new DataContractJsonSerializer(root.GetType());
                    root = ser.ReadObject(ms) as Rootobject;
                    FOVFeedback.instance.ModifyText($"id={root.value[0].id}");
                }
                catch (Exception ex)
                {
                    FOVFeedback.instance.ModifyText($"Failed to query MS Graph: {ex.Message}");
                }
            }
        }
#else
        FOVFeedback.instance.ModifyText(":-( The app does not work in the Editor");
#endif
    }
}


public class Rootobject
{
    public string odatacontext { get; set; }
    public Value[] value { get; set; }
}

public class Value
{
    public string id { get; set; }
}

Note: This code uses FOVFeedback.instance.ModifyText() which is in the repo. Do not hesitate to replace with Debug.Log() if you create the sample from scratch.

via Gfycat

Conclusion

References

• Source code on GitHub - https://github.com/Microsoft/Windows-AppConsult-Samples-MixedReality/tree/master/MixedReality-MSGraph
• 30 Days of Microsoft Graph - https://developer.microsoft.com/en-us/graph/blogs/announcing-30-days-of-microsoft-graph-blog-series
• Build UWP apps with Microsoft Graph - https://docs.microsoft.com/en-us/graph/training/uwp-tutorial
• Graph Explorer - https://developer.microsoft.com/en-us/graph/graph-explorer
• Microsoft Graph and HoloLens: Authentication - https://blogs.msdn.microsoft.com/appconsult/2018/04/12/microsoft-graph-and-hololens-authentication/
• Graph APIs reference - https://docs.microsoft.com/en-us/graph/api/overview?view=graph-rest-1.0
• How to: Serialize and Deserialize JSON Data https://docs.microsoft.com/en-us/dotnet/framework/wcf/feature-details/how-to-serialize-and-deserialize-json-data

For these scenarios, Alexa provides a feature called account linking. When the user enables the skill, he will be requested to authenticate to a 3rd party service using an OAuth 2.0 flow. Once the linking is successful, every request to the skill will be authenticated against the service and Alexa will send you, in the JSON request, the access token which is required to perform further operations.

You can identify skills which require account linking thanks to the message displayed under the Enable button in the Alexa Store.

In this blog post we're going to see how to implement this flow by adding support to Azure Active Directory authentication and the Microsoft Graph to a skill. We'll welcome the user by saying his name and we'll be able to provide information about his OneDrive storage.

Let's start!

The account linking

To setup the account linking you need to select your skill on the Alexa Developer Console and choose Account linking from the menu on the left:

In the first part of the page you'll be able to customize the configuration of the account linking. In order to enable it, you have to turn on the first option Do you allow users to create an account or link to an existing account with you?. This will enable the message you have seen in the previous image in the Alexa Store. The second option can be used if your skill offers some features which can be leveraged also without linking an account. For example, the food delivery skill could offer a command to know the restaurants around you which can be freely invoked; however, as soon as you try to order some food, you'll be asked to link your account. In our scenario, we leave this option turned off. Our skill needs to use the Microsoft Graph, so we aren't able to provide any feature if the user hasn't login with his Microsoft Account.

Then make sure to check Auth Code Grant as authorization grant type, since it's the safest one.

The rest of the page is dedicated to configure the OAuth 2.0 flow, which will be triggered when the user enables the skill and authenticates with the 3rd party service. Let's take a step back and take a look at how the OAuth 2.0 flow works:

The process involves three steps:

1. First, the client application (in our case, the skill) sends an authorization request to the specific endpoint exposed by the service. The user will see a popup or he will be redirected to a different page (in case of a web application), where he will be asked to login with his credentials. This page belongs to the owner of the service. This means that the client application (in our case, neither Amazon nor our skill) will never get access to the credentials of the user.
2. If the authorization request is approved, the application can use the received grant to request an access token. The access token is a unique identifier of the user which, however, doesn't contain any information that can be used to identify his credentials.
3. Once we have the access token, we can now finally access to all the protected resources exposed by the service. The access token will be included as authorization header of every request and, if it's valid, we will receive the resource we have asked.

If we take a look now at the section to configure the OAuth 2.0 flow, we will see how the various fields match the flow we have just highlighted:

• Authorization URI: this is the endpoint exposed by the service to ask the permission to start using it. You send a request with all the details to identify your application and, if approved, you will receive back the authorization code.
• Access Token URI: this is the endpoint to use as next step. After we have been authorized to use the service, we can use the authorization code to send a request to this URI to get back the access token.
• Client ID is the unique identifier of our application.
• Client secret is a password which has been provided by the owner of the service to authenticate our application.
• Scope is a set of values which which identiy the features of the service we want to leverage.

In our scenario, we need to authenticate to the Microsoft Graph using a Microsoft Account or an Office 365 account. Where can we find all the information we need to fill the Account Linking page in the Amazon portal? In order to leverage authentication with a 3d party service, we need to register our application against it. For this reason, all the 3rd party services which provide authentication (Microsoft, Google, Twitter, GitHub, etc.) offer a developer portal that can be used to register an application and to obtain all the credentials we need to implement the authentication flow.

The one offered by Microsoft to register an application which needs to support Azure AD authentication is available in the Azure Portal. As such, the first step is to login to https://portal.azure.com using the account which you want to use to register your application (Microsoft Account or Office 365). Once you are logged into Azure, open the Azure Active Directory section and choose App registrations (Preview):

Let's start to register our application. Click on the New registration button.

First give to your application a unique name. In my case, I chose MyAlexaSkill. Then you must specify which account types you want to support:

• Accounts in the organizational directory only: if you're logged in with an Office 365 account, you will be able to create an application which is authorized to allow the login only for users from your organization. This approach is useful when you're building an internal app, which my be used only by people from your company.
• Accounts in any organizational directory: this option will enable any Office 365 user to login with their account, regardless of the tenant they belong to.
• Accounts in any organizational directory and personal Microsoft Account: this option will enable not only Office 365 users to login, but also users with a personal Microsoft Account.

In our scenario, let's enable the third option. We want to enable every user to use the skill to know how much space they have left on OneDrive, regardless if it's the personal space or the business one.

Ignore, for the moment, the Redirect URI section. We're going to fill it later. Press Register at the end of the section.

Now we can start to fill the various field in the Alexa Developer Portal. Let's see where to retrieve the various parameters:

• Authorization URI and Access Token URI: we can find this information by clicking on the Endpoints button.

From there, we need to copy the two URIs under OAuth 2.0 authorization endpoint (v2) and OAuth 2.0 token endpoint (v2).

• Client Id can be retrieved in the main page under Application (client) ID:

• Client Secret must be generated from the Azure portal. Click on Certificates & secrets in the left panel, then click on New client secret. Give it a description and choose an expiration date. In the context of a skill, you can safely choose Never. This way, you won't have to remember to update, from time to time, the client secret in the Alexa Developer Portal. Once the secret has been generated, copy it in the portal and store it also somewhere safe. The reason is that this is the only time you'll be able to see the client secret in clear. After that, it will be automatically masked and there won't be any way to see it again. You'll be forced to generate a new one, if you lose it.

• Client Authentication Scheme: keep HTTP Basic.
• Scope must match the various scopes that we have enabled in the Azure portal. You can see them by clicking on API permissions in the Azure app configuration. By default, the standard one is User.Read, which allows the application to read the basic information of the user profile, like his name or his mail address. If you want to add more scopes, you can click on Add a permission and explore all the other scopes that are offered by the Azure services or by your custom services. In this case, we're building a skill which integrates with the Microsoft Graph, so you'll find all the available permissions under Microsoft Graph:

Scopes are categorized as Delegated permissions and Application permissions. In our case, we want the skill to access to the API as the signed user, so we can focus on the first category. From there, you'll be able to explore all the available scopes and enable the ones you need. Pay attention that, for some of them, you may see the value Yes in the Admin consent required column. If you're building an enterprise skill which authenticates with an Office 365 account, the administrator of the tenant may block the usage of some scopes for security reasons. In such cases, you will need to reach him to ask to enable the proper permissions.

Once you have defined the scopes you need, you will need to copy them inside the Scope field of the Alexa Developer Portal. If you have multiple scopes, you can click on the + button to add new fields.

The last step requires the opposite process compared to what we have done so far. We'll need to take some info from the Alexa Developer Portal and copy them to the Azure one. In the OAuth 2.0 flow, in fact, once the authentication process is completed the service will forward the response to a specific endpoint exposed by our application. In this case, since it's an Alexa skill, the endpoints are provided directly by Amazon. We can find them at the end of the Account Linking configuration page:

We need to copy them in the Azure Portal, in the Authentication section:

Hit Save to complete the process, then move back to the Alexa Developer Portal and hit Save also there. We're done! Now account linking should have been properly enabled.

This is how the configuration of your Alexa skill should look like:

In order to test the linking, we need to open the Alexa web application, which you can find at https://alexa.amazon.com. Login with your account, move to the Skills section and click on the Your skills button on the top right.

Please note! I apologize if the following screenshots are in Italian, but it seems that there's no way to force Amazon to display the application in English if your account is based in Italy. However, they should be helpful to understand where to look for the various options.

You will find multiple categories, including one called Skills for developers. Here you will find all the skills you have created in the Alexa Developer Portal. Look for the one you're working on and notice how, below the name, you will see now a message informing that account linking is required.

Click on the skill and enable it. If you did everything correctly, another windows or tab of your browser will open up to display the Microsoft Account authentication page. Login with your Microsoft Account or Office 365 account and you should be all set! You should see a confirm message notifying you that your account is now linked to the skill.

Handling the authentication in the Azure Function

Now that we have completed the configuration on the portal, we need to start working on our backend, which is hosted by an Azure Function. We need, in fact, to retrieve the access token and use it to perform operations with the Microsoft Graph. I won't explain from scratch how to build an Alexa Skill hosted by an Azure Function, since we already did it in the previous posts. Here we will focus only on the relevant snippets for handling the authentication process.

Thanks to Alexa.NET it's easy to retrieve the token, since it's stored in the Session.User.AccessToken property of the SkillRequest object, which is the one that maps the JSON coming from Alexa with all the details about the request:

public static async Task<IActionResult> Run([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequest req, ILogger log)
{
    var json = await req.ReadAsStringAsync();
    var skillRequest = JsonConvert.DeserializeObject<SkillRequest>(json);

    var token = skillRequest.Session.User.AccessToken;
    
    ///handle the request and return the response
}

Now that we have the token, we can use it to perform operations against the Microsoft Graph. The Graph is nothing more than a set of REST endpoints, which you can call by sending HTTP commands. Based on the type of operation, it could be a GET, a POST, a PUT, etc. The easiest way to understand the opportunities offered by the Microsoft Graph is to use Graph Explorer, which is a sort of playground. You can login with your Microsoft Account or Office 365 account and then start making request to the APIs. The tool will offer you the opportunity to compose requests and to observe the JSON response, other than discovering all the available endpoints.

However, we are not required to perform manual HTTP requests in order to use the Graph. Microsoft, in fact, provides multiple SDKs which encapsulate all the logic to work with the Graph, similarly to how Alexa.NET abstracts all the JSON requests and responses and allows you to work with classes and objects. Of course, we have also a SDK for .NET, so we can easily integrate it in our Azure Function.

Right click on your project in Visual Studio, choose Manage NuGet packages and install the package called Microsoft.Graph.

The object that we can use to perform operations against the Graph is called GraphServiceClient, which is included in the Microsoft.Graph namespace. However, we can't use it as it is. We need to use an authenticated client, so we will need to supply the access token included in the request.

Here is a simple setup method that we can add to our Azure Function:

public static GraphServiceClient GetAuthenticatedClientForUser(string token, ILogger logger)
{
   GraphServiceClient graphClient = null;

   // Create Microsoft Graph client.
   try
   {
       graphClient = new GraphServiceClient(
           "https://graph.microsoft.com/v1.0",
           new DelegateAuthenticationProvider(
               (requestMessage) =>
               {
                   requestMessage.Headers.Authorization = new AuthenticationHeaderValue("bearer", token);
                   return Task.CompletedTask;
               }));

       return graphClient;

   }
   catch (Exception exc)
   {
       logger.LogError(exc, "Could not create a graph client");
   }

   return graphClient;
}

We create a new instance of the GraphServiceClient object, passing as parameter:

1. The endpoint we want to use for the Graph. We use the production one, which is https://graph.microsoft.com/v1.0. There's also a preview one, which provides access to all the beta APIs, and it's available at https://graph.microsoft.com/beta.
2. A DelegateAuthenticationProvider, which is the delegate used to handle the authentication. The delegate provides a reference to the HTTP request which is sent to the Microsoft Graph, that we need to customize by supplying the access token in the authorization header.

Now that we have an authenticated client, we can use it to perform operations with the Graph. For example, let's customize the welcome message of the skill by adding the name of the user. We do this in case the incoming request type is LaunchRequest:

public static async Task<IActionResult> Run([HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = null)] HttpRequest req, ILogger log)
{
   var json = await req.ReadAsStringAsync();
   var skillRequest = JsonConvert.DeserializeObject<SkillRequest>(json);

   // Verifies that the request is indeed coming from Alexa.
   var isValid = await skillRequest.ValidateRequestAsync(req, log);
   if (!isValid)
   {
       return new BadRequestResult();
   }

   var request = skillRequest.Request;
   SkillResponse response = null;

   var token = skillRequest.Session.User.AccessToken;
   var client = GetAuthenticatedClientForUser(token, log);

   if (request is LaunchRequest launchRequest)
   {

       var me = await client.Me.Request().GetAsync();

       var welcomeMessage = $"Welcome {me.DisplayName}";
       response = ResponseBuilder.Tell(welcomeMessage);
   }
   
    return new OkObjectResult(response);
}

The client simply maps all the endpoints that you can see in the Graph Explorer as objects. As such, in order to get the basic profile of the user (which is mapped with the endpoint https://graph.microsoft.com/v1.0/me/), we can use the Me property exposed by the client. To actually perform the request we need to call the Request() method, followed by the HTTP method we need to use. In this case, the me endpoint replies to a HTTP GET, so we call the GetAsync() method. We get in return an object which maps all the properties that we can see in the response JSON in Graph Explorer. For our scenario, we retrieve the name using the DisplayName property and we add it to the welcome message. The rest of the code is the same we've seen in the other posts: we use the ResponseBuilder class to create a response and then we send it back as response to the Alexa service which invoked our function.

Now we that we have seen how to handle the authentication in the backend, the sky is the limit 😃 For example, if we follow our original idea of integrating OneDrive with our Alexa skill, we can create a new intent in the Interaction Model with name Quota and then, in the function, handle it with the help of the Graph client:

if (intentRequest.Intent.Name == "Quota")
{
    var drive = await graphClient.Me.Drive.Request().GetAsync();
    int free = (int)(drive.Quota.Remaining.Value / 1024 / 1024 / 1024);

    var quotaMessage = $"You have {free.ToString()} GB available";
    response = ResponseBuilder.Tell(quotaMessage);
}

The structure of the code is the same as the previous example. If we use Graph Explorer, we can see how the information about the OneDrive storage of the user is available through the https://graph.microsoft.com/v1.0/me/drive endpoint. This endpoint is mapped by the property Me.Drive, which returns a Drive object that contains all the properties of the storage. Thanks to the Quota property we can get the information we're looking for, which is the available space, and we use it to build a response for the user.

Wrapping up

In this post we have seen how to enable account linking for an Alexa Skill, allowing it to identify the user and to provide customized responses. In this specific example we have leveraged AAD authentication and the Microsoft Graph, but any service which supports OAuth 2.0 for the authentication flow is supported. You could integrate your Alexa skill, for example, with Twitter, GitHub, Facebook, etc.

Thanks to the OAuth 2.0 configuration provided in the Alexa Developer Portal, you don't have to handle the authentication flow on your own, by manually communicating with the various authentication and access token endpoints. You just have to provide to Amazon the configuration of your service and, automatically, you will receive the access token in the body of the request which is delivered to your skill.

You can find the full OneDrive sample on GitHub at https://github.com/Microsoft/Windows-AppConsult-samples-PWA/tree/master/OneDriveAlexaSkill. Of course, you will find only the code of the backend Azure Function. It's up to you to create the most appropriate interaction model and to setup an Azure AD application to support the authentication.

A huge thanks to my friend Marco Minerva, who is sharing with me lot of interesting activities around Alexa skills and who helped me in setting up Account Linking.

Happy coding!

1. The Why

2. The Basics

2.1 The big picture

2.2 Supervised Learning

2.3 Neural Networks

2.4 Convolutional Neural Networks

2.5 Testing the prediction

private bool IsCat(byte[] image) 
{
    return true;
}

1. True positive: a cat picture is correctly identified as cat
2. False positive: a dog picture is mistakenly labeled as cat
3. True negative: a dog picture is correctly identified as not-a-cat
4. False negative: a cat picture is mistakenly identified as not-a-cat

3 The Implementation

public IEnumerator AnalyseCapturedImage(byte[] imageBytes)
{
    using (var unityWebRequest = UnityWebRequest.Post(endpoint, new WWWForm()))
    {
        unityWebRequest.SetRequestHeader("Content-Type", contentType);
        unityWebRequest.SetRequestHeader(headerKey, predictionKey);
        unityWebRequest.downloadHandler = new DownloadHandlerBuffer();
        unityWebRequest.uploadHandler = new UploadHandlerRaw(imageBytes) { contentType = contentType };
        yield return unityWebRequest.SendWebRequest();

        try
        {
            var jsonResponse = unityWebRequest.downloadHandler.text;
            var analysedObject = JsonUtility.FromJson<AnalysedObject>(jsonResponse);
            
            //you can now find the prediction result in analysedObject.predictions
        }
        catch (Exception e)
        {
            // exception handling
        }
    }
}

Welcome Visual Studio 2017 15.9

dotnet new wpf

<Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">

  <PropertyGroup>
    <OutputType>WinExe</OutputType>
    <TargetFramework>netcoreapp3.0</TargetFramework>
    <UseWPF>true</UseWPF>
  </PropertyGroup>

</Project>

<Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">

  <PropertyGroup>
    <OutputType>WinExe</OutputType>
    <TargetFramework>netcoreapp3.0</TargetFramework>
    <UseWpf>true</UseWpf>
  </PropertyGroup>

  <ItemGroup>
    <Content Include="Watermark.png">
      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
    </Content>
  </ItemGroup>
</Project>

• We have changed the Sdk name in the Project entry from Microsoft.NET.Sdk.Wpf to Microsoft.NET.Sdk.WindowsDesktop. Now, in fact, the SDK is unique regardless of the platform's choice (WPF or Windows Forms).
• We have added the UseWpf entry and we have set it to true. This is the new approach to specify the application's platform, in this case WPF.
• We have removed all the previous entries inside the ItemGroup section, like ApplicationDefinition or Page. The only exception is for the Watermark.png file, since we need to customize the build action and to set it as Content.

dotnet build

dotnet run

Wrapping up

Note: My colleagues Etienne Margraff and Matteo Pagani already gave lot of details and useful explanations about Windows ML. Feel free to refer to their good blog posts in the References section at the end of this article.

• We use a Custom Vision model that we train in Azure to recognize objects/situations/machines/persons/etc. (In fact, almost what you want 😏).
• We export this model as a ONNX 1.2 file.
• We develop a HoloLens app which gets video frames in continue.

Step 1. Train 🥼 a Machine Learning Model in Azure

🆕 Create the Custom Vision model

• A name and a description.

• Choose if you want to use the Limited Trial mode or use/create an Azure resource group for hosting the Custom Vision service in Azure.

Note: Choosing the resource group option makes sense if you want to use your Custom Vision model everywhere. I mean: it uses Azure to provide a REST endpoint API that can be used in mobile apps or on computers running Windows, MacOS, Linux or even in Web sites. And all this, without to worry about the technology used. For evaluating the model locally on the HoloLens, you can go for the Limited Trial as we will not use the Azure Custom Vision endpoint. Please note that, later, if you want to instantiate an Azure resource for this Custom Vision model's evaluation on other platforms, you can do it by clicking on the gear on the top right of the page: Then, at the bottom of the details' page for your model, you create the Azure resource by clicking on the Move To Azure button:

• Choose Classification for the project type. That means that we would like to tag the whole image.

Note: For your information, Object Detection finds also the location of the object within an image.

• Go for Multilabel (Multiple tags per image) as we would like to not be blocked to use only one tag. As an example: The image can contain a machine factory B214 type 2 and another machine whom id is A78 Alpha2. Tags applied to the image will be "B214 type 2" and "A78 Alpha2".

• Lastly, but may be the most important, ⚠ take General (compact) ⚠. Choosing a compact model is mandatory to be able to use it with Windows ML on HoloLens or in a Windows 10 UWP app.

⚗ Train the model

1. Just upload photos using the Add images button.

2. Give one or more tags to the images to upload.

1. You can change the Probability Threshold for a prediction to be valid. For example: 85% means that any predictions whom score < 85% will not be considered as a valid prediction.
2. Precision = Gives you the % of correct predictions. For example: For Benoit, we have 7 tagged pictures and the model's evaluation predicted this tag for all 7 pictures (so 100%).
3. Recall = Gives you the % of found tags. For example: For Sebastien, we have 12 tagged pictures and the model was able to predict this tag for 91,7% of the pictures (11 pictures).

Step 2. Export 📦 the model as ONNX 1.2

Note: As I write this article, the Windows version 1809 can be installed on HoloLens by participating to the Windows Insider Preview Slow ring.

Step 3. Create a Unity App 🕹 for HoloLens that captures the camera 🎥 frames

Project setup 📏

• Windows version 1809 for the PC and HoloLens
• Visual Studio 2017 version 15.9.0
• Windows SDK version 10.0.17763.0
• Unity 2017.4.13f1

Note: You can skip this part if you are familiar with Unity and Mixed Reality. If not, please refer to Create a Windows Mixed Reality App from 0 – The Unity project.

• File menu / Build Settings ➡ Universal Windows Platform selected as the Build platform
• Edit menu / Project Settings / Player ➡ Virtual Reality Supported checked
• Edit menu / Project Settings / Player ➡ Other Settings / .NET as Scripting Backend. Please also use the Experimental (.NET 4.6 Equivalent) for the Scripting Runtime Version.

Technical piping 🔩 for being able to give update to the user

• We use an interface IUnityScanScene with one unique public method ModifyOutputText.
• The startup script we are modifying (UnityScanScene) implements this interface. The method ModifyOutputText just modifies a string.
• The Update method of the script (UnityScanScene) is executed for each frame. We just modify the text of a 3D Text object based on the modified string.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

/// <summary>
/// The Interface gives one method to implement. This method modifies the text to display in Unity
/// Any code can then call this method outside the Unity MonoBehavior object
/// </summary>
public interface IUnityScanScene
{
	void ModifyOutputText(string newText);
}

public class UnityScanScene : MonoBehaviour, IUnityScanScene
{
	// Unity 3D Text object that contains 
	// the displayed TextMesh in the FOV
	public GameObject OutputText;
	// TextMesh object provided by the OutputText game object
	private TextMesh OutputTextMesh;
	// string to be affected to the TextMesh object
	private string OutputTextString = string.Empty;
	// Indicate if we have to Update the text displayed
	private bool OutputTextChanged = false;

	// Use this for initialization
	async void Start ()
	{
            OutputTextMesh = OutputText.GetComponent<TextMesh>();
            OutputTextMesh.text = string.Empty;

#if UNITY_WSA && !UNITY_EDITOR // **RUNNING ON WINDOWS**
            // We will create our Camera scan engine here
#else                          // **RUNNING IN UNITY**
            ModifyOutputText("Sorry ;-( The app is not supported in the Unity player.");
#endif
	}

	/// <summary>
	/// Modify the text to be displayed in the FOV
	/// or/and in the debug traces
	/// + Indicate that we have to update the text to display
	/// </summary>
	/// <param name="newText">new string value to display</param>
	public void ModifyOutputText(string newText)
	{
		OutputTextString = newText;
		OutputTextChanged = true;
	}

	// Update is called once per frame
	void Update ()
	{
		if (OutputTextChanged)
		{
			OutputTextMesh.text = OutputTextString;
			OutputTextChanged = false;
		}
	}

}

Display 📺 some text in the field of view

Camera scan engine ⚙ 🎥

#if UNITY_WSA && !UNITY_EDITOR
using System;
using System.Linq;
using System.Threading.Tasks;
using Windows.Media.Capture;
using Windows.Media.Capture.Frames;
using Windows.Media.MediaProperties;

public class ScanEngine
{
    public TimeSpan PredictionFrequency = TimeSpan.FromMilliseconds(400);

    private MediaCapture CameraCapture;
    private MediaFrameReader CameraFrameReader;

    private Int64 FramesCaptured;

    IUnityScanScene UnityApp;


    public ScanEngine()
    { 
    }

    public async Task Inititalize(IUnityScanScene unityApp)
    {
        UnityApp = unityApp;

        await InitializeCameraCapture();
        await InitializeCameraFrameReader();
    }

    private async Task InitializeCameraCapture()
    {
        CameraCapture = new MediaCapture();
        MediaCaptureInitializationSettings settings = new MediaCaptureInitializationSettings();
        settings.StreamingCaptureMode = StreamingCaptureMode.Video;
        await CameraCapture.InitializeAsync(settings);
    }

    private async Task InitializeCameraFrameReader()
    {
        var frameSourceGroups = await MediaFrameSourceGroup.FindAllAsync(); 
        MediaFrameSourceGroup selectedGroup = null;
        MediaFrameSourceInfo colorSourceInfo = null;

        foreach (var sourceGroup in frameSourceGroups)
        {
            foreach (var sourceInfo in sourceGroup.SourceInfos)
            {
                if (sourceInfo.MediaStreamType == MediaStreamType.VideoPreview
                    && sourceInfo.SourceKind == MediaFrameSourceKind.Color)
                {
                    colorSourceInfo = sourceInfo;
                    break;
                }
            }
            if (colorSourceInfo != null)
            {
                selectedGroup = sourceGroup;
                break;
            }
        }

        var colorFrameSource = CameraCapture.FrameSources[colorSourceInfo.Id];
        var preferredFormat = colorFrameSource.SupportedFormats.Where(format =>
        {
            return format.Subtype == MediaEncodingSubtypes.Argb32;

        }).FirstOrDefault();

        CameraFrameReader = await CameraCapture.CreateFrameReaderAsync(colorFrameSource);
        await CameraFrameReader.StartAsync();
    }

    public void StartPullCameraFrames()
    {
        Task.Run(async () =>
        {
            for (; ; ) // Forever = While the app runs
            {
                FramesCaptured++;
                await Task.Delay(PredictionFrequency);
                using (var frameReference = CameraFrameReader.TryAcquireLatestFrame())
                using (var videoFrame = frameReference?.VideoMediaFrame?.GetVideoFrame())
                {

                    if (videoFrame == null)
                    {
                        continue; //ignoring frame
                    }

                    if (videoFrame.Direct3DSurface == null)
                    {
                        videoFrame.Dispose();
                        continue; //ignoring frame
                    }

                    try
                    {
                        // HERE, We will evaluate the model
                    }
                    catch (Exception ex)
                    {
                        System.Diagnostics.Debug.WriteLine(ex.Message);
                    }
                    finally
                    {
                    }
                }

            }

        });
    }

}
#endif

	async void Start ()
	{
        OutputTextMesh = OutputText.GetComponent<TextMesh>();
        OutputTextMesh.text = string.Empty;

#if UNITY_WSA && !UNITY_EDITOR // RUNNING ON WINDOWS**
        CameraScanEngine = new ScanEngine();       // <-- 1
        await CameraScanEngine.Inititalize(this);  // <-- 2
        CameraScanEngine.StartPullCameraFrames();  // <-- 3
#else                          // RUNNING IN UNITY**
        ModifyOutputText("Sorry ;-( The app is not supported in the Unity player.");
#endif
	}

1. We create our Camera helper CameraScanEngine from the ScanEngine class.
2. Calling CameraScanEngine.Inititalize(this) in the UnityScanScene.cs startup script gives the ability to call the public ModifyOutputText method of the IUnityScanScene interface from the CameraScanEngine object. Here is the proof:

3. The HoloLens camera is initialized and we can get the frames. The StartPullCameraFrames() call initiates an infinite loop for pulling for frames. We will replace the line // HERE, We will evaluate the model by the code for evaluating the model on the frame later.

Step 4. Integrate ⤵ the model in the app

Adding the file in the package 📥

Note: It is also possible to add the ONNX file directly to the resulting C# UWP application. The Unity way of adding it is not mandatory.

Add a wrapper class 🔧 for the model

#if UNITY_WSA && !UNITY_EDITOR
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
using Windows.AI.MachineLearning;
using Windows.Media;
using Windows.Storage;

public sealed class ONNXModelInput
{
    public VideoFrame Data { get; set; }
}

public sealed class ONNXModelOutput
{
    public TensorString ClassLabel = TensorString.Create(new long[] { 1, 1 });
    public IList<IDictionary<string, float>> Loss = new List<IDictionary<string, float>>();
}

/// <summary>
/// These classes ONNXModel, ONNXModelInput, ONNXModelOutput never change
/// We just load a different custom model
/// 
/// In order to evalutate a model:
/// 1. Call the static method ONNXModel.CreateOnnxModel(<<MyModelStorageFile>>)
/// 2. Call EvaluateAsync method on the created Model before
/// </summary>
public sealed class ONNXModel
{
    private LearningModel _learningModel = null;
    private LearningModelSession _session;

    public static async Task<ONNXModel> CreateOnnxModel(StorageFile file)
    {
        LearningModel learningModel = null;

        try
        {
            learningModel = await LearningModel.LoadFromStorageFileAsync(file);
        }
        catch (Exception e)
        {
            var exceptionStr = e.ToString();
            Debug.WriteLine(exceptionStr);
            throw e;
        }
        return new ONNXModel()
        {
            _learningModel = learningModel,
            _session = new LearningModelSession(learningModel)
        };
    }

    public async Task<ONNXModelOutput> EvaluateAsync(ONNXModelInput input)
    {
        var output = new ONNXModelOutput();
        var binding = new LearningModelBinding(_session);
        binding.Bind("data", input.Data);
        binding.Bind("classLabel", output.ClassLabel);
        binding.Bind("loss", output.Loss);
        LearningModelEvaluationResult evalResult = await _session.EvaluateAsync(binding, "0");

        return output;
    }
}
#endif

Note: All code dedicated to UWP is surrounded by #if UNITY_WSA && !UNITY_EDITOR. It allows us to be sure to run this code only on Windows.

Load 📤 and evaluate ⚙ the model for each frame

#if UNITY_WSA && !UNITY_EDITOR
using System;
using System.Diagnostics;
using System.Threading.Tasks;
using Windows.Media;
using Windows.Storage;

public class ONNXModelHelper
{
    private ONNXModel Model = null;
    private string ModelFilename = "ONNXModel.onnx";
    private Stopwatch TimeRecorder = new Stopwatch();
    private IUnityScanScene UnityApp;

    public ONNXModelHelper()
    {
        UnityApp = null;
    }

    public ONNXModelHelper(IUnityScanScene unityApp)
    {
        UnityApp = unityApp;
    }

    public async Task LoadModelAsync() // <-- 1
    {
        ModifyText($"Loading {ModelFilename}... Patience");

        try
        {
            TimeRecorder = Stopwatch.StartNew();

            var modelFile = await StorageFile.GetFileFromApplicationUriAsync(
                new Uri($"ms-appx:///Data/StreamingAssets/{ModelFilename}"));
            Model = await ONNXModel.CreateOnnxModel(modelFile);

            TimeRecorder.Stop();
            ModifyText($"Loaded {ModelFilename}: Elapsed time: {TimeRecorder.ElapsedMilliseconds} ms");
        }
        catch (Exception ex)
        {
            ModifyText($"error: {ex.Message}");
            Model = null;
        }
    }

    public async Task EvaluateVideoFrameAsync(VideoFrame frame) // <-- 2
    {
        if (frame != null)
        {
            try
            {
                TimeRecorder.Restart();
                ONNXModelInput inputData = new ONNXModelInput();
                inputData.Data = frame;
                var output = await Model.EvaluateAsync(inputData).ConfigureAwait(false); // <-- 3

                var product = output.ClassLabel.GetAsVectorView()[0]; // <-- 4 
                var loss = output.Loss[0][product]; // <-- 5
                TimeRecorder.Stop();

                var lossStr = string.Join(product, " " + (loss * 100.0f).ToString("#0.00") + "%");
                string message = $"({DateTime.Now.Hour}:{DateTime.Now.Minute}:{DateTime.Now.Second})" +
                    $" Evaluation took {TimeRecorder.ElapsedMilliseconds}ms\n";

                string prediction = $"Prediction: {product} {lossStr}";
                if (loss > 0.5f)
                {
                    message += prediction;
                }

                message = message.Replace("\\n", "\n");

                ModifyText(message); // <-- 6
            }
            catch (Exception ex)
            {
                var err_message = $"error: {ex.Message}";
                ModifyText(err_message);
            }
        }
    }

    private void ModifyText(string text)
    {
        System.Diagnostics.Debug.WriteLine(text);
        if (UnityApp != null)
        {
            UnityApp.ModifyOutputText(text);
        }
    }
}
#endif

1. The helper method effectively loads the ONNX Model using StorageFile.GetFileFromApplicationUriAsync().
2. EvaluateVideoFrameAsync() is called by the ScanEngine for every frame.

3. Model.EvaluateAsync() calls the wrapper method for the model's evaluation. We get the result in the output object.
4. var product = output.ClassLabel.GetAsVectorView()[0] is the name of the most scored prediction.
5. output.Loss[0][product] is the score of this prediction.
6. We call the private method ModifyText to modify the text to display in HoloLens thanks to UnityApp.ModifyOutputText(). The display is about giving the predication and the score.

Disclaimer ⚖: Again, this model was only trained with less than 20 pictures which is a way too low to pretend having a correct model for prediction. But you have the idea!

Conclusion

...the second version of the HPU, currently under development, will incorporate an AI coprocessor to natively and flexibly implement DNNs.

Mixed reality and artificial intelligence represent the future of computing, and we’re excited to be advancing this frontier.

Credits

• Etienne Margraff for the first article on this topic on HoloLens/
• Matteo Pagani for his articles and reviewing this one.
• Benoit my co-presenter on Mixed Reality events
• Florent for his support

References

• All source code - https://github.com/Microsoft/Windows-AppConsult-Samples-MixedReality/tree/master/HoloLensWinML
• Custom Vision - https://www.customvision.ai/
• Windows Machine Learning - https://docs.microsoft.com/fr-fr/windows/ai/
• Second version of HoloLens HPU will incorporate AI coprocessor for implementing DNNs - https://www.microsoft.com/en-us/research/blog/second-version-hololens-hpu-will-incorporate-ai-coprocessor-implementing-dnns/
• Open Neural Network Exchange Github - https://github.com/onnx/onnx
• ONNX Format initiative - https://onnx.ai/
• (Preview API) Experimenting with Windows Machine Learning and Mixed Reality - https://meulta.com/en/2018/05/18/experimenting-with-windows-machine-learning-and-mixed-reality/
• (Preview API) Add a bit of machine learning to your Windows application thanks to WinML - https://blogs.msdn.microsoft.com/appconsult/2018/05/23/add-a-bit-of-machine-learning-to-your-windows-application-thanks-to-winml/
• Upgrade your WinML application to the latest bits - https://blogs.msdn.microsoft.com/appconsult/2018/11/06/upgrade-your-winml-application-to-the-latest-bits/
• Integrate a model into your app with Windows ML - https://docs.microsoft.com/fr-fr/windows/ai/integrate-model

Dark Mode in File Explorer

Notepad enhancements

The new clipboard

Registry autocomplete