MR and Azure 304:顔認識

続いてUnityの設定を行っていきます。以下の手順で進めていきましょう。

1.Unityを開き、[New]から新しいプロジェクトを作成します。

名前をFaceRecognitionとして、[Create project]をクリックします。

1. [Build Settings..]からさまざまな項目を編集します。

[File]->[Build Settings..]を開きます。

a.プラットフォームの変更

[PC, Mac & Linux Standalone]を[Universal Windows Platform]に変更し、[Switch Platform]をクリックします

b.[Player Settings..]を編集します。

[Player Settings..]をクリックします。

そのあと[Other Settings]、[Publishing Settings]->[Capabilities]、[XR Settings]を以下のように設定します。

続いて、[Unity C#]にチェックを入れます。

[Build Settings..]の変更は以上です。

1. 続いてシーンを保存します。[Add Open Scenes]をクリックして、[新しいフォルダー]を選択しScenesと名前をつけた後、そのフォルダー内に名前をSceneとして[保存]します。

2.シーンの編集

ここでは以下の2つのスクリプトを作ります。

・FaceAnalysis

・ImageCaptureFace

スクリプトの作り方はまず、すべてのスクリプトをまとめておくフォルダーを作成します。[Project]->[Create]をクリックし、[Folder]を選択して新しいフォルダーを作ります。名前をScriptsとします。続いて[Project]->[Create]->[C# Script]をクリックし、クラス名をつけていきます。

このスクリプトはAzure Face Recognition Serviceとやりとりしたり、データをシリアライズ、デシリアライズしたりするためのクラスやメソッドが記述されています。ここで0.準備で取得したFace APIの鍵と入力したPerson Group IDを使います。–Insert your key–、–Insert your ID–へそれぞれ挿入します。コードは以下の通りです。

using Newtonsoft.Json;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using UnityEngine;
using UnityEngine.Networking;

public class FaceAnalysis : MonoBehaviour
{

/// <summary>
/// Allows this class to behave like a singleton
/// </summary>
public static FaceAnalysis Instance;

/// <summary>
/// The analysis result text
/// </summary>
private TextMesh labelText;

/// <summary>
/// Bytes of the image captured with camera
/// </summary>
internal byte[] imageBytes;

/// <summary>
/// Path of the image captured with camera
/// </summary>
internal string imagePath;

/// <summary>
/// Base endpoint of Face Recognition Service
/// </summary>
const string baseEndpoint = "https://westus.api.cognitive.microsoft.com/face/v1.0/";

/// <summary>
/// Auth key of Face Recognition Service
/// </summary>
private const string key = "--Insert your key--";

/// <summary>
/// Id (name) of the created person group
/// </summary>
private const string personGroupId = "--Insert your ID--";

/// <summary>
/// Initialises this class
/// </summary>
private void Awake()
{
// Allows this instance to behave like a singleton
Instance = this;

// Add the ImageCapture Class to this Game Object
gameObject.AddComponent<ImageCaptureFace>();

// Create the text label in the scene
CreateLabel();
Debug.Log("createLabel");
}

/// <summary>
/// Spawns cursor for the Main Camera
/// </summary>
private void CreateLabel()
{
// Create a sphere as new cursor
GameObject newLabel = new GameObject();

// Attach the label to the Main Camera
newLabel.transform.parent = gameObject.transform;

// Resize and position the new cursor
newLabel.transform.localScale = new Vector3(0.4f, 0.4f, 0.4f);
newLabel.transform.position = new Vector3(0f, 3f, 60f);

// Creating the text of the Label
labelText = newLabel.AddComponent<TextMesh>();
labelText.anchor = TextAnchor.MiddleCenter;
labelText.alignment = TextAlignment.Center;
labelText.tabSize = 4;
labelText.fontSize = 50;
labelText.text = ".";
}

/// <summary>
/// Detect faces from a submitted image
/// </summary>
internal IEnumerator DetectFacesFromImage()
{
WWWForm webForm = new WWWForm();
string detectFacesEndpoint = $"{baseEndpoint}detect";

// Change the image into a bytes array
imageBytes = GetImageAsByteArray(imagePath);

using (UnityWebRequest www =
UnityWebRequest.Post(detectFacesEndpoint, webForm))
{
www.SetRequestHeader("Ocp-Apim-Subscription-Key", key);
www.SetRequestHeader("Content-Type", "application/octet-stream");
www.uploadHandler.contentType = "application/octet-stream";
www.uploadHandler = new UploadHandlerRaw(imageBytes);
www.downloadHandler = new DownloadHandlerBuffer();

yield return www.SendWebRequest();
string jsonResponse = www.downloadHandler.text;
FaceRootObject[] faceRootObject =
JsonConvert.DeserializeObject<Face_RootObject[]>(jsonResponse);

List<string> facesIdList = new List<string>();
// Create a list with the face Ids of faces detected in image
foreach (FaceRootObject faceRO in faceRootObject)
{
facesIdList.Add(faceRO.faceId);
Debug.Log($"Detected face - Id: {faceRO.faceId}");
}

StartCoroutine(IdentifyFaces(facesIdList));
}
}

/// <summary>
/// Returns the contents of the specified file as a byte array.
/// </summary>
static byte[] GetImageAsByteArray(string imageFilePath)
{
FileStream fileStream = new FileStream(imageFilePath, FileMode.Open, FileAccess.Read);
BinaryReader binaryReader = new BinaryReader(fileStream);
return binaryReader.ReadBytes((int)fileStream.Length);
}

/// <summary>
/// Identify the faces found in the image within the person group
/// </summary>
internal IEnumerator IdentifyFaces(List<string> listOfFacesIdToIdentify)
{
// Create the object hosting the faces to identify
FacesToIdentifyRootObject facesToIdentify = new FacesToIdentifyRootObject();
facesToIdentify.faceIds = new List<string>();
facesToIdentify.personGroupId = personGroupId;
foreach (string facesId in listOfFacesIdToIdentify)
{
facesToIdentify.faceIds.Add(facesId);
}
facesToIdentify.maxNumOfCandidatesReturned = 1;
facesToIdentify.confidenceThreshold = 0.5;

// Serialise to Json format
string facesToIdentifyJson = JsonConvert.SerializeObject(facesToIdentify);
// Change the object into a bytes array
byte[] facesData = Encoding.UTF8.GetBytes(facesToIdentifyJson);

WWWForm webForm = new WWWForm();
string detectFacesEndpoint = $"{baseEndpoint}identify";

using (UnityWebRequest www = UnityWebRequest.Post(detectFacesEndpoint, webForm))
{
www.SetRequestHeader("Ocp-Apim-Subscription-Key", key);
www.SetRequestHeader("Content-Type", "application/json");
www.uploadHandler.contentType = "application/json";
www.uploadHandler = new UploadHandlerRaw(facesData);
www.downloadHandler = new DownloadHandlerBuffer();

yield return www.SendWebRequest();
string jsonResponse = www.downloadHandler.text;
Debug.Log($"Get Person - jsonResponse: {jsonResponse}");

CandidateRootObject[] candidateRootObject = JsonConvert.DeserializeObject<Candidate_RootObject[]>(jsonResponse);

// For each face to identify that ahs been submitted, display its candidate
foreach (CandidateRootObject candidateRO in candidateRootObject)
{
StartCoroutine(GetPerson(candidateRO.candidates[0].personId));

// Delay the next "GetPerson" call, so all faces candidate are displayed properly
yield return new WaitForSeconds(3);
}
}
}

/// <summary>
/// Provided a personId, retrieve the person name associated with it
/// </summary>
internal IEnumerator GetPerson(string personId)
{
string getGroupEndpoint = $"{baseEndpoint}persongroups/{personGroupId}/persons/{personId}?";
WWWForm webForm = new WWWForm();

using (UnityWebRequest www = UnityWebRequest.Get(getGroupEndpoint))
{
www.SetRequestHeader("Ocp-Apim-Subscription-Key", key);
www.downloadHandler = new DownloadHandlerBuffer();
yield return www.SendWebRequest();
string jsonResponse = www.downloadHandler.text;

Debug.Log($"Get Person - jsonResponse: {jsonResponse}");

IdentifiedPersonRootObject identifiedPersonRootObject = JsonConvert.DeserializeObject<IdentifiedPerson_RootObject>(jsonResponse);

// Display the name of the person in the UI
labelText.text = identifiedPerson_RootObject.name;
}
}

/// <summary>
/// The Person Group object
/// </summary>
public class Group_RootObject
{
public string personGroupId { get; set; }
public string name { get; set; }
public object userData { get; set; }
}

/// <summary>
/// The Person Face object
/// </summary>
public class Face_RootObject
{
public string faceId { get; set; }
}

/// <summary>
/// Collection of faces that needs to be identified
/// </summary>
public class FacesToIdentify_RootObject
{
public string personGroupId { get; set; }
public List<string> faceIds { get; set; }
public int maxNumOfCandidatesReturned { get; set; }
public double confidenceThreshold { get; set; }
}

/// <summary>
/// Collection of Candidates for the face
/// </summary>
public class Candidate_RootObject
{
public string faceId { get; set; }
public List<Candidate> candidates { get; set; }
}

public class Candidate
{
public string personId { get; set; }
public double confidence { get; set; }
}

/// <summary>
/// Name and Id of the identified Person
/// </summary>
public class IdentifiedPerson_RootObject
{
public string personId { get; set; }
public string name { get; set; }
}
}

記述し終わったら、スクリプトを[Hierarchy]->[MixedRealityCameraParent]->[MixedRealityCamera]へD&Dします。

このクラスにはtapジェスチャーやキーボード上の操作を利用して認識を始めるためのメソッドや画像の情報などを利用するためのメソッドなどが記述されています。コードは以下の通りです。

using System.IO;
using System.Linq;
using UnityEngine;
using System.Collections;
using System.Collections.Generic;
using UnityEngine.XR.WSA.Input;
using UnityEngine.XR.WSA.WebCam;
using HoloToolkit.Unity.InputModule;

public class ImageCaptureFace : MonoBehaviour//, IInputClickHandler
{

/// <summary>
/// Allows this class to behave like a singleton
/// </summary>
public static ImageCaptureFace instance;

/// <summary>
/// Keeps track of tapCounts to name the captured images
/// </summary>
private int tapsCount;

/// <summary>
/// PhotoCapture object used to capture images on HoloLens
/// </summary>
private PhotoCapture photoCaptureObject = null;

/// <summary>
/// HoloLens class to capture user gestures
/// </summary>
//private GestureRecognizer recognizer;

/// <summary>
/// Initialises this class
/// </summary>
private void Awake()
{
instance = this;
Debug.Log("awake imagecapture");
}

/// <summary>
/// Called right after Awake
/// </summary>
void Start()
{
// Initialises user gestures capture
//recognizer = new GestureRecognizer();
//recognizer.SetRecognizableGestures(GestureSettings.Tap);
//recognizer.Tapped += TapHandler;
//recognizer.StartCapturingGestures();
InputManager.Instance.PushFallbackInputHandler(gameObject);
Debug.Log("start");
}

/// <summary>
/// Respond to Tap Input.
/// </summary>
//private void TapHandler(TappedEventArgs obj)
//{
// tapsCount++;
// ExecuteImageCaptureAndAnalysis();
//}

/*public void OnInputClicked(InputClickedEventData eventData)
{
tapsCount++;
Debug.Log("call analisis");
ExecuteImageCaptureAndAnalysis();
}*/

void Update()
{
if (Input.GetKey(KeyCode.Return))
{

tapsCount++;
Debug.Log("call analisis");
ExecuteImageCaptureAndAnalysis();
}
}
/// <summary>
/// Begin process of Image Capturing and send To Azure Computer Vision service.
/// </summary>
private void ExecuteImageCaptureAndAnalysis()
{
Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending
((res) => res.width * res.height).First();
Texture2D targetTexture = new Texture2D(cameraResolution.width, cameraResolution.height);

PhotoCapture.CreateAsync(false, delegate (PhotoCapture captureObject)
{
photoCaptureObject = captureObject;

CameraParameters c = new CameraParameters();
c.hologramOpacity = 0.0f;
c.cameraResolutionWidth = targetTexture.width;
c.cameraResolutionHeight = targetTexture.height;
c.pixelFormat = CapturePixelFormat.BGRA32;

captureObject.StartPhotoModeAsync(c, delegate (PhotoCapture.PhotoCaptureResult result)
{
string filename = string.Format(@"CapturedImage{0}.jpg", tapsCount);
string filePath = Path.Combine(Application.persistentDataPath, filename);

// Set the image path on the FaceAnalysis class
FaceAnalysis.Instance.imagePath = filePath;

photoCaptureObject.TakePhotoAsync
(filePath, PhotoCaptureFileOutputFormat.JPG, OnCapturedPhotoToDisk);
});
});
}

/// <summary>
/// Called right after the photo capture process has concluded
/// </summary>
void OnCapturedPhotoToDisk(PhotoCapture.PhotoCaptureResult result)
{
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}

/// <summary>

/// Register the full execution of the Photo Capture. If successfull, it will begin the Image Analysis process.
/// </summary>
void OnStoppedPhotoMode(PhotoCapture.PhotoCaptureResult result)
{
photoCaptureObject.Dispose();
photoCaptureObject = null;

// Request image caputer analysis
StartCoroutine(FaceAnalysis.Instance.DetectFacesFromImage());
}

}

4.ビルド