はじめに
HoloLensアドベントカレンダー2020 の13日目の記事です。
開発環境
- Azure
- Computer Vision API (Read API)
- Speech SDK 1.14.0
- Unity 2019.4.1f1
- MRTK 2.5.1
- Windows 10 PC
- HoloLens2
導入
1．前回の記事まで終わらせてください。
3．前回の記事のMySpeechRecognizer.csは、Actionワードが一つしか認識できませんでしたが、リストにして複数のActionワードを認識できるようにします。あとはUpdate関数を下記のように編集して、Actionワードが「文字を読んで」のときにTapToCaptureReadAPI.csのAirTap関数を実行します。
1MySpeechRecognizer.cs
2 // public string ActionWord = "";
3 public List ActionWords;
4
5 async void Update()
6 {
7 if (recognizedString != "")
8 {
9 // Debug.Log(recognizedString);
10 if (action){
11 foreach(string ActionWord in ActionWords){
12 if (recognizedString.ToLower().Contains(ActionWord.ToLower()))
13 {
14 Debug.Log("Action");
15 if(ActionWord == "何が見える"){
16 Debug.Log("Analyze Image");
17 this.GetComponent().AirTap();
18 }else if(ActionWord == "文字を読んで"){
19 Debug.Log("Read");
20 this.GetComponent().AirTap();
21 }
22 action = false;
23 }
24 }
25 }else if (recognizedString.ToLower().Contains(WakeWord.ToLower()))
26 {
27 Debug.Log("Wake");
28 await this.GetComponent().SynthesizeAudioAsync("はい");
29 action = true;
30 }
31 }
32 }4．「TapToCaptureReadAPI.cs」スクリプトは以下のようになります。
1TapToCaptureReadAPI.cs
2using System.Collections;
3using System.Collections.Generic;
4using System.Linq;
5using System;
6using UnityEngine;
7using Microsoft.MixedReality.Toolkit.Utilities;
8using System.Threading.Tasks;
9
10// SpeechSDK ここから
11using System.IO;
12using System.Text;
13using Microsoft.CognitiveServices.Speech;
14using Microsoft.CognitiveServices.Speech.Audio;
15// SpeechSDK ここまで
16
17public class TapToCaptureReadAPI : MonoBehaviour
18{
19 // Read ここから
20 private string read_endpoint = "https:///vision/v3.1/read/analyze";
21 private string read_subscription_key = "";
22
23 // {"status":"succeeded","createdDateTime":"2020-12-13T03:05:57Z","lastUpdatedDateTime":"2020-12-13T03:05:58Z","analyzeResult":{"version":"3.0.0","readResults":[{"page":1,"angle":-1.4203,"width":1280,"height":720,"unit":"pixel","lines":[{"boundingBox":[417,351,481,348,482,365,417,367],"text":"BANANA","words":[{"boundingBox":[420,352,481,348,482,365,421,367],"text":"BANANA","confidence":0.595}]},{"boundingBox":[541,348,604,347,605,360,542,362],"text":"ALMOND","words":[{"boundingBox":[542,349,605,347,605,360,543,363],"text":"ALMOND","confidence":0.916}]},{"boundingBox":[644,345,724,344,724,358,645,359],"text":"STRAWBERRY","words":[{"boundingBox":[647,346,724,344,724,358,647,359],"text":"STRAWBERRY","confidence":0.638}]},{"boundingBox":[757,334,843,337,843,356,757,353],"text":"PINEAPPLE","words":[{"boundingBox":[763,335,842,338,843,355,763,353],"text":"PINEAPPLE","confidence":0.623}]},{"boundingBox":[385,382,748,374,750,465,387,475],"text":"LOOK","words":[{"boundingBox":[385,384,746,374,746,467,387,475],"text":"LOOK","confidence":0.983}]},{"boundingBox":[479,474,668,468,670,501,480,508],"text":"A La Mode","words":[{"boundingBox":[482,475,512,474,513,508,483,509],"text":"A","confidence":0.987},{"boundingBox":[519,474,567,472,567,506,520,508],"text":"La","confidence":0.987},{"boundingBox":[573,472,669,468,670,501,574,505],"text":"Mode","confidence":0.984}]},{"boundingBox":[489,508,666,501,666,518,490,526],"text":"CHOCOLATE","words":[{"boundingBox":[489,509,666,501,666,519,490,527],"text":"CHOCOLATE","confidence":0.981}]}]}]}}
24 [System.Serializable]
25 public class Read
26 {
27 public string status;
28 public string createdDateTime;
29 public string lastUpdatedDateTime;
30 public AnalyzeResult analyzeResult;
31 }
32
33 [System.Serializable]
34 public class AnalyzeResult
35 {
36 public ReadResults[] readResults;
37 }
38
39 [System.Serializable]
40 public class ReadResults
41 {
42 public Lines[] lines;
43 }
44
45 [System.Serializable]
46 public class Lines
47 {
48 public string text;
49 }
50 // Read ここまで
51
52 // SpeechSDK ここから
53 public AudioSource audioSource;
54
55 public async Task SynthesizeAudioAsync(string text)
56 {
57 var config = SpeechConfig.FromSubscription("YourSubscriptionKey", "YourServiceRegion");
58 var synthesizer = new SpeechSynthesizer(config, null); // nullを省略するとPCのスピーカーから出力されるが、HoloLensでは出力されない。
59
60 string ssml = " " + text + " ";
61
62 // Starts speech synthesis, and returns after a single utterance is synthesized.
63 // using (var result = synthesizer.SpeakTextAsync(text).Result)
64 using (var result = synthesizer.SpeakSsmlAsync(ssml).Result)
65 {
66 // Checks result.
67 if (result.Reason == ResultReason.SynthesizingAudioCompleted)
68 {
69 // Native playback is not supported on Unity yet (currently only supported on Windows/Linux Desktop).
70 // Use the Unity API to play audio here as a short term solution.
71 // Native playback support will be added in the future release.
72 var sampleCount = result.AudioData.Length / 2;
73 var audioData = new float[sampleCount];
74 for (var i = 0; i < sampleCount; ++i)
75 {
76 audioData[i] = (short)(result.AudioData[i * 2 + 1] << 8 | result.AudioData[i * 2]) / 32768.0F; } // The output audio format is 16K 16bit mono var audioClip = AudioClip.Create("SynthesizedAudio", sampleCount, 1, 16000, false); audioClip.SetData(audioData, 0); audioSource.clip = audioClip; audioSource.Play(); // newMessage = "Speech synthesis succeeded!"; } else if (result.Reason == ResultReason.Canceled) { var cancellation = SpeechSynthesisCancellationDetails.FromResult(result); // newMessage = $"CANCELED:\nReason=[{cancellation.Reason}]\nErrorDetails=[{cancellation.ErrorDetails}]\nDid you update the subscription info?"; } } } // SpeechSDK ここまで public GameObject quad; UnityEngine.Windows.WebCam.PhotoCapture photoCaptureObject = null; Texture2D targetTexture = null; private bool waitingForCapture; void Start(){ waitingForCapture = false; } public void AirTap() { if (waitingForCapture) return; waitingForCapture = true; Resolution cameraResolution = UnityEngine.Windows.WebCam.PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
77 targetTexture = new Texture2D(cameraResolution.width, cameraResolution.height);
78
79 // PhotoCapture オブジェクトを作成します
80 UnityEngine.Windows.WebCam.PhotoCapture.CreateAsync(false, delegate (UnityEngine.Windows.WebCam.PhotoCapture captureObject) {
81 photoCaptureObject = captureObject;
82 UnityEngine.Windows.WebCam.CameraParameters cameraParameters = new UnityEngine.Windows.WebCam.CameraParameters();
83 cameraParameters.hologramOpacity = 0.0f;
84 cameraParameters.cameraResolutionWidth = cameraResolution.width;
85 cameraParameters.cameraResolutionHeight = cameraResolution.height;
86 cameraParameters.pixelFormat = UnityEngine.Windows.WebCam.CapturePixelFormat.BGRA32;
87
88 // カメラをアクティベートします
89 photoCaptureObject.StartPhotoModeAsync(cameraParameters, delegate (UnityEngine.Windows.WebCam.PhotoCapture.PhotoCaptureResult result) {
90 // 写真を撮ります
91 photoCaptureObject.TakePhotoAsync(OnCapturedPhotoToMemoryAsync);
92 });
93 });
94 }
95
96 async void OnCapturedPhotoToMemoryAsync(UnityEngine.Windows.WebCam.PhotoCapture.PhotoCaptureResult result, UnityEngine.Windows.WebCam.PhotoCaptureFrame photoCaptureFrame)
97 {
98 // ターゲットテクスチャに RAW 画像データをコピーします
99 photoCaptureFrame.UploadImageDataToTexture(targetTexture);
100 byte[] bodyData = targetTexture.EncodeToJPG();
101
102 Response response = new Response();
103 Dictionary<string, string> headers = new Dictionary<string, string>();
104 headers.Add("Ocp-Apim-Subscription-Key", read_subscription_key);
105
106 try
107 {
108 string query = read_endpoint;
109 // headers.Add("Content-Type": "application/octet-stream");
110 response = await Rest.PostAsync(query, bodyData, headers, -1, true);
111 }
112 catch (Exception e)
113 {
114 photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
115 return;
116 }
117
118 if (!response.Successful)
119 {
120 photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
121 return;
122 }
123
124 Debug.Log(response.ResponseCode);
125 Debug.Log(response.ResponseBody);
126
127 string operation_url = response.ResponseBody;
128 bool poll = true;
129 int i = 0;
130 Read read;
131
132 do {
133 System.Threading.Thread.Sleep(1000);
134 // https://docs.microsoft.com/ja-jp/azure/cognitive-services/computer-vision/concept-recognizing-text
135 // https://github.com/Azure-Samples/cognitive-services-quickstart-code/blob/master/dotnet/ComputerVision/REST/CSharp-hand-text.md
136 // {"status":"running","createdDateTime":"2020-12-12T19:39:44Z","lastUpdatedDateTime":"2020-12-12T19:39:44Z"}
137 response = await Rest.GetAsync(operation_url, headers, -1, null, true);
138 Debug.Log(response.ResponseCode);
139 Debug.Log(response.ResponseBody);
140 read = JsonUtility.FromJson(response.ResponseBody);
141 if (read.status == "succeeded"){
142 poll = false;
143 }
144 ++i;
145 } while (i < 60 && poll);
146
147 if (i == 60 && poll == true)
148 {
149 photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
150 return;
151 }
152
153 string read_text = "";
154 foreach (ReadResults readResult in read.analyzeResult.readResults){
155 foreach (Lines line in readResult.lines){
156 read_text = read_text + line.text + "\n";
157 // Debug.Log(line.text);
158 }
159 }
160 read_text += "以上です。";
161
162 // SpeechSDK 追加分ここから
163 Debug.Log(read_text);
164 await SynthesizeAudioAsync(read_text); // jp
165 // SpeechSDK 追加分ここまで
166
167 // カメラを非アクティブにします
168 photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
169 }
170
171 void OnStoppedPhotoMode(UnityEngine.Windows.WebCam.PhotoCapture.PhotoCaptureResult result)
172 {
173 // photo capture のリソースをシャットダウンします
174 photoCaptureObject.Dispose();
175 photoCaptureObject = null;
176 waitingForCapture = false;
177 }
178}5．Computer Vision API (Read API) のエンドポイントとキーは、 画像説明文生成 のときと同じものを用います。
6．TapToCaptureReadAPI.csのAirTap関数で画像をキャプチャしたあと、Read APIに画像をPOSTします。すると、ResponseのHeadersに{“Operation-Location”:URL}が返ってくるので、そのURLに対してGETする必要があります。しかし、MRTKのRest.csはResponseHeadersを返してくれないので、ProcessRequestAsync関数の275行目からを次のように編集します。
1Rest.cs
2・・・
3 if (readResponseData)
4 {
5 // For Read API
6 Dictionary<string, string> tmp = webRequest.GetResponseHeaders();
7 if (tmp == null)
8 {
9 return new Response(true, webRequest.downloadHandler?.text, webRequest.downloadHandler?.data, webRequest.responseCode);
10 }else{
11 if(tmp.ContainsKey("Operation-Location")){
12 string responseHeaders = tmp["Operation-Location"];
13 string downloadHandlerText = webRequest.downloadHandler?.text;
14 return new Response(true, responseHeaders, webRequest.downloadHandler?.data, webRequest.responseCode);
15 }else{
16 return new Response(true, webRequest.downloadHandler?.text, webRequest.downloadHandler?.data, webRequest.responseCode);
17 }
18 }
19 }
20 else // This option can be used only if action will be triggered in the same scope as the webrequest
21 {
22 return new Response(true, () => webRequest.downloadHandler?.text, () => webRequest.downloadHandler?.data, webRequest.responseCode);
23 }
24・・・7．これでResponseBodyにResponseHeadersのURLが返ってくるので、そのURLに対してGETします。
8．テキスト抽出が実行中の場合は下記のようなjsonが返ってくるので、”status”が”succeeded”になるまで、1秒おきにGETします。
1{"status":"running","createdDateTime":"2020-12-12T19:39:44Z","lastUpdatedDateTime":"2020-12-12T19:39:44Z"}9．”status”が”succeeded”になったら、テキスト抽出結果が次のようなjsonで返っくるので、仕様に合わせてReadクラス、AnalyzeResultクラス、ReadResultsクラス、Linesクラスを作成しました。
1{"status":"succeeded","createdDateTime":"2020-12-13T03:05:57Z","lastUpdatedDateTime":"2020-12-13T03:05:58Z","analyzeResult":{"version":"3.0.0","readResults":[{"page":1,"angle":-1.4203,"width":1280,"height":720,"unit":"pixel","lines":[{"boundingBox":[417,351,481,348,482,365,417,367],"text":"BANANA","words":[{"boundingBox":[420,352,481,348,482,365,421,367],"text":"BANANA","confidence":0.595}]},{"boundingBox":[541,348,604,347,605,360,542,362],"text":"ALMOND","words":[{"boundingBox":[542,349,605,347,605,360,543,363],"text":"ALMOND","confidence":0.916}]},{"boundingBox":[644,345,724,344,724,358,645,359],"text":"STRAWBERRY","words":[{"boundingBox":[647,346,724,344,724,358,647,359],"text":"STRAWBERRY","confidence":0.638}]},{"boundingBox":[757,334,843,337,843,356,757,353],"text":"PINEAPPLE","words":[{"boundingBox":[763,335,842,338,843,355,763,353],"text":"PINEAPPLE","confidence":0.623}]},{"boundingBox":[385,382,748,374,750,465,387,475],"text":"LOOK","words":[{"boundingBox":[385,384,746,374,746,467,387,475],"text":"LOOK","confidence":0.983}]},{"boundingBox":[479,474,668,468,670,501,480,508],"text":"A La Mode","words":[{"boundingBox":[482,475,512,474,513,508,483,509],"text":"A","confidence":0.987},{"boundingBox":[519,474,567,472,567,506,520,508],"text":"La","confidence":0.987},{"boundingBox":[573,472,669,468,670,501,574,505],"text":"Mode","confidence":0.984}]},{"boundingBox":[489,508,666,501,666,518,490,526],"text":"CHOCOLATE","words":[{"boundingBox":[489,509,666,501,666,519,490,527],"text":"CHOCOLATE","confidence":0.981}]}]}]}}
実行
実行動画を見てください。こんな感じで文字を読めるようになりました！割と小さい文字もいけます。名刺とかも読めるし、便利かもしれないです。日本語対応はまだなので待つしかないですね。（現在の対応言語：Dutch, English, French, German, Italian, Portuguese and Spanish）
ちなみにWakeワードの「ヨンシル」は「4種類」や「キャンセル」などに誤認しやすいので、変えた方がいいです。「ガチモト」は「が地元」や「合致もっと」になるので、「藤本」がいいです。。
