BlueGit

blue-star ggj-2020

Files

ggj-2020 / Assets / Oculus / LipSync / Scripts / OVRLipSync.cs

OVRLipSync.cs

Project Import
timfa Tim Falken · pushed by timfa · 6 years ago
b77834a1 History
OVRLipSync.cs 14.9 KiB Download Raw
1/************************************************************************************
2Filename : OVRLipSync.cs
3Content : Interface to Oculus Lip Sync engine
4Created : August 4th, 2015
5Copyright : Copyright Facebook Technologies, LLC and its affiliates.
6 All rights reserved.
7
8Licensed under the Oculus Audio SDK License Version 3.3 (the "License");
9you may not use the Oculus Audio SDK except in compliance with the License,
10which is provided at the time of installation or download, or which
11otherwise accompanies this software in either electronic or hard copy form.
12
13You may obtain a copy of the License at
14
15https://developer.oculus.com/licenses/audio-3.3/
16
17Unless required by applicable law or agreed to in writing, the Oculus Audio SDK
18distributed under the License is distributed on an "AS IS" BASIS,
19WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20See the License for the specific language governing permissions and
21limitations under the License.
22************************************************************************************/
23using UnityEngine;
24using System;
25using System.Runtime.InteropServices;
26
27//-------------------------------------------------------------------------------------
28// ***** OVRLipSync
29//
30/// <summary>
31/// OVRLipSync interfaces into the Oculus lip sync engine. This component should be added
32/// into the scene once.
33///
34/// </summary>
35public class OVRLipSync : MonoBehaviour
36{
37 // Error codes that may return from Lip Sync engine
38 public enum Result
39 {
40 Success = 0,
41 Unknown = -2200, //< An unknown error has occurred
42 CannotCreateContext = -2201, //< Unable to create a context
43 InvalidParam = -2202, //< An invalid parameter, e.g. NULL pointer or out of range
44 BadSampleRate = -2203, //< An unsupported sample rate was declared
45 MissingDLL = -2204, //< The DLL or shared library could not be found
46 BadVersion = -2205, //< Mismatched versions between header and libs
47 UndefinedFunction = -2206 //< An undefined function
48 };
49
50 // Audio buffer data type
51 public enum AudioDataType
52 {
53 // Signed 16-bit integer mono audio stream
54 S16_Mono,
55 // Signed 16-bit integer stereo audio stream
56 S16_Stereo,
57 // Signed 32-bit float mono audio stream
58 F32_Mono,
59 // Signed 32-bit float stereo audio stream
60 F32_Stereo
61 };
62
63 // Various visemes
64 public enum Viseme
65 {
66 sil,
67 PP,
68 FF,
69 TH,
70 DD,
71 kk,
72 CH,
73 SS,
74 nn,
75 RR,
76 aa,
77 E,
78 ih,
79 oh,
80 ou
81 };
82
83 public static readonly int VisemeCount = Enum.GetNames(typeof(Viseme)).Length;
84
85 // Enum for sending lip-sync engine specific signals
86 public enum Signals
87 {
88 VisemeOn,
89 VisemeOff,
90 VisemeAmount,
91 VisemeSmoothing,
92 LaughterAmount
93 };
94
95 public static readonly int SignalCount = Enum.GetNames(typeof(Signals)).Length;
96
97 // Enum for provider context to create
98 public enum ContextProviders
99 {
100 Original,
101 Enhanced,
102 Enhanced_with_Laughter,
103 };
104
105 /// NOTE: Opaque typedef for lip-sync context is an unsigned int (uint)
106
107 /// Current phoneme frame results
108 [System.Serializable]
109 public class Frame
110 {
111 public void CopyInput(Frame input)
112 {
113 frameNumber = input.frameNumber;
114 frameDelay = input.frameDelay;
115 input.Visemes.CopyTo(Visemes, 0);
116 laughterScore = input.laughterScore;
117 }
118
119 public void Reset()
120 {
121 frameNumber = 0;
122 frameDelay = 0;
123 Array.Clear(Visemes, 0, VisemeCount);
124 laughterScore = 0;
125 }
126
127 public int frameNumber; // count from start of recognition
128 public int frameDelay; // in ms
129 public float[] Visemes = new float[VisemeCount]; // Array of floats for viseme frame. Size of Viseme Count, above
130 public float laughterScore; // probability of laughter presence.
131 };
132
133 // * * * * * * * * * * * * *
134 // Import functions
135 #if !UNITY_IOS || UNITY_EDITOR
136 public const string strOVRLS = "OVRLipSync";
137 #else
138 public const string strOVRLS = "__Internal";
139 #endif
140 [DllImport(strOVRLS)]
141 private static extern int ovrLipSyncDll_Initialize(int samplerate, int buffersize);
142 [DllImport(strOVRLS)]
143 private static extern void ovrLipSyncDll_Shutdown();
144 [DllImport(strOVRLS)]
145 private static extern IntPtr ovrLipSyncDll_GetVersion(ref int Major,
146 ref int Minor,
147 ref int Patch);
148 [DllImport(strOVRLS)]
149 private static extern int ovrLipSyncDll_CreateContextEx(ref uint context,
150 ContextProviders provider,
151 int sampleRate,
152 bool enableAcceleration);
153
154 [DllImport(strOVRLS)]
155 private static extern int ovrLipSyncDll_CreateContextWithModelFile(ref uint context,
156 ContextProviders provider,
157 string modelPath,
158 int sampleRate,
159 bool enableAcceleration);
160
161 [DllImport(strOVRLS)]
162 private static extern int ovrLipSyncDll_DestroyContext(uint context);
163
164
165 [DllImport(strOVRLS)]
166 private static extern int ovrLipSyncDll_ResetContext(uint context);
167 [DllImport(strOVRLS)]
168 private static extern int ovrLipSyncDll_SendSignal(uint context,
169 Signals signal,
170 int arg1, int arg2);
171 [DllImport(strOVRLS)]
172 private static extern int ovrLipSyncDll_ProcessFrameEx(
173 uint context,
174 IntPtr audioBuffer,
175 uint bufferSize,
176 AudioDataType dataType,
177 ref int frameNumber,
178 ref int frameDelay,
179 float[] visemes,
180 int visemeCount,
181 ref float laughterScore,
182 float[] laughterCategories,
183 int laughterCategoriesLength);
184
185 // * * * * * * * * * * * * *
186 // Public members
187
188 // * * * * * * * * * * * * *
189 // Static members
190 private static Result sInitialized = Result.Unknown;
191
192 // interface through this static member.
193 public static OVRLipSync sInstance = null;
194
195
196 // * * * * * * * * * * * * *
197 // MonoBehaviour overrides
198
199 /// <summary>
200 /// Awake this instance.
201 /// </summary>
202 void Awake()
203 {
204 // We can only have one instance of OVRLipSync in a scene (use this for local property query)
205 if (sInstance == null)
206 {
207 sInstance = this;
208 }
209 else
210 {
211 Debug.LogWarning(System.String.Format("OVRLipSync Awake: Only one instance of OVRPLipSync can exist in the scene."));
212 return;
213 }
214
215 if (IsInitialized() != Result.Success)
216 {
217 sInitialized = Initialize();
218
219 if (sInitialized != Result.Success)
220 {
221 Debug.LogWarning(System.String.Format
222 ("OvrLipSync Awake: Failed to init Speech Rec library"));
223 }
224 }
225
226 // Important: Use the touchpad mechanism for input, call Create on the OVRTouchpad helper class
227 OVRTouchpad.Create();
228
229 }
230
231 /// <summary>
232 /// Raises the destroy event.
233 /// </summary>
234 void OnDestroy()
235 {
236 if (sInstance != this)
237 {
238 Debug.LogWarning(
239 "OVRLipSync OnDestroy: This is not the correct OVRLipSync instance.");
240 return;
241 }
242
243 // Do not shut down at this time
244 // ovrLipSyncDll_Shutdown();
245 // sInitialized = (int)Result.Unknown;
246 }
247
248
249 // * * * * * * * * * * * * *
250 // Public Functions
251
252 public static Result Initialize()
253 {
254 int sampleRate;
255 int bufferSize;
256 int numbuf;
257
258 // Get the current sample rate
259 sampleRate = AudioSettings.outputSampleRate;
260 // Get the current buffer size and number of buffers
261 AudioSettings.GetDSPBufferSize(out bufferSize, out numbuf);
262
263 String str = System.String.Format
264 ("OvrLipSync Awake: Queried SampleRate: {0:F0} BufferSize: {1:F0}", sampleRate, bufferSize);
265 Debug.LogWarning(str);
266
267 sInitialized = (Result)ovrLipSyncDll_Initialize(sampleRate, bufferSize);
268 return sInitialized;
269 }
270
271 public static Result Initialize(int sampleRate, int bufferSize)
272 {
273 String str = System.String.Format
274 ("OvrLipSync Awake: Queried SampleRate: {0:F0} BufferSize: {1:F0}", sampleRate, bufferSize);
275 Debug.LogWarning(str);
276
277 sInitialized = (Result)ovrLipSyncDll_Initialize(sampleRate, bufferSize);
278 return sInitialized;
279 }
280
281 public static void Shutdown()
282 {
283 ovrLipSyncDll_Shutdown();
284 sInitialized = Result.Unknown;
285 }
286
287 /// <summary>
288 /// Determines if is initialized.
289 /// </summary>
290 /// <returns><c>true</c> if is initialized; otherwise, <c>false</c>.</returns>
291 public static Result IsInitialized()
292 {
293 return sInitialized;
294 }
295
296 /// <summary>
297 /// Creates a lip-sync context.
298 /// </summary>
299 /// <returns>error code</returns>
300 /// <param name="context">Context.</param>
301 /// <param name="provider">Provider.</param>
302 /// <param name="enableAcceleration">Enable DSP Acceleration.</param>
303 public static Result CreateContext(
304 ref uint context,
305 ContextProviders provider,
306 int sampleRate = 0,
307 bool enableAcceleration = false)
308 {
309 if (IsInitialized() != Result.Success && Initialize() != Result.Success)
310 return Result.CannotCreateContext;
311
312 return (Result)ovrLipSyncDll_CreateContextEx(ref context, provider, sampleRate, enableAcceleration);
313 }
314
315 /// <summary>
316 /// Creates a lip-sync context with specified model file.
317 /// </summary>
318 /// <returns>error code</returns>
319 /// <param name="context">Context.</param>
320 /// <param name="provider">Provider.</param>
321 /// <param name="modelPath">Model Dir.</param>
322 /// <param name="sampleRate">Sampling Rate.</param>
323 /// <param name="enableAcceleration">Enable DSP Acceleration.</param>
324 public static Result CreateContextWithModelFile(
325 ref uint context,
326 ContextProviders provider,
327 string modelPath,
328 int sampleRate = 0,
329 bool enableAcceleration = false)
330 {
331 if (IsInitialized() != Result.Success && Initialize() != Result.Success)
332 return Result.CannotCreateContext;
333
334 return (Result)ovrLipSyncDll_CreateContextWithModelFile(
335 ref context,
336 provider,
337 modelPath,
338 sampleRate,
339 enableAcceleration);
340 }
341
342 /// <summary>
343 /// Destroy a lip-sync context.
344 /// </summary>
345 /// <returns>The context.</returns>
346 /// <param name="context">Context.</param>
347 public static Result DestroyContext(uint context)
348 {
349 if (IsInitialized() != Result.Success)
350 return Result.Unknown;
351
352 return (Result)ovrLipSyncDll_DestroyContext(context);
353 }
354
355 /// <summary>
356 /// Resets the context.
357 /// </summary>
358 /// <returns>error code</returns>
359 /// <param name="context">Context.</param>
360 public static Result ResetContext(uint context)
361 {
362 if (IsInitialized() != Result.Success)
363 return Result.Unknown;
364
365 return (Result)ovrLipSyncDll_ResetContext(context);
366 }
367
368 /// <summary>
369 /// Sends a signal to the lip-sync engine.
370 /// </summary>
371 /// <returns>error code</returns>
372 /// <param name="context">Context.</param>
373 /// <param name="signal">Signal.</param>
374 /// <param name="arg1">Arg1.</param>
375 /// <param name="arg2">Arg2.</param>
376 public static Result SendSignal(uint context, Signals signal, int arg1, int arg2)
377 {
378 if (IsInitialized() != Result.Success)
379 return Result.Unknown;
380
381 return (Result)ovrLipSyncDll_SendSignal(context, signal, arg1, arg2);
382 }
383
384 /// <summary>
385 /// Process float[] audio buffer by lip-sync engine.
386 /// </summary>
387 /// <returns>error code</returns>
388 /// <param name="context">Context.</param>
389 /// <param name="audioBuffer"> PCM audio buffer.</param>
390 /// <param name="frame">Lip-sync Frame.</param>
391 /// <param name="stereo">Whether buffer is part of stereo or mono stream.</param>
392 public static Result ProcessFrame(
393 uint context, float[] audioBuffer, Frame frame, bool stereo = true)
394 {
395 if (IsInitialized() != Result.Success)
396 return Result.Unknown;
397
398 var dataType = stereo ? AudioDataType.F32_Stereo : AudioDataType.F32_Mono;
399 var numSamples = (uint)(stereo ? audioBuffer.Length / 2 : audioBuffer.Length);
400 var handle = GCHandle.Alloc(audioBuffer, GCHandleType.Pinned);
401 var rc = ovrLipSyncDll_ProcessFrameEx(context,
402 handle.AddrOfPinnedObject(), numSamples, dataType,
403 ref frame.frameNumber, ref frame.frameDelay,
404 frame.Visemes, frame.Visemes.Length,
405 ref frame.laughterScore,
406 null, 0
407 );
408 handle.Free();
409 return (Result)rc;
410
411 }
412
413 /// <summary>
414 /// Process short[] audio buffer by lip-sync engine.
415 /// </summary>
416 /// <returns>error code</returns>
417 /// <param name="context">Context.</param>
418 /// <param name="audioBuffer"> PCM audio buffer.</param>
419 /// <param name="frame">Lip-sync Frame.</param>
420 /// <param name="stereo">Whether buffer is part of stereo or mono stream.</param>
421 public static Result ProcessFrame(
422 uint context, short[] audioBuffer, Frame frame, bool stereo = true)
423 {
424 if (IsInitialized() != Result.Success)
425 return Result.Unknown;
426
427 var dataType = stereo ? AudioDataType.S16_Stereo : AudioDataType.S16_Mono;
428 var numSamples = (uint)(stereo ? audioBuffer.Length / 2 : audioBuffer.Length);
429 var handle = GCHandle.Alloc(audioBuffer, GCHandleType.Pinned);
430 var rc = ovrLipSyncDll_ProcessFrameEx(context,
431 handle.AddrOfPinnedObject(), numSamples, dataType,
432 ref frame.frameNumber, ref frame.frameDelay,
433 frame.Visemes, frame.Visemes.Length,
434 ref frame.laughterScore,
435 null, 0
436 );
437 handle.Free();
438 return (Result)rc;
439 }
440
441}