| 1 | /************************************************************************************ |
| 2 | Filename : OVRLipSyncContext.cs |
| 3 | Content : Interface to Oculus Lip-Sync engine |
| 4 | Created : August 6th, 2015 |
| 5 | Copyright : Copyright Facebook Technologies, LLC and its affiliates. |
| 6 | All rights reserved. |
| 7 | |
| 8 | Licensed under the Oculus Audio SDK License Version 3.3 (the "License"); |
| 9 | you may not use the Oculus Audio SDK except in compliance with the License, |
| 10 | which is provided at the time of installation or download, or which |
| 11 | otherwise accompanies this software in either electronic or hard copy form. |
| 12 | |
| 13 | You may obtain a copy of the License at |
| 14 | |
| 15 | https://developer.oculus.com/licenses/audio-3.3/ |
| 16 | |
| 17 | Unless required by applicable law or agreed to in writing, the Oculus Audio SDK |
| 18 | distributed under the License is distributed on an "AS IS" BASIS, |
| 19 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 20 | See the License for the specific language governing permissions and |
| 21 | limitations under the License. |
| 22 | ************************************************************************************/ |
| 23 | using UnityEngine; |
| 24 | |
| 25 | [RequireComponent(typeof(AudioSource))] |
| 26 | |
| 27 | //------------------------------------------------------------------------------------- |
| 28 | // ***** OVRLipSyncContext |
| 29 | // |
| 30 | /// <summary> |
| 31 | /// OVRLipSyncContext interfaces into the Oculus phoneme recognizer. |
| 32 | /// This component should be added into the scene once for each Audio Source. |
| 33 | /// |
| 34 | /// </summary> |
| 35 | public class OVRLipSyncContext : OVRLipSyncContextBase |
| 36 | { |
| 37 | // * * * * * * * * * * * * * |
| 38 | // Public members |
| 39 | |
| 40 | |
| 41 | [Tooltip("Allow capturing of keyboard input to control operation.")] |
| 42 | public bool enableKeyboardInput = false; |
| 43 | [Tooltip("Register a mouse/touch callback to control loopback and gain (requires script restart).")] |
| 44 | public bool enableTouchInput = false; |
| 45 | [Tooltip("Play input audio back through audio output.")] |
| 46 | public bool audioLoopback = false; |
| 47 | [Tooltip("Key to toggle audio loopback.")] |
| 48 | public KeyCode loopbackKey = KeyCode.L; |
| 49 | [Tooltip("Show viseme scores in an OVRLipSyncDebugConsole display.")] |
| 50 | public bool showVisemes = false; |
| 51 | [Tooltip("Key to toggle viseme score display.")] |
| 52 | public KeyCode debugVisemesKey = KeyCode.D; |
| 53 | [Tooltip("Skip data from the Audio Source. Use if you intend to pass audio data in manually.")] |
| 54 | public bool skipAudioSource = false; |
| 55 | [Tooltip("Adjust the linear audio gain multiplier before processing lipsync")] |
| 56 | public float gain = 1.0f; |
| 57 | |
| 58 | private bool hasDebugConsole = false; |
| 59 | |
| 60 | public KeyCode debugLaughterKey = KeyCode.H; |
| 61 | public bool showLaughter = false; |
| 62 | public float laughterScore = 0.0f; |
| 63 | |
| 64 | // * * * * * * * * * * * * * |
| 65 | // Private members |
| 66 | |
| 67 | /// <summary> |
| 68 | /// Start this instance. |
| 69 | /// Note: make sure to always have a Start function for classes that have editor scripts. |
| 70 | /// </summary> |
| 71 | void Start() |
| 72 | { |
| 73 | // Add a listener to the OVRTouchpad for touch events |
| 74 | if (enableTouchInput) |
| 75 | { |
| 76 | OVRTouchpad.AddListener(LocalTouchEventCallback); |
| 77 | } |
| 78 | |
| 79 | // Find console |
| 80 | OVRLipSyncDebugConsole[] consoles = FindObjectsOfType<OVRLipSyncDebugConsole>(); |
| 81 | if (consoles.Length > 0) |
| 82 | { |
| 83 | hasDebugConsole = consoles[0]; |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | /// <summary> |
| 88 | /// Handle keyboard input |
| 89 | /// </summary> |
| 90 | void HandleKeyboard() |
| 91 | { |
| 92 | // Turn loopback on/off |
| 93 | if (Input.GetKeyDown(loopbackKey)) |
| 94 | { |
| 95 | ToggleAudioLoopback(); |
| 96 | } |
| 97 | else if (Input.GetKeyDown(debugVisemesKey)) |
| 98 | { |
| 99 | showVisemes = !showVisemes; |
| 100 | |
| 101 | if (showVisemes) |
| 102 | { |
| 103 | if (hasDebugConsole) |
| 104 | { |
| 105 | Debug.Log("DEBUG SHOW VISEMES: ENABLED"); |
| 106 | } |
| 107 | else |
| 108 | { |
| 109 | Debug.LogWarning("Warning: No OVRLipSyncDebugConsole in the scene!"); |
| 110 | showVisemes = false; |
| 111 | } |
| 112 | } |
| 113 | else |
| 114 | { |
| 115 | if (hasDebugConsole) |
| 116 | { |
| 117 | OVRLipSyncDebugConsole.Clear(); |
| 118 | } |
| 119 | Debug.Log("DEBUG SHOW VISEMES: DISABLED"); |
| 120 | } |
| 121 | } |
| 122 | else if (Input.GetKeyDown(debugLaughterKey)) |
| 123 | { |
| 124 | showLaughter = !showLaughter; |
| 125 | |
| 126 | if (showLaughter) |
| 127 | { |
| 128 | if (hasDebugConsole) |
| 129 | { |
| 130 | Debug.Log("DEBUG SHOW LAUGHTER: ENABLED"); |
| 131 | } |
| 132 | else |
| 133 | { |
| 134 | Debug.LogWarning("Warning: No OVRLipSyncDebugConsole in the scene!"); |
| 135 | showLaughter = false; |
| 136 | } |
| 137 | } |
| 138 | else |
| 139 | { |
| 140 | if (hasDebugConsole) |
| 141 | { |
| 142 | OVRLipSyncDebugConsole.Clear(); |
| 143 | } |
| 144 | Debug.Log("DEBUG SHOW LAUGHTER: DISABLED"); |
| 145 | } |
| 146 | } |
| 147 | else if (Input.GetKeyDown(KeyCode.LeftArrow)) |
| 148 | { |
| 149 | gain -= 1.0f; |
| 150 | if (gain < 1.0f) gain = 1.0f; |
| 151 | |
| 152 | string g = "LINEAR GAIN: "; |
| 153 | g += gain; |
| 154 | |
| 155 | if (hasDebugConsole) |
| 156 | { |
| 157 | OVRLipSyncDebugConsole.Clear(); |
| 158 | OVRLipSyncDebugConsole.Log(g); |
| 159 | OVRLipSyncDebugConsole.ClearTimeout(1.5f); |
| 160 | } |
| 161 | } |
| 162 | else if (Input.GetKeyDown(KeyCode.RightArrow)) |
| 163 | { |
| 164 | gain += 1.0f; |
| 165 | if (gain > 15.0f) |
| 166 | gain = 15.0f; |
| 167 | |
| 168 | string g = "LINEAR GAIN: "; |
| 169 | g += gain; |
| 170 | |
| 171 | if (hasDebugConsole) |
| 172 | { |
| 173 | OVRLipSyncDebugConsole.Clear(); |
| 174 | OVRLipSyncDebugConsole.Log(g); |
| 175 | OVRLipSyncDebugConsole.ClearTimeout(1.5f); |
| 176 | } |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | /// <summary> |
| 181 | /// Run processes that need to be updated in our game thread |
| 182 | /// </summary> |
| 183 | void Update() |
| 184 | { |
| 185 | if (enableKeyboardInput) |
| 186 | { |
| 187 | HandleKeyboard(); |
| 188 | } |
| 189 | laughterScore = this.Frame.laughterScore; |
| 190 | DebugShowVisemesAndLaughter(); |
| 191 | } |
| 192 | |
| 193 | /// <summary> |
| 194 | /// Preprocess F32 PCM audio buffer |
| 195 | /// </summary> |
| 196 | /// <param name="data">Data.</param> |
| 197 | /// <param name="channels">Channels.</param> |
| 198 | public void PreprocessAudioSamples(float[] data, int channels) |
| 199 | { |
| 200 | // Increase the gain of the input |
| 201 | for (int i = 0; i < data.Length; ++i) |
| 202 | { |
| 203 | data[i] = data[i] * gain; |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | /// <summary> |
| 208 | /// Postprocess F32 PCM audio buffer |
| 209 | /// </summary> |
| 210 | /// <param name="data">Data.</param> |
| 211 | /// <param name="channels">Channels.</param> |
| 212 | public void PostprocessAudioSamples(float[] data, int channels) |
| 213 | { |
| 214 | // Turn off output (so that we don't get feedback from mics too close to speakers) |
| 215 | if (!audioLoopback) |
| 216 | { |
| 217 | for (int i = 0; i < data.Length; ++i) |
| 218 | data[i] = data[i] * 0.0f; |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | /// <summary> |
| 223 | /// Pass F32 PCM audio buffer to the lip sync module |
| 224 | /// </summary> |
| 225 | /// <param name="data">Data.</param> |
| 226 | /// <param name="channels">Channels.</param> |
| 227 | public void ProcessAudioSamplesRaw(float[] data, int channels) |
| 228 | { |
| 229 | // Send data into Phoneme context for processing (if context is not 0) |
| 230 | lock (this) |
| 231 | { |
| 232 | if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) |
| 233 | { |
| 234 | return; |
| 235 | } |
| 236 | var frame = this.Frame; |
| 237 | OVRLipSync.ProcessFrame(Context, data, frame, channels == 2); |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | /// <summary> |
| 242 | /// Pass S16 PCM audio buffer to the lip sync module |
| 243 | /// </summary> |
| 244 | /// <param name="data">Data.</param> |
| 245 | /// <param name="channels">Channels.</param> |
| 246 | public void ProcessAudioSamplesRaw(short[] data, int channels) |
| 247 | { |
| 248 | // Send data into Phoneme context for processing (if context is not 0) |
| 249 | lock (this) |
| 250 | { |
| 251 | if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) |
| 252 | { |
| 253 | return; |
| 254 | } |
| 255 | var frame = this.Frame; |
| 256 | OVRLipSync.ProcessFrame(Context, data, frame, channels == 2); |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | |
| 261 | /// <summary> |
| 262 | /// Process F32 audio sample and pass it to the lip sync module for computation |
| 263 | /// </summary> |
| 264 | /// <param name="data">Data.</param> |
| 265 | /// <param name="channels">Channels.</param> |
| 266 | public void ProcessAudioSamples(float[] data, int channels) |
| 267 | { |
| 268 | // Do not process if we are not initialized, or if there is no |
| 269 | // audio source attached to game object |
| 270 | if ((OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) || audioSource == null) |
| 271 | { |
| 272 | return; |
| 273 | } |
| 274 | PreprocessAudioSamples(data, channels); |
| 275 | ProcessAudioSamplesRaw(data, channels); |
| 276 | PostprocessAudioSamples(data, channels); |
| 277 | } |
| 278 | |
| 279 | /// <summary> |
| 280 | /// Raises the audio filter read event. |
| 281 | /// </summary> |
| 282 | /// <param name="data">Data.</param> |
| 283 | /// <param name="channels">Channels.</param> |
| 284 | void OnAudioFilterRead(float[] data, int channels) |
| 285 | { |
| 286 | if (!skipAudioSource) |
| 287 | { |
| 288 | ProcessAudioSamples(data, channels); |
| 289 | } |
| 290 | } |
| 291 | |
| 292 | /// <summary> |
| 293 | /// Print the visemes and laughter score to game window |
| 294 | /// </summary> |
| 295 | void DebugShowVisemesAndLaughter() |
| 296 | { |
| 297 | if (hasDebugConsole) |
| 298 | { |
| 299 | string seq = ""; |
| 300 | if (showLaughter) |
| 301 | { |
| 302 | seq += "Laughter:"; |
| 303 | int count = (int)(50.0f * this.Frame.laughterScore); |
| 304 | for (int c = 0; c < count; c++) |
| 305 | seq += "*"; |
| 306 | seq += "\n"; |
| 307 | } |
| 308 | if (showVisemes) |
| 309 | { |
| 310 | for (int i = 0; i < this.Frame.Visemes.Length; i++) |
| 311 | { |
| 312 | seq += ((OVRLipSync.Viseme)i).ToString(); |
| 313 | seq += ":"; |
| 314 | |
| 315 | int count = (int)(50.0f * this.Frame.Visemes[i]); |
| 316 | for (int c = 0; c < count; c++) |
| 317 | seq += "*"; |
| 318 | |
| 319 | seq += "\n"; |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | OVRLipSyncDebugConsole.Clear(); |
| 324 | |
| 325 | if (seq != "") |
| 326 | { |
| 327 | OVRLipSyncDebugConsole.Log(seq); |
| 328 | } |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | void ToggleAudioLoopback() |
| 333 | { |
| 334 | audioLoopback = !audioLoopback; |
| 335 | |
| 336 | if (hasDebugConsole) |
| 337 | { |
| 338 | OVRLipSyncDebugConsole.Clear(); |
| 339 | OVRLipSyncDebugConsole.ClearTimeout(1.5f); |
| 340 | |
| 341 | if (audioLoopback) |
| 342 | OVRLipSyncDebugConsole.Log("LOOPBACK MODE: ENABLED"); |
| 343 | else |
| 344 | OVRLipSyncDebugConsole.Log("LOOPBACK MODE: DISABLED"); |
| 345 | } |
| 346 | } |
| 347 | |
| 348 | // LocalTouchEventCallback |
| 349 | void LocalTouchEventCallback(OVRTouchpad.TouchEvent touchEvent) |
| 350 | { |
| 351 | string g = "LINEAR GAIN: "; |
| 352 | |
| 353 | switch (touchEvent) |
| 354 | { |
| 355 | case (OVRTouchpad.TouchEvent.SingleTap): |
| 356 | ToggleAudioLoopback(); |
| 357 | break; |
| 358 | |
| 359 | case (OVRTouchpad.TouchEvent.Up): |
| 360 | gain += 1.0f; |
| 361 | if (gain > 15.0f) |
| 362 | gain = 15.0f; |
| 363 | |
| 364 | g += gain; |
| 365 | |
| 366 | if (hasDebugConsole) |
| 367 | { |
| 368 | OVRLipSyncDebugConsole.Clear(); |
| 369 | OVRLipSyncDebugConsole.Log(g); |
| 370 | OVRLipSyncDebugConsole.ClearTimeout(1.5f); |
| 371 | } |
| 372 | |
| 373 | break; |
| 374 | |
| 375 | case (OVRTouchpad.TouchEvent.Down): |
| 376 | gain -= 1.0f; |
| 377 | if (gain < 1.0f) gain = 1.0f; |
| 378 | |
| 379 | g += gain; |
| 380 | |
| 381 | if (hasDebugConsole) |
| 382 | { |
| 383 | OVRLipSyncDebugConsole.Clear(); |
| 384 | OVRLipSyncDebugConsole.Log(g); |
| 385 | OVRLipSyncDebugConsole.ClearTimeout(1.5f); |
| 386 | } |
| 387 | |
| 388 | break; |
| 389 | } |
| 390 | } |
| 391 | } |