BlueGit

blue-star ggj-2020

Files

ggj-2020 / Assets / Oculus / LipSync / Scripts / OVRLipSyncContext.cs

OVRLipSyncContext.cs

Project Import
timfa Tim Falken · pushed by timfa · 6 years ago
b77834a1 History
OVRLipSyncContext.cs 11.7 KiB Download Raw
1/************************************************************************************
2Filename : OVRLipSyncContext.cs
3Content : Interface to Oculus Lip-Sync engine
4Created : August 6th, 2015
5Copyright : Copyright Facebook Technologies, LLC and its affiliates.
6 All rights reserved.
7
8Licensed under the Oculus Audio SDK License Version 3.3 (the "License");
9you may not use the Oculus Audio SDK except in compliance with the License,
10which is provided at the time of installation or download, or which
11otherwise accompanies this software in either electronic or hard copy form.
12
13You may obtain a copy of the License at
14
15https://developer.oculus.com/licenses/audio-3.3/
16
17Unless required by applicable law or agreed to in writing, the Oculus Audio SDK
18distributed under the License is distributed on an "AS IS" BASIS,
19WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20See the License for the specific language governing permissions and
21limitations under the License.
22************************************************************************************/
23using UnityEngine;
24
25[RequireComponent(typeof(AudioSource))]
26
27//-------------------------------------------------------------------------------------
28// ***** OVRLipSyncContext
29//
30/// <summary>
31/// OVRLipSyncContext interfaces into the Oculus phoneme recognizer.
32/// This component should be added into the scene once for each Audio Source.
33///
34/// </summary>
35public class OVRLipSyncContext : OVRLipSyncContextBase
36{
37 // * * * * * * * * * * * * *
38 // Public members
39
40
41 [Tooltip("Allow capturing of keyboard input to control operation.")]
42 public bool enableKeyboardInput = false;
43 [Tooltip("Register a mouse/touch callback to control loopback and gain (requires script restart).")]
44 public bool enableTouchInput = false;
45 [Tooltip("Play input audio back through audio output.")]
46 public bool audioLoopback = false;
47 [Tooltip("Key to toggle audio loopback.")]
48 public KeyCode loopbackKey = KeyCode.L;
49 [Tooltip("Show viseme scores in an OVRLipSyncDebugConsole display.")]
50 public bool showVisemes = false;
51 [Tooltip("Key to toggle viseme score display.")]
52 public KeyCode debugVisemesKey = KeyCode.D;
53 [Tooltip("Skip data from the Audio Source. Use if you intend to pass audio data in manually.")]
54 public bool skipAudioSource = false;
55 [Tooltip("Adjust the linear audio gain multiplier before processing lipsync")]
56 public float gain = 1.0f;
57
58 private bool hasDebugConsole = false;
59
60 public KeyCode debugLaughterKey = KeyCode.H;
61 public bool showLaughter = false;
62 public float laughterScore = 0.0f;
63
64 // * * * * * * * * * * * * *
65 // Private members
66
67 /// <summary>
68 /// Start this instance.
69 /// Note: make sure to always have a Start function for classes that have editor scripts.
70 /// </summary>
71 void Start()
72 {
73 // Add a listener to the OVRTouchpad for touch events
74 if (enableTouchInput)
75 {
76 OVRTouchpad.AddListener(LocalTouchEventCallback);
77 }
78
79 // Find console
80 OVRLipSyncDebugConsole[] consoles = FindObjectsOfType<OVRLipSyncDebugConsole>();
81 if (consoles.Length > 0)
82 {
83 hasDebugConsole = consoles[0];
84 }
85 }
86
87 /// <summary>
88 /// Handle keyboard input
89 /// </summary>
90 void HandleKeyboard()
91 {
92 // Turn loopback on/off
93 if (Input.GetKeyDown(loopbackKey))
94 {
95 ToggleAudioLoopback();
96 }
97 else if (Input.GetKeyDown(debugVisemesKey))
98 {
99 showVisemes = !showVisemes;
100
101 if (showVisemes)
102 {
103 if (hasDebugConsole)
104 {
105 Debug.Log("DEBUG SHOW VISEMES: ENABLED");
106 }
107 else
108 {
109 Debug.LogWarning("Warning: No OVRLipSyncDebugConsole in the scene!");
110 showVisemes = false;
111 }
112 }
113 else
114 {
115 if (hasDebugConsole)
116 {
117 OVRLipSyncDebugConsole.Clear();
118 }
119 Debug.Log("DEBUG SHOW VISEMES: DISABLED");
120 }
121 }
122 else if (Input.GetKeyDown(debugLaughterKey))
123 {
124 showLaughter = !showLaughter;
125
126 if (showLaughter)
127 {
128 if (hasDebugConsole)
129 {
130 Debug.Log("DEBUG SHOW LAUGHTER: ENABLED");
131 }
132 else
133 {
134 Debug.LogWarning("Warning: No OVRLipSyncDebugConsole in the scene!");
135 showLaughter = false;
136 }
137 }
138 else
139 {
140 if (hasDebugConsole)
141 {
142 OVRLipSyncDebugConsole.Clear();
143 }
144 Debug.Log("DEBUG SHOW LAUGHTER: DISABLED");
145 }
146 }
147 else if (Input.GetKeyDown(KeyCode.LeftArrow))
148 {
149 gain -= 1.0f;
150 if (gain < 1.0f) gain = 1.0f;
151
152 string g = "LINEAR GAIN: ";
153 g += gain;
154
155 if (hasDebugConsole)
156 {
157 OVRLipSyncDebugConsole.Clear();
158 OVRLipSyncDebugConsole.Log(g);
159 OVRLipSyncDebugConsole.ClearTimeout(1.5f);
160 }
161 }
162 else if (Input.GetKeyDown(KeyCode.RightArrow))
163 {
164 gain += 1.0f;
165 if (gain > 15.0f)
166 gain = 15.0f;
167
168 string g = "LINEAR GAIN: ";
169 g += gain;
170
171 if (hasDebugConsole)
172 {
173 OVRLipSyncDebugConsole.Clear();
174 OVRLipSyncDebugConsole.Log(g);
175 OVRLipSyncDebugConsole.ClearTimeout(1.5f);
176 }
177 }
178 }
179
180 /// <summary>
181 /// Run processes that need to be updated in our game thread
182 /// </summary>
183 void Update()
184 {
185 if (enableKeyboardInput)
186 {
187 HandleKeyboard();
188 }
189 laughterScore = this.Frame.laughterScore;
190 DebugShowVisemesAndLaughter();
191 }
192
193 /// <summary>
194 /// Preprocess F32 PCM audio buffer
195 /// </summary>
196 /// <param name="data">Data.</param>
197 /// <param name="channels">Channels.</param>
198 public void PreprocessAudioSamples(float[] data, int channels)
199 {
200 // Increase the gain of the input
201 for (int i = 0; i < data.Length; ++i)
202 {
203 data[i] = data[i] * gain;
204 }
205 }
206
207 /// <summary>
208 /// Postprocess F32 PCM audio buffer
209 /// </summary>
210 /// <param name="data">Data.</param>
211 /// <param name="channels">Channels.</param>
212 public void PostprocessAudioSamples(float[] data, int channels)
213 {
214 // Turn off output (so that we don't get feedback from mics too close to speakers)
215 if (!audioLoopback)
216 {
217 for (int i = 0; i < data.Length; ++i)
218 data[i] = data[i] * 0.0f;
219 }
220 }
221
222 /// <summary>
223 /// Pass F32 PCM audio buffer to the lip sync module
224 /// </summary>
225 /// <param name="data">Data.</param>
226 /// <param name="channels">Channels.</param>
227 public void ProcessAudioSamplesRaw(float[] data, int channels)
228 {
229 // Send data into Phoneme context for processing (if context is not 0)
230 lock (this)
231 {
232 if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success)
233 {
234 return;
235 }
236 var frame = this.Frame;
237 OVRLipSync.ProcessFrame(Context, data, frame, channels == 2);
238 }
239 }
240
241 /// <summary>
242 /// Pass S16 PCM audio buffer to the lip sync module
243 /// </summary>
244 /// <param name="data">Data.</param>
245 /// <param name="channels">Channels.</param>
246 public void ProcessAudioSamplesRaw(short[] data, int channels)
247 {
248 // Send data into Phoneme context for processing (if context is not 0)
249 lock (this)
250 {
251 if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success)
252 {
253 return;
254 }
255 var frame = this.Frame;
256 OVRLipSync.ProcessFrame(Context, data, frame, channels == 2);
257 }
258 }
259
260
261 /// <summary>
262 /// Process F32 audio sample and pass it to the lip sync module for computation
263 /// </summary>
264 /// <param name="data">Data.</param>
265 /// <param name="channels">Channels.</param>
266 public void ProcessAudioSamples(float[] data, int channels)
267 {
268 // Do not process if we are not initialized, or if there is no
269 // audio source attached to game object
270 if ((OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) || audioSource == null)
271 {
272 return;
273 }
274 PreprocessAudioSamples(data, channels);
275 ProcessAudioSamplesRaw(data, channels);
276 PostprocessAudioSamples(data, channels);
277 }
278
279 /// <summary>
280 /// Raises the audio filter read event.
281 /// </summary>
282 /// <param name="data">Data.</param>
283 /// <param name="channels">Channels.</param>
284 void OnAudioFilterRead(float[] data, int channels)
285 {
286 if (!skipAudioSource)
287 {
288 ProcessAudioSamples(data, channels);
289 }
290 }
291
292 /// <summary>
293 /// Print the visemes and laughter score to game window
294 /// </summary>
295 void DebugShowVisemesAndLaughter()
296 {
297 if (hasDebugConsole)
298 {
299 string seq = "";
300 if (showLaughter)
301 {
302 seq += "Laughter:";
303 int count = (int)(50.0f * this.Frame.laughterScore);
304 for (int c = 0; c < count; c++)
305 seq += "*";
306 seq += "\n";
307 }
308 if (showVisemes)
309 {
310 for (int i = 0; i < this.Frame.Visemes.Length; i++)
311 {
312 seq += ((OVRLipSync.Viseme)i).ToString();
313 seq += ":";
314
315 int count = (int)(50.0f * this.Frame.Visemes[i]);
316 for (int c = 0; c < count; c++)
317 seq += "*";
318
319 seq += "\n";
320 }
321 }
322
323 OVRLipSyncDebugConsole.Clear();
324
325 if (seq != "")
326 {
327 OVRLipSyncDebugConsole.Log(seq);
328 }
329 }
330 }
331
332 void ToggleAudioLoopback()
333 {
334 audioLoopback = !audioLoopback;
335
336 if (hasDebugConsole)
337 {
338 OVRLipSyncDebugConsole.Clear();
339 OVRLipSyncDebugConsole.ClearTimeout(1.5f);
340
341 if (audioLoopback)
342 OVRLipSyncDebugConsole.Log("LOOPBACK MODE: ENABLED");
343 else
344 OVRLipSyncDebugConsole.Log("LOOPBACK MODE: DISABLED");
345 }
346 }
347
348 // LocalTouchEventCallback
349 void LocalTouchEventCallback(OVRTouchpad.TouchEvent touchEvent)
350 {
351 string g = "LINEAR GAIN: ";
352
353 switch (touchEvent)
354 {
355 case (OVRTouchpad.TouchEvent.SingleTap):
356 ToggleAudioLoopback();
357 break;
358
359 case (OVRTouchpad.TouchEvent.Up):
360 gain += 1.0f;
361 if (gain > 15.0f)
362 gain = 15.0f;
363
364 g += gain;
365
366 if (hasDebugConsole)
367 {
368 OVRLipSyncDebugConsole.Clear();
369 OVRLipSyncDebugConsole.Log(g);
370 OVRLipSyncDebugConsole.ClearTimeout(1.5f);
371 }
372
373 break;
374
375 case (OVRTouchpad.TouchEvent.Down):
376 gain -= 1.0f;
377 if (gain < 1.0f) gain = 1.0f;
378
379 g += gain;
380
381 if (hasDebugConsole)
382 {
383 OVRLipSyncDebugConsole.Clear();
384 OVRLipSyncDebugConsole.Log(g);
385 OVRLipSyncDebugConsole.ClearTimeout(1.5f);
386 }
387
388 break;
389 }
390 }
391}