BlueGit

blue-star ggj-2020

Files

ggj-2020 / Assets / Oculus / LipSync / Scripts / OVRLipSyncSequence.cs

OVRLipSyncSequence.cs

Project Import
timfa Tim Falken · pushed by timfa · 6 years ago
b77834a1 History
OVRLipSyncSequence.cs 5.5 KiB Download Raw
1/************************************************************************************
2Filename : OVRLipSyncSequence.cs
3Content : LipSync frames container
4Created : May 17th, 2018
5Copyright : Copyright Facebook Technologies, LLC and its affiliates.
6 All rights reserved.
7
8Licensed under the Oculus Audio SDK License Version 3.3 (the "License");
9you may not use the Oculus Audio SDK except in compliance with the License,
10which is provided at the time of installation or download, or which
11otherwise accompanies this software in either electronic or hard copy form.
12
13You may obtain a copy of the License at
14
15https://developer.oculus.com/licenses/audio-3.3/
16
17Unless required by applicable law or agreed to in writing, the Oculus Audio SDK
18distributed under the License is distributed on an "AS IS" BASIS,
19WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20See the License for the specific language governing permissions and
21limitations under the License.
22************************************************************************************/
23using System;
24using System.Collections.Generic;
25using System.IO;
26using UnityEngine;
27
28// Sequence - holds ordered entries for playback
29[System.Serializable]
30public class OVRLipSyncSequence : ScriptableObject
31{
32 public List<OVRLipSync.Frame> entries = new List<OVRLipSync.Frame>();
33 public float length; // in seconds
34
35 public OVRLipSync.Frame GetFrameAtTime(float time)
36 {
37 OVRLipSync.Frame frame = null;
38 if (time < length && entries.Count > 0)
39 {
40 float percentComplete = time / length;
41 frame = entries[(int)(entries.Count * percentComplete)];
42 }
43 return frame;
44 }
45
46#if UNITY_EDITOR
47
48 private static readonly int sSampleSize = 1024;
49
50 public static OVRLipSyncSequence CreateSequenceFromAudioClip(
51 AudioClip clip, bool useOfflineModel = false)
52 {
53 OVRLipSyncSequence sequence = null;
54
55 if (clip.channels > 2)
56 {
57 Debug.LogError(clip.name +
58 ": Cannot process phonemes from an audio clip with " +
59 "more than 2 channels");
60 return null;
61 }
62
63 if (clip.loadType != AudioClipLoadType.DecompressOnLoad)
64 {
65 Debug.LogError(clip.name +
66 ": Cannot process phonemes from an audio clip unless " +
67 "its load type is set to DecompressOnLoad.");
68 return null;
69 }
70
71 if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success)
72 {
73 Debug.LogError("Could not create Lip Sync engine.");
74 return null;
75 }
76
77 if (clip.loadState != AudioDataLoadState.Loaded)
78 {
79 Debug.LogError("Clip is not loaded!");
80 return null;
81 }
82
83 uint context = 0;
84
85 OVRLipSync.Result result = useOfflineModel
86 ? OVRLipSync.CreateContextWithModelFile(
87 ref context,
88 OVRLipSync.ContextProviders.Enhanced,
89 Path.Combine(Application.dataPath, "Oculus/LipSync/Assets/OfflineModel/ovrlipsync_offline_model.pb"))
90 : OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Enhanced);
91
92 if (result != OVRLipSync.Result.Success)
93 {
94 Debug.LogError("Could not create Phoneme context. (" + result + ")");
95 OVRLipSync.Shutdown();
96 return null;
97 }
98
99 List<OVRLipSync.Frame> frames = new List<OVRLipSync.Frame>();
100 float[] samples = new float[sSampleSize * clip.channels];
101
102 OVRLipSync.Frame dummyFrame = new OVRLipSync.Frame();
103 OVRLipSync.ProcessFrame(
104 context,
105 samples,
106 dummyFrame,
107 (clip.channels == 2) ? true : false
108 );
109 // frame delay in ms
110 float frameDelayInMs = dummyFrame.frameDelay;
111
112 int frameOffset = (int)(frameDelayInMs * clip.frequency / 1000);
113
114 int totalSamples = clip.samples;
115 for (int x = 0; x < totalSamples + frameOffset; x += sSampleSize)
116 {
117 int remainingSamples = totalSamples - x;
118 if (remainingSamples >= sSampleSize) {
119 clip.GetData(samples, x);
120 } else if (remainingSamples > 0) {
121 float[] samples_clip = new float[remainingSamples * clip.channels];
122 clip.GetData(samples_clip, x);
123 Array.Copy(samples_clip, samples, samples_clip.Length);
124 Array.Clear(samples, samples_clip.Length, samples.Length - samples_clip.Length);
125 } else {
126 Array.Clear(samples, 0, samples.Length);
127 }
128
129 OVRLipSync.Frame frame = new OVRLipSync.Frame();
130 if (clip.channels == 2)
131 {
132 // interleaved = stereo data, alternating floats
133 OVRLipSync.ProcessFrame(context, samples, frame);
134 }
135 else
136 {
137 // mono
138 OVRLipSync.ProcessFrame(context, samples, frame, false);
139 }
140
141 if (x < frameOffset)
142 {
143 continue;
144 }
145
146 frames.Add(frame);
147 }
148
149 Debug.Log(clip.name + " produced " + frames.Count +
150 " viseme frames, playback rate is " + (frames.Count / clip.length) +
151 " fps");
152 OVRLipSync.DestroyContext(context);
153 OVRLipSync.Shutdown();
154
155 sequence = ScriptableObject.CreateInstance<OVRLipSyncSequence>();
156 sequence.entries = frames;
157 sequence.length = clip.length;
158
159 return sequence;
160 }
161#endif
162};