BlueGit

blue-star ggj-2020

Files

ggj-2020 / Assets / Oculus / LipSync / Scripts / OVRLipSyncContextMorphTarget.cs

OVRLipSyncContextMorphTarget.cs

Project Import
timfa Tim Falken · pushed by timfa · 6 years ago
b77834a1 History
OVRLipSyncContextMorphTarget.cs 7.3 KiB Download Raw
1/************************************************************************************
2Filename : OVRLipSyncContextMorphTarget.cs
3Content : This bridges the viseme output to the morph targets
4Created : August 7th, 2015
5Copyright : Copyright Facebook Technologies, LLC and its affiliates.
6 All rights reserved.
7
8Licensed under the Oculus Audio SDK License Version 3.3 (the "License");
9you may not use the Oculus Audio SDK except in compliance with the License,
10which is provided at the time of installation or download, or which
11otherwise accompanies this software in either electronic or hard copy form.
12
13You may obtain a copy of the License at
14
15https://developer.oculus.com/licenses/audio-3.3/
16
17Unless required by applicable law or agreed to in writing, the Oculus Audio SDK
18distributed under the License is distributed on an "AS IS" BASIS,
19WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20See the License for the specific language governing permissions and
21limitations under the License.
22************************************************************************************/
23using UnityEngine;
24using System.Linq;
25
26public class OVRLipSyncContextMorphTarget : MonoBehaviour
27{
28 // PUBLIC
29
30 // Manually assign the skinned mesh renderer to this script
31 [Tooltip("Skinned Mesh Rendered target to be driven by Oculus Lipsync")]
32 public SkinnedMeshRenderer skinnedMeshRenderer = null;
33
34 // Set the blendshape index to go to (-1 means there is not one assigned)
35 [Tooltip("Blendshape index to trigger for each viseme.")]
36 public int [] visemeToBlendTargets = Enumerable.Range(0, OVRLipSync.VisemeCount).ToArray();
37
38 // enable/disable sending signals to viseme engine
39 [Tooltip("Enable using the test keys defined below to manually trigger each viseme.")]
40 public bool enableVisemeTestKeys = false;
41 [Tooltip("Test keys used to manually trigger an individual viseme - by " +
42 "default the QWERTY row of a US keyboard.")]
43 public KeyCode[] visemeTestKeys =
44 {
45 KeyCode.BackQuote,
46 KeyCode.Tab,
47 KeyCode.Q,
48 KeyCode.W,
49 KeyCode.E,
50 KeyCode.R,
51 KeyCode.T,
52 KeyCode.Y,
53 KeyCode.U,
54 KeyCode.I,
55 KeyCode.O,
56 KeyCode.P,
57 KeyCode.LeftBracket,
58 KeyCode.RightBracket,
59 KeyCode.Backslash,
60 };
61
62 [Tooltip("Test key used to manually trigger laughter and visualise the results")]
63 public KeyCode laughterKey = KeyCode.CapsLock;
64
65 [Tooltip("Blendshape index to trigger for laughter")]
66 public int laughterBlendTarget = OVRLipSync.VisemeCount;
67
68 [Range(0.0f, 1.0f)]
69 [Tooltip("Laughter probability threshold above which the laughter blendshape will be activated")]
70 public float laughterThreshold = 0.5f;
71
72 [Range(0.0f, 3.0f)]
73 [Tooltip("Laughter animation linear multiplier, the final output will be clamped to 1.0")]
74 public float laughterMultiplier = 1.5f;
75
76 // smoothing amount
77 [Range(1, 100)]
78 [Tooltip("Smoothing of 1 will yield only the current predicted viseme, 100 will yield an extremely smooth viseme response.")]
79 public int smoothAmount = 70;
80
81 // PRIVATE
82
83 // Look for a lip-sync Context (should be set at the same level as this component)
84 private OVRLipSyncContextBase lipsyncContext = null;
85
86
87 /// <summary>
88 /// Start this instance.
89 /// </summary>
90 void Start ()
91 {
92 // morph target needs to be set manually; possibly other components will need the same
93 if(skinnedMeshRenderer == null)
94 {
95 Debug.LogError("LipSyncContextMorphTarget.Start Error: " +
96 "Please set the target Skinned Mesh Renderer to be controlled!");
97 return;
98 }
99
100 // make sure there is a phoneme context assigned to this object
101 lipsyncContext = GetComponent<OVRLipSyncContextBase>();
102 if(lipsyncContext == null)
103 {
104 Debug.LogError("LipSyncContextMorphTarget.Start Error: " +
105 "No OVRLipSyncContext component on this object!");
106 }
107 else
108 {
109 // Send smoothing amount to context
110 lipsyncContext.Smoothing = smoothAmount;
111 }
112 }
113
114 /// <summary>
115 /// Update this instance.
116 /// </summary>
117 void Update ()
118 {
119 if((lipsyncContext != null) && (skinnedMeshRenderer != null))
120 {
121 // get the current viseme frame
122 OVRLipSync.Frame frame = lipsyncContext.GetCurrentPhonemeFrame();
123 if (frame != null)
124 {
125 SetVisemeToMorphTarget(frame);
126
127 SetLaughterToMorphTarget(frame);
128 }
129
130 // TEST visemes by capturing key inputs and sending a signal
131 CheckForKeys();
132
133 // Update smoothing value
134 if (smoothAmount != lipsyncContext.Smoothing)
135 {
136 lipsyncContext.Smoothing = smoothAmount;
137 }
138 }
139 }
140
141 /// <summary>
142 /// Sends the signals.
143 /// </summary>
144 void CheckForKeys()
145 {
146 if (enableVisemeTestKeys)
147 {
148 for (int i = 0; i < OVRLipSync.VisemeCount; ++i)
149 {
150 CheckVisemeKey(visemeTestKeys[i], i, 100);
151 }
152 }
153
154 CheckLaughterKey();
155 }
156
157 /// <summary>
158 /// Sets the viseme to morph target.
159 /// </summary>
160 void SetVisemeToMorphTarget(OVRLipSync.Frame frame)
161 {
162 for (int i = 0; i < visemeToBlendTargets.Length; i++)
163 {
164 if (visemeToBlendTargets[i] != -1)
165 {
166 // Viseme blend weights are in range of 0->1.0, we need to make range 100
167 skinnedMeshRenderer.SetBlendShapeWeight(
168 visemeToBlendTargets[i],
169 frame.Visemes[i] * 100.0f);
170 }
171 }
172 }
173
174 /// <summary>
175 /// Sets the laughter to morph target.
176 /// </summary>
177 void SetLaughterToMorphTarget(OVRLipSync.Frame frame)
178 {
179 if (laughterBlendTarget != -1)
180 {
181 // Laughter score will be raw classifier output in [0,1]
182 float laughterScore = frame.laughterScore;
183
184 // Threshold then re-map to [0,1]
185 laughterScore = laughterScore < laughterThreshold ? 0.0f : laughterScore - laughterThreshold;
186 laughterScore = Mathf.Min(laughterScore * laughterMultiplier, 1.0f);
187 laughterScore *= 1.0f / laughterThreshold;
188
189 skinnedMeshRenderer.SetBlendShapeWeight(
190 laughterBlendTarget,
191 laughterScore * 100.0f);
192 }
193 }
194
195 /// <summary>
196 /// Sends the viseme signal.
197 /// </summary>
198 /// <param name="key">Key.</param>
199 /// <param name="viseme">Viseme.</param>
200 /// <param name="arg1">Arg1.</param>
201 void CheckVisemeKey(KeyCode key, int viseme, int amount)
202 {
203 if (Input.GetKeyDown(key))
204 {
205 lipsyncContext.SetVisemeBlend(visemeToBlendTargets[viseme], amount);
206 }
207 if (Input.GetKeyUp(key))
208 {
209 lipsyncContext.SetVisemeBlend(visemeToBlendTargets[viseme], 0);
210 }
211 }
212
213 /// <summary>
214 /// Sends the laughter signal.
215 /// </summary>
216 void CheckLaughterKey()
217 {
218 if (Input.GetKeyDown(laughterKey))
219 {
220 lipsyncContext.SetLaughterBlend(100);
221 }
222 if (Input.GetKeyUp(laughterKey))
223 {
224 lipsyncContext.SetLaughterBlend(0);
225 }
226 }
227}