| 1 | /************************************************************************************ |
| 2 | Filename : OVRLipSyncContextMorphTarget.cs |
| 3 | Content : This bridges the viseme output to the morph targets |
| 4 | Created : August 7th, 2015 |
| 5 | Copyright : Copyright Facebook Technologies, LLC and its affiliates. |
| 6 | All rights reserved. |
| 7 | |
| 8 | Licensed under the Oculus Audio SDK License Version 3.3 (the "License"); |
| 9 | you may not use the Oculus Audio SDK except in compliance with the License, |
| 10 | which is provided at the time of installation or download, or which |
| 11 | otherwise accompanies this software in either electronic or hard copy form. |
| 12 | |
| 13 | You may obtain a copy of the License at |
| 14 | |
| 15 | https://developer.oculus.com/licenses/audio-3.3/ |
| 16 | |
| 17 | Unless required by applicable law or agreed to in writing, the Oculus Audio SDK |
| 18 | distributed under the License is distributed on an "AS IS" BASIS, |
| 19 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 20 | See the License for the specific language governing permissions and |
| 21 | limitations under the License. |
| 22 | ************************************************************************************/ |
| 23 | using UnityEngine; |
| 24 | using System.Linq; |
| 25 | |
| 26 | public class OVRLipSyncContextMorphTarget : MonoBehaviour |
| 27 | { |
| 28 | // PUBLIC |
| 29 | |
| 30 | // Manually assign the skinned mesh renderer to this script |
| 31 | [Tooltip("Skinned Mesh Rendered target to be driven by Oculus Lipsync")] |
| 32 | public SkinnedMeshRenderer skinnedMeshRenderer = null; |
| 33 | |
| 34 | // Set the blendshape index to go to (-1 means there is not one assigned) |
| 35 | [Tooltip("Blendshape index to trigger for each viseme.")] |
| 36 | public int [] visemeToBlendTargets = Enumerable.Range(0, OVRLipSync.VisemeCount).ToArray(); |
| 37 | |
| 38 | // enable/disable sending signals to viseme engine |
| 39 | [Tooltip("Enable using the test keys defined below to manually trigger each viseme.")] |
| 40 | public bool enableVisemeTestKeys = false; |
| 41 | [Tooltip("Test keys used to manually trigger an individual viseme - by " + |
| 42 | "default the QWERTY row of a US keyboard.")] |
| 43 | public KeyCode[] visemeTestKeys = |
| 44 | { |
| 45 | KeyCode.BackQuote, |
| 46 | KeyCode.Tab, |
| 47 | KeyCode.Q, |
| 48 | KeyCode.W, |
| 49 | KeyCode.E, |
| 50 | KeyCode.R, |
| 51 | KeyCode.T, |
| 52 | KeyCode.Y, |
| 53 | KeyCode.U, |
| 54 | KeyCode.I, |
| 55 | KeyCode.O, |
| 56 | KeyCode.P, |
| 57 | KeyCode.LeftBracket, |
| 58 | KeyCode.RightBracket, |
| 59 | KeyCode.Backslash, |
| 60 | }; |
| 61 | |
| 62 | [Tooltip("Test key used to manually trigger laughter and visualise the results")] |
| 63 | public KeyCode laughterKey = KeyCode.CapsLock; |
| 64 | |
| 65 | [Tooltip("Blendshape index to trigger for laughter")] |
| 66 | public int laughterBlendTarget = OVRLipSync.VisemeCount; |
| 67 | |
| 68 | [Range(0.0f, 1.0f)] |
| 69 | [Tooltip("Laughter probability threshold above which the laughter blendshape will be activated")] |
| 70 | public float laughterThreshold = 0.5f; |
| 71 | |
| 72 | [Range(0.0f, 3.0f)] |
| 73 | [Tooltip("Laughter animation linear multiplier, the final output will be clamped to 1.0")] |
| 74 | public float laughterMultiplier = 1.5f; |
| 75 | |
| 76 | // smoothing amount |
| 77 | [Range(1, 100)] |
| 78 | [Tooltip("Smoothing of 1 will yield only the current predicted viseme, 100 will yield an extremely smooth viseme response.")] |
| 79 | public int smoothAmount = 70; |
| 80 | |
| 81 | // PRIVATE |
| 82 | |
| 83 | // Look for a lip-sync Context (should be set at the same level as this component) |
| 84 | private OVRLipSyncContextBase lipsyncContext = null; |
| 85 | |
| 86 | |
| 87 | /// <summary> |
| 88 | /// Start this instance. |
| 89 | /// </summary> |
| 90 | void Start () |
| 91 | { |
| 92 | // morph target needs to be set manually; possibly other components will need the same |
| 93 | if(skinnedMeshRenderer == null) |
| 94 | { |
| 95 | Debug.LogError("LipSyncContextMorphTarget.Start Error: " + |
| 96 | "Please set the target Skinned Mesh Renderer to be controlled!"); |
| 97 | return; |
| 98 | } |
| 99 | |
| 100 | // make sure there is a phoneme context assigned to this object |
| 101 | lipsyncContext = GetComponent<OVRLipSyncContextBase>(); |
| 102 | if(lipsyncContext == null) |
| 103 | { |
| 104 | Debug.LogError("LipSyncContextMorphTarget.Start Error: " + |
| 105 | "No OVRLipSyncContext component on this object!"); |
| 106 | } |
| 107 | else |
| 108 | { |
| 109 | // Send smoothing amount to context |
| 110 | lipsyncContext.Smoothing = smoothAmount; |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | /// <summary> |
| 115 | /// Update this instance. |
| 116 | /// </summary> |
| 117 | void Update () |
| 118 | { |
| 119 | if((lipsyncContext != null) && (skinnedMeshRenderer != null)) |
| 120 | { |
| 121 | // get the current viseme frame |
| 122 | OVRLipSync.Frame frame = lipsyncContext.GetCurrentPhonemeFrame(); |
| 123 | if (frame != null) |
| 124 | { |
| 125 | SetVisemeToMorphTarget(frame); |
| 126 | |
| 127 | SetLaughterToMorphTarget(frame); |
| 128 | } |
| 129 | |
| 130 | // TEST visemes by capturing key inputs and sending a signal |
| 131 | CheckForKeys(); |
| 132 | |
| 133 | // Update smoothing value |
| 134 | if (smoothAmount != lipsyncContext.Smoothing) |
| 135 | { |
| 136 | lipsyncContext.Smoothing = smoothAmount; |
| 137 | } |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | /// <summary> |
| 142 | /// Sends the signals. |
| 143 | /// </summary> |
| 144 | void CheckForKeys() |
| 145 | { |
| 146 | if (enableVisemeTestKeys) |
| 147 | { |
| 148 | for (int i = 0; i < OVRLipSync.VisemeCount; ++i) |
| 149 | { |
| 150 | CheckVisemeKey(visemeTestKeys[i], i, 100); |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | CheckLaughterKey(); |
| 155 | } |
| 156 | |
| 157 | /// <summary> |
| 158 | /// Sets the viseme to morph target. |
| 159 | /// </summary> |
| 160 | void SetVisemeToMorphTarget(OVRLipSync.Frame frame) |
| 161 | { |
| 162 | for (int i = 0; i < visemeToBlendTargets.Length; i++) |
| 163 | { |
| 164 | if (visemeToBlendTargets[i] != -1) |
| 165 | { |
| 166 | // Viseme blend weights are in range of 0->1.0, we need to make range 100 |
| 167 | skinnedMeshRenderer.SetBlendShapeWeight( |
| 168 | visemeToBlendTargets[i], |
| 169 | frame.Visemes[i] * 100.0f); |
| 170 | } |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | /// <summary> |
| 175 | /// Sets the laughter to morph target. |
| 176 | /// </summary> |
| 177 | void SetLaughterToMorphTarget(OVRLipSync.Frame frame) |
| 178 | { |
| 179 | if (laughterBlendTarget != -1) |
| 180 | { |
| 181 | // Laughter score will be raw classifier output in [0,1] |
| 182 | float laughterScore = frame.laughterScore; |
| 183 | |
| 184 | // Threshold then re-map to [0,1] |
| 185 | laughterScore = laughterScore < laughterThreshold ? 0.0f : laughterScore - laughterThreshold; |
| 186 | laughterScore = Mathf.Min(laughterScore * laughterMultiplier, 1.0f); |
| 187 | laughterScore *= 1.0f / laughterThreshold; |
| 188 | |
| 189 | skinnedMeshRenderer.SetBlendShapeWeight( |
| 190 | laughterBlendTarget, |
| 191 | laughterScore * 100.0f); |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | /// <summary> |
| 196 | /// Sends the viseme signal. |
| 197 | /// </summary> |
| 198 | /// <param name="key">Key.</param> |
| 199 | /// <param name="viseme">Viseme.</param> |
| 200 | /// <param name="arg1">Arg1.</param> |
| 201 | void CheckVisemeKey(KeyCode key, int viseme, int amount) |
| 202 | { |
| 203 | if (Input.GetKeyDown(key)) |
| 204 | { |
| 205 | lipsyncContext.SetVisemeBlend(visemeToBlendTargets[viseme], amount); |
| 206 | } |
| 207 | if (Input.GetKeyUp(key)) |
| 208 | { |
| 209 | lipsyncContext.SetVisemeBlend(visemeToBlendTargets[viseme], 0); |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | /// <summary> |
| 214 | /// Sends the laughter signal. |
| 215 | /// </summary> |
| 216 | void CheckLaughterKey() |
| 217 | { |
| 218 | if (Input.GetKeyDown(laughterKey)) |
| 219 | { |
| 220 | lipsyncContext.SetLaughterBlend(100); |
| 221 | } |
| 222 | if (Input.GetKeyUp(laughterKey)) |
| 223 | { |
| 224 | lipsyncContext.SetLaughterBlend(0); |
| 225 | } |
| 226 | } |
| 227 | } |