BlueGit

blue-star ggj-2020

Files

ggj-2020 / Assets / Oculus / VR / Scripts / Composition / OVRExternalComposition.cs

OVRExternalComposition.cs

Project Import
timfa Tim Falken · pushed by timfa · 6 years ago
b77834a1 History
OVRExternalComposition.cs 14.9 KiB Download Raw
1/************************************************************************************
2Copyright : Copyright (c) Facebook Technologies, LLC and its affiliates. All rights reserved.
3
4Licensed under the Oculus Utilities SDK License Version 1.31 (the "License"); you may not use
5the Utilities SDK except in compliance with the License, which is provided at the time of installation
6or download, or which otherwise accompanies this software in either electronic or hard copy form.
7
8You may obtain a copy of the License at
9https://developer.oculus.com/licenses/utilities-1.31
10
11Unless required by applicable law or agreed to in writing, the Utilities SDK distributed
12under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
13ANY KIND, either express or implied. See the License for the specific language governing
14permissions and limitations under the License.
15************************************************************************************/
16
17#if UNITY_ANDROID && !UNITY_EDITOR
18#define OVR_ANDROID_MRC
19#endif
20
21using UnityEngine;
22using System.Collections.Generic;
23using System.Threading;
24
25#if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN || UNITY_ANDROID
26
27public class OVRExternalComposition : OVRComposition
28{
29 public GameObject foregroundCameraGameObject;
30 public Camera foregroundCamera;
31 public GameObject backgroundCameraGameObject;
32 public Camera backgroundCamera;
33 public GameObject cameraProxyPlane;
34#if OVR_ANDROID_MRC
35 public AudioListener audioListener;
36 public OVRMRAudioFilter audioFilter;
37 public RenderTexture[] mrcRenderTextureArray = new RenderTexture[2];
38 public int frameIndex;
39 public int lastMrcEncodeFrameSyncId;
40#endif
41
42 public override OVRManager.CompositionMethod CompositionMethod() { return OVRManager.CompositionMethod.External; }
43
44 public OVRExternalComposition(GameObject parentObject, Camera mainCamera)
45 : base(parentObject, mainCamera)
46 {
47#if OVR_ANDROID_MRC
48 int frameWidth;
49 int frameHeight;
50 OVRPlugin.Media.GetMrcFrameSize(out frameWidth, out frameHeight);
51 Debug.LogFormat("[OVRExternalComposition] Create render texture {0}, {1}", frameWidth, frameHeight);
52 for (int i=0; i<2; ++i)
53 {
54 mrcRenderTextureArray[i] = new RenderTexture(frameWidth, frameHeight, 24, RenderTextureFormat.ARGB32);
55 mrcRenderTextureArray[i].Create();
56 }
57
58 frameIndex = 0;
59 lastMrcEncodeFrameSyncId = -1;
60#endif
61
62 Debug.Assert(backgroundCameraGameObject == null);
63 backgroundCameraGameObject = new GameObject();
64 backgroundCameraGameObject.name = "MRBackgroundCamera";
65 backgroundCameraGameObject.transform.parent = cameraInTrackingSpace ? cameraRig.trackingSpace : parentObject.transform;
66 backgroundCamera = backgroundCameraGameObject.AddComponent<Camera>();
67 backgroundCamera.stereoTargetEye = StereoTargetEyeMask.None;
68 backgroundCamera.depth = 99990.0f;
69 backgroundCamera.rect = new Rect(0.0f, 0.0f, 0.5f, 1.0f);
70 backgroundCamera.clearFlags = mainCamera.clearFlags;
71 backgroundCamera.backgroundColor = mainCamera.backgroundColor;
72 backgroundCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
73 backgroundCamera.nearClipPlane = mainCamera.nearClipPlane;
74 backgroundCamera.farClipPlane = mainCamera.farClipPlane;
75#if OVR_ANDROID_MRC
76 backgroundCamera.targetTexture = mrcRenderTextureArray[0];
77#endif
78
79 Debug.Assert(foregroundCameraGameObject == null);
80 foregroundCameraGameObject = new GameObject();
81 foregroundCameraGameObject.name = "MRForgroundCamera";
82 foregroundCameraGameObject.transform.parent = cameraInTrackingSpace ? cameraRig.trackingSpace : parentObject.transform;
83 foregroundCamera = foregroundCameraGameObject.AddComponent<Camera>();
84 foregroundCamera.stereoTargetEye = StereoTargetEyeMask.None;
85 foregroundCamera.depth = backgroundCamera.depth + 1.0f; // enforce the forground be rendered after the background
86 foregroundCamera.rect = new Rect(0.5f, 0.0f, 0.5f, 1.0f);
87 foregroundCamera.clearFlags = CameraClearFlags.Color;
88 foregroundCamera.backgroundColor = OVRManager.instance.externalCompositionBackdropColor;
89 foregroundCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
90 foregroundCamera.nearClipPlane = mainCamera.nearClipPlane;
91 foregroundCamera.farClipPlane = mainCamera.farClipPlane;
92#if OVR_ANDROID_MRC
93 foregroundCamera.targetTexture = mrcRenderTextureArray[0];
94#endif
95
96 // Create cameraProxyPlane for clipping
97 Debug.Assert(cameraProxyPlane == null);
98 cameraProxyPlane = GameObject.CreatePrimitive(PrimitiveType.Quad);
99 cameraProxyPlane.name = "MRProxyClipPlane";
100 cameraProxyPlane.transform.parent = cameraInTrackingSpace ? cameraRig.trackingSpace : parentObject.transform;
101 cameraProxyPlane.GetComponent<Collider>().enabled = false;
102 cameraProxyPlane.GetComponent<MeshRenderer>().shadowCastingMode = UnityEngine.Rendering.ShadowCastingMode.Off;
103 Material clipMaterial = new Material(Shader.Find("Oculus/OVRMRClipPlane"));
104 cameraProxyPlane.GetComponent<MeshRenderer>().material = clipMaterial;
105 clipMaterial.SetColor("_Color", OVRManager.instance.externalCompositionBackdropColor);
106 clipMaterial.SetFloat("_Visible", 0.0f);
107 cameraProxyPlane.transform.localScale = new Vector3(1000, 1000, 1000);
108 cameraProxyPlane.SetActive(true);
109 OVRMRForegroundCameraManager foregroundCameraManager = foregroundCameraGameObject.AddComponent<OVRMRForegroundCameraManager>();
110 foregroundCameraManager.composition = this;
111 foregroundCameraManager.clipPlaneGameObj = cameraProxyPlane;
112 }
113
114#if OVR_ANDROID_MRC
115 private void RefreshAudioFilter()
116 {
117 if (cameraRig != null && (audioListener == null || !audioListener.enabled || !audioListener.gameObject.activeInHierarchy))
118 {
119 CleanupAudioFilter();
120
121 AudioListener tmpAudioListener = cameraRig.centerEyeAnchor.gameObject.activeInHierarchy ? cameraRig.centerEyeAnchor.GetComponent<AudioListener>() : null;
122 if (tmpAudioListener != null && !tmpAudioListener.enabled) tmpAudioListener = null;
123 if (tmpAudioListener == null)
124 {
125 if (Camera.main != null && Camera.main.gameObject.activeInHierarchy)
126 {
127 tmpAudioListener = Camera.main.GetComponent<AudioListener>();
128 if (tmpAudioListener != null && !tmpAudioListener.enabled) tmpAudioListener = null;
129 }
130 }
131 if (tmpAudioListener == null)
132 {
133 Object[] allListeners = Object.FindObjectsOfType<AudioListener>();
134 foreach (var l in allListeners)
135 {
136 AudioListener al = l as AudioListener;
137 if (al != null && al.enabled && al.gameObject.activeInHierarchy)
138 {
139 tmpAudioListener = al;
140 break;
141 }
142 }
143 }
144 if (tmpAudioListener == null)
145 {
146 Debug.LogWarning("[OVRExternalComposition] No AudioListener in scene");
147 }
148 else
149 {
150 Debug.LogFormat("[OVRExternalComposition] AudioListener found, obj {0}", tmpAudioListener.gameObject.name);
151 }
152 audioListener = tmpAudioListener;
153
154 audioFilter = audioListener.gameObject.AddComponent<OVRMRAudioFilter>();
155 audioFilter.composition = this;
156 Debug.LogFormat("OVRMRAudioFilter added");
157 }
158 }
159
160 private int CastMrcFrame(int castTextureIndex)
161 {
162 float[] audioData;
163 int audioChannels;
164 GetAndResetAudioData(out audioData, out audioChannels);
165
166 int syncId = -1;
167 //Debug.Log("EncodeFrameThreadObject EncodeMrcFrame");
168 bool ret = false;
169 if (OVRPlugin.Media.GetMrcInputVideoBufferType() == OVRPlugin.Media.InputVideoBufferType.TextureHandle)
170 {
171 ret = OVRPlugin.Media.EncodeMrcFrame(mrcRenderTextureArray[castTextureIndex].GetNativeTexturePtr(), audioData, audioChannels, AudioSettings.dspTime, ref syncId);
172 }
173 else
174 {
175 ret = OVRPlugin.Media.EncodeMrcFrame(mrcRenderTextureArray[castTextureIndex], audioData, audioChannels, AudioSettings.dspTime, ref syncId);
176 }
177
178 if (!ret)
179 {
180 Debug.LogWarning("EncodeMrcFrame failed. Likely caused by OBS plugin disconnection");
181 return -1;
182 }
183
184 return syncId;
185 }
186
187 private void SetCameraTargetTexture(int drawTextureIndex)
188 {
189 RenderTexture texture = mrcRenderTextureArray[drawTextureIndex];
190 if (backgroundCamera.targetTexture != texture)
191 {
192 backgroundCamera.targetTexture = texture;
193 }
194 if (foregroundCamera.targetTexture != texture)
195 {
196 foregroundCamera.targetTexture = texture;
197 }
198 }
199#endif
200
201
202 public override void Update(Camera mainCamera)
203 {
204 OVRPlugin.SetHandNodePoseStateLatency(0.0); // the HandNodePoseStateLatency doesn't apply to the external composition. Always enforce it to 0.0
205
206#if OVR_ANDROID_MRC
207 RefreshAudioFilter();
208
209 int drawTextureIndex = (frameIndex / 2) % 2;
210 int castTextureIndex = 1 - drawTextureIndex;
211
212 backgroundCamera.enabled = (frameIndex % 2) == 0;
213 foregroundCamera.enabled = (frameIndex % 2) == 1;
214
215 if (frameIndex % 2 == 0)
216 {
217 if (lastMrcEncodeFrameSyncId != -1)
218 {
219 OVRPlugin.Media.SyncMrcFrame(lastMrcEncodeFrameSyncId);
220 lastMrcEncodeFrameSyncId = -1;
221 }
222 lastMrcEncodeFrameSyncId = CastMrcFrame(castTextureIndex);
223 SetCameraTargetTexture(drawTextureIndex);
224 }
225
226 ++ frameIndex;
227#endif
228
229 backgroundCamera.clearFlags = mainCamera.clearFlags;
230 backgroundCamera.backgroundColor = mainCamera.backgroundColor;
231 backgroundCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
232 backgroundCamera.nearClipPlane = mainCamera.nearClipPlane;
233 backgroundCamera.farClipPlane = mainCamera.farClipPlane;
234
235 foregroundCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
236 foregroundCamera.nearClipPlane = mainCamera.nearClipPlane;
237 foregroundCamera.farClipPlane = mainCamera.farClipPlane;
238
239 if (OVRMixedReality.useFakeExternalCamera || OVRPlugin.GetExternalCameraCount() == 0)
240 {
241 OVRPose worldSpacePose = new OVRPose();
242 OVRPose trackingSpacePose = new OVRPose();
243 trackingSpacePose.position = OVRManager.instance.trackingOriginType == OVRManager.TrackingOrigin.EyeLevel ?
244 OVRMixedReality.fakeCameraEyeLevelPosition :
245 OVRMixedReality.fakeCameraFloorLevelPosition;
246 trackingSpacePose.orientation = OVRMixedReality.fakeCameraRotation;
247 worldSpacePose = OVRExtensions.ToWorldSpacePose(trackingSpacePose);
248
249 backgroundCamera.fieldOfView = OVRMixedReality.fakeCameraFov;
250 backgroundCamera.aspect = OVRMixedReality.fakeCameraAspect;
251 foregroundCamera.fieldOfView = OVRMixedReality.fakeCameraFov;
252 foregroundCamera.aspect = OVRMixedReality.fakeCameraAspect;
253
254 if (cameraInTrackingSpace)
255 {
256 backgroundCamera.transform.FromOVRPose(trackingSpacePose, true);
257 foregroundCamera.transform.FromOVRPose(trackingSpacePose, true);
258 }
259 else
260 {
261 backgroundCamera.transform.FromOVRPose(worldSpacePose);
262 foregroundCamera.transform.FromOVRPose(worldSpacePose);
263 }
264 }
265 else
266 {
267 OVRPlugin.CameraExtrinsics extrinsics;
268 OVRPlugin.CameraIntrinsics intrinsics;
269 OVRPlugin.Posef calibrationRawPose;
270
271 // So far, only support 1 camera for MR and always use camera index 0
272 if (OVRPlugin.GetMixedRealityCameraInfo(0, out extrinsics, out intrinsics, out calibrationRawPose))
273 {
274 float fovY = Mathf.Atan(intrinsics.FOVPort.UpTan) * Mathf.Rad2Deg * 2;
275 float aspect = intrinsics.FOVPort.LeftTan / intrinsics.FOVPort.UpTan;
276 backgroundCamera.fieldOfView = fovY;
277 backgroundCamera.aspect = aspect;
278 foregroundCamera.fieldOfView = fovY;
279 foregroundCamera.aspect = intrinsics.FOVPort.LeftTan / intrinsics.FOVPort.UpTan;
280
281 if (cameraInTrackingSpace)
282 {
283 OVRPose trackingSpacePose = ComputeCameraTrackingSpacePose(extrinsics, calibrationRawPose);
284 backgroundCamera.transform.FromOVRPose(trackingSpacePose, true);
285 foregroundCamera.transform.FromOVRPose(trackingSpacePose, true);
286 }
287 else
288 {
289 OVRPose worldSpacePose = ComputeCameraWorldSpacePose(extrinsics, calibrationRawPose);
290 backgroundCamera.transform.FromOVRPose(worldSpacePose);
291 foregroundCamera.transform.FromOVRPose(worldSpacePose);
292 }
293 }
294 else
295 {
296 Debug.LogError("Failed to get external camera information");
297 return;
298 }
299 }
300
301 // Assume player always standing straightly
302 Vector3 externalCameraToHeadXZ = mainCamera.transform.position - foregroundCamera.transform.position;
303 externalCameraToHeadXZ.y = 0;
304 cameraProxyPlane.transform.position = mainCamera.transform.position;
305 cameraProxyPlane.transform.LookAt(cameraProxyPlane.transform.position + externalCameraToHeadXZ);
306 }
307
308#if OVR_ANDROID_MRC
309 private void CleanupAudioFilter()
310 {
311 if (audioFilter)
312 {
313 audioFilter.composition = null;
314 Object.Destroy(audioFilter);
315 Debug.LogFormat("OVRMRAudioFilter destroyed");
316 audioFilter = null;
317 }
318
319 }
320#endif
321
322 public override void Cleanup()
323 {
324 OVRCompositionUtil.SafeDestroy(ref backgroundCameraGameObject);
325 backgroundCamera = null;
326 OVRCompositionUtil.SafeDestroy(ref foregroundCameraGameObject);
327 foregroundCamera = null;
328 OVRCompositionUtil.SafeDestroy(ref cameraProxyPlane);
329 Debug.Log("ExternalComposition deactivated");
330
331#if OVR_ANDROID_MRC
332 if (lastMrcEncodeFrameSyncId != -1)
333 {
334 OVRPlugin.Media.SyncMrcFrame(lastMrcEncodeFrameSyncId);
335 lastMrcEncodeFrameSyncId = -1;
336 }
337
338 CleanupAudioFilter();
339
340 for (int i=0; i<2; ++i)
341 {
342 mrcRenderTextureArray[i].Release();
343 mrcRenderTextureArray[i] = null;
344 }
345
346 frameIndex = 0;
347#endif
348 }
349
350 private readonly object audioDataLock = new object();
351 private List<float> cachedAudioData = new List<float>();
352 private int cachedChannels = 0;
353
354 public void CacheAudioData(float[] data, int channels)
355 {
356 lock(audioDataLock)
357 {
358 if (channels != cachedChannels)
359 {
360 cachedAudioData.Clear();
361 }
362 cachedChannels = channels;
363 cachedAudioData.AddRange(data);
364 //Debug.LogFormat("[CacheAudioData] dspTime {0} indata {1} channels {2} accu_len {3}", AudioSettings.dspTime, data.Length, channels, cachedAudioData.Count);
365 }
366 }
367
368 public void GetAndResetAudioData(out float[] audioData, out int channels)
369 {
370 lock(audioDataLock)
371 {
372 //Debug.LogFormat("[GetAndResetAudioData] dspTime {0} accu_len {1}", AudioSettings.dspTime, cachedAudioData.Count);
373 audioData = new float[cachedAudioData.Count];
374 cachedAudioData.CopyTo(audioData);
375 channels = cachedChannels;
376 cachedAudioData.Clear();
377 }
378 }
379
380}
381
382/// <summary>
383/// Helper internal class for foregroundCamera, don't call it outside
384/// </summary>
385internal class OVRMRForegroundCameraManager : MonoBehaviour
386{
387 public OVRExternalComposition composition;
388 public GameObject clipPlaneGameObj;
389 private Material clipPlaneMaterial;
390 void OnPreRender()
391 {
392 // the clipPlaneGameObj should be only visible to foreground camera
393 if (clipPlaneGameObj)
394 {
395 if (clipPlaneMaterial == null)
396 clipPlaneMaterial = clipPlaneGameObj.GetComponent<MeshRenderer>().material;
397 clipPlaneGameObj.GetComponent<MeshRenderer>().material.SetFloat("_Visible", 1.0f);
398 }
399 }
400 void OnPostRender()
401 {
402 if (clipPlaneGameObj)
403 {
404 Debug.Assert(clipPlaneMaterial);
405 clipPlaneGameObj.GetComponent<MeshRenderer>().material.SetFloat("_Visible", 0.0f);
406 }
407 }
408}
409
410#if OVR_ANDROID_MRC
411
412public class OVRMRAudioFilter : MonoBehaviour
413{
414 private bool running = false;
415
416 public OVRExternalComposition composition;
417
418 void Start()
419 {
420 running = true;
421 }
422
423 void OnAudioFilterRead(float[] data, int channels)
424 {
425 if (!running)
426 return;
427
428 if (composition != null)
429 {
430 composition.CacheAudioData(data, channels);
431 }
432 }
433}
434#endif
435
436#endif