BlueGit

blue-star ggj-2020

Files

ggj-2020 / Assets / Oculus / VR / Scripts / Composition / OVRSandwichComposition.cs

OVRSandwichComposition.cs

Project Import
timfa Tim Falken · pushed by timfa · 6 years ago
b77834a1 History
OVRSandwichComposition.cs 14.9 KiB Download Raw
1/************************************************************************************
2Copyright : Copyright (c) Facebook Technologies, LLC and its affiliates. All rights reserved.
3
4Licensed under the Oculus Utilities SDK License Version 1.31 (the "License"); you may not use
5the Utilities SDK except in compliance with the License, which is provided at the time of installation
6or download, or which otherwise accompanies this software in either electronic or hard copy form.
7
8You may obtain a copy of the License at
9https://developer.oculus.com/licenses/utilities-1.31
10
11Unless required by applicable law or agreed to in writing, the Utilities SDK distributed
12under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
13ANY KIND, either express or implied. See the License for the specific language governing
14permissions and limitations under the License.
15************************************************************************************/
16
17using UnityEngine;
18using UnityEngine.Rendering;
19
20#if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN
21
22public class OVRSandwichComposition : OVRCameraComposition
23{
24 public float frameRealtime;
25
26 public Camera fgCamera;
27 public Camera bgCamera;
28
29 public class HistoryRecord
30 {
31 public float timestamp = float.MinValue;
32 public RenderTexture fgRenderTexture;
33 public RenderTexture bgRenderTexture;
34 public RenderTexture boundaryMeshMaskTexture;
35 }
36
37 public readonly int historyRecordCount = 8; // enough to compensate 88ms latency @ 90 fps
38 public readonly HistoryRecord[] historyRecordArray;
39 public int historyRecordCursorIndex = 0;
40
41 public GameObject cameraProxyPlane;
42
43 public Camera compositionCamera;
44 public OVRSandwichCompositionManager compositionManager;
45
46 private int _cameraFramePlaneLayer = -1;
47
48 // find an unnamed layer between 24..29
49 public int cameraFramePlaneLayer
50 {
51 get
52 {
53 if (_cameraFramePlaneLayer < 0)
54 {
55 for (int i=24; i<=29; ++i)
56 {
57 string layerName = LayerMask.LayerToName(i);
58 if (layerName == null || layerName.Length == 0)
59 {
60 _cameraFramePlaneLayer = i;
61 break;
62 }
63 }
64 if (_cameraFramePlaneLayer == -1)
65 {
66 Debug.LogWarning("Unable to find an unnamed layer between 24 and 29.");
67 _cameraFramePlaneLayer = 25;
68 }
69 Debug.LogFormat("Set the CameraFramePlaneLayer in SandwichComposition to {0}. Please do NOT put any other gameobject in this layer.", _cameraFramePlaneLayer);
70 }
71 return _cameraFramePlaneLayer;
72 }
73 }
74
75 public override OVRManager.CompositionMethod CompositionMethod() { return OVRManager.CompositionMethod.Sandwich; }
76
77 public OVRSandwichComposition(GameObject parentObject, Camera mainCamera, OVRManager.CameraDevice cameraDevice, bool useDynamicLighting, OVRManager.DepthQuality depthQuality)
78 : base(parentObject, mainCamera, cameraDevice, useDynamicLighting, depthQuality)
79 {
80 frameRealtime = Time.realtimeSinceStartup;
81
82 historyRecordCount = OVRManager.instance.sandwichCompositionBufferedFrames;
83 if (historyRecordCount < 1)
84 {
85 Debug.LogWarning("Invalid sandwichCompositionBufferedFrames in OVRManager. It should be at least 1");
86 historyRecordCount = 1;
87 }
88 if (historyRecordCount > 16)
89 {
90 Debug.LogWarning("The value of sandwichCompositionBufferedFrames in OVRManager is too big. It would consume a lot of memory. It has been override to 16");
91 historyRecordCount = 16;
92 }
93 historyRecordArray = new HistoryRecord[historyRecordCount];
94 for (int i=0; i<historyRecordCount; ++i)
95 {
96 historyRecordArray[i] = new HistoryRecord();
97 }
98 historyRecordCursorIndex = 0;
99
100 GameObject fgObject = new GameObject("MRSandwichForegroundCamera");
101 fgObject.transform.parent = cameraInTrackingSpace ? cameraRig.trackingSpace : parentObject.transform;
102 fgCamera = fgObject.AddComponent<Camera>();
103 fgCamera.depth = 200;
104 fgCamera.clearFlags = CameraClearFlags.SolidColor;
105 fgCamera.backgroundColor = Color.clear;
106 fgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
107 fgCamera.nearClipPlane = mainCamera.nearClipPlane;
108 fgCamera.farClipPlane = mainCamera.farClipPlane;
109
110 GameObject bgObject = new GameObject("MRSandwichBackgroundCamera");
111 bgObject.transform.parent = cameraInTrackingSpace ? cameraRig.trackingSpace : parentObject.transform;
112 bgCamera = bgObject.AddComponent<Camera>();
113 bgCamera.depth = 100;
114 bgCamera.clearFlags = mainCamera.clearFlags;
115 bgCamera.backgroundColor = mainCamera.backgroundColor;
116 bgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
117 bgCamera.nearClipPlane = mainCamera.nearClipPlane;
118 bgCamera.farClipPlane = mainCamera.farClipPlane;
119
120 // Create cameraProxyPlane for clipping
121 Debug.Assert(cameraProxyPlane == null);
122 cameraProxyPlane = GameObject.CreatePrimitive(PrimitiveType.Quad);
123 cameraProxyPlane.name = "MRProxyClipPlane";
124 cameraProxyPlane.transform.parent = cameraInTrackingSpace ? cameraRig.trackingSpace : parentObject.transform;
125 cameraProxyPlane.GetComponent<Collider>().enabled = false;
126 cameraProxyPlane.GetComponent<MeshRenderer>().shadowCastingMode = UnityEngine.Rendering.ShadowCastingMode.Off;
127 Material clipMaterial = new Material(Shader.Find("Oculus/OVRMRClipPlane"));
128 cameraProxyPlane.GetComponent<MeshRenderer>().material = clipMaterial;
129 clipMaterial.SetColor("_Color", Color.clear);
130 clipMaterial.SetFloat("_Visible", 0.0f);
131 cameraProxyPlane.transform.localScale = new Vector3(1000, 1000, 1000);
132 cameraProxyPlane.SetActive(true);
133 OVRMRForegroundCameraManager foregroundCameraManager = fgCamera.gameObject.AddComponent<OVRMRForegroundCameraManager>();
134 foregroundCameraManager.clipPlaneGameObj = cameraProxyPlane;
135
136 GameObject compositionCameraObject = new GameObject("MRSandwichCaptureCamera");
137 compositionCameraObject.transform.parent = cameraInTrackingSpace ? cameraRig.trackingSpace : parentObject.transform;
138 compositionCamera = compositionCameraObject.AddComponent<Camera>();
139 compositionCamera.stereoTargetEye = StereoTargetEyeMask.None;
140 compositionCamera.depth = float.MaxValue;
141 compositionCamera.rect = new Rect(0.0f, 0.0f, 1.0f, 1.0f);
142 compositionCamera.clearFlags = CameraClearFlags.Depth;
143 compositionCamera.backgroundColor = mainCamera.backgroundColor;
144 compositionCamera.cullingMask = 1 << cameraFramePlaneLayer;
145 compositionCamera.nearClipPlane = mainCamera.nearClipPlane;
146 compositionCamera.farClipPlane = mainCamera.farClipPlane;
147
148 if (!hasCameraDeviceOpened)
149 {
150 Debug.LogError("Unable to open camera device " + cameraDevice);
151 }
152 else
153 {
154 Debug.Log("SandwichComposition activated : useDynamicLighting " + (useDynamicLighting ? "ON" : "OFF"));
155 CreateCameraFramePlaneObject(parentObject, compositionCamera, useDynamicLighting);
156 cameraFramePlaneObject.layer = cameraFramePlaneLayer;
157 RefreshRenderTextures(mainCamera);
158 compositionManager = compositionCamera.gameObject.AddComponent<OVRSandwichCompositionManager>();
159 compositionManager.fgTexture = historyRecordArray[historyRecordCursorIndex].fgRenderTexture;
160 compositionManager.bgTexture = historyRecordArray[historyRecordCursorIndex].bgRenderTexture;
161 }
162 }
163
164 public override void Update(Camera mainCamera)
165 {
166 if (!hasCameraDeviceOpened)
167 {
168 return;
169 }
170
171 frameRealtime = Time.realtimeSinceStartup;
172
173 ++historyRecordCursorIndex;
174 if (historyRecordCursorIndex >= historyRecordCount)
175 {
176 historyRecordCursorIndex = 0;
177 }
178
179 if (!OVRPlugin.SetHandNodePoseStateLatency(OVRManager.instance.handPoseStateLatency))
180 {
181 Debug.LogWarning("HandPoseStateLatency is invalid. Expect a value between 0.0 to 0.5, get " + OVRManager.instance.handPoseStateLatency);
182 }
183
184 RefreshRenderTextures(mainCamera);
185
186 bgCamera.clearFlags = mainCamera.clearFlags;
187 bgCamera.backgroundColor = mainCamera.backgroundColor;
188 bgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
189
190 fgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers);
191
192 if (OVRMixedReality.useFakeExternalCamera || OVRPlugin.GetExternalCameraCount() == 0)
193 {
194 OVRPose cameraPose = new OVRPose();
195 OVRPose trackingSpacePose = new OVRPose();
196 trackingSpacePose.position = OVRManager.instance.trackingOriginType == OVRManager.TrackingOrigin.EyeLevel ?
197 OVRMixedReality.fakeCameraEyeLevelPosition :
198 OVRMixedReality.fakeCameraFloorLevelPosition;
199 trackingSpacePose.orientation = OVRMixedReality.fakeCameraRotation;
200
201 if (!cameraInTrackingSpace)
202 {
203 cameraPose = OVRExtensions.ToWorldSpacePose(trackingSpacePose);
204 }
205
206 RefreshCameraPoses(OVRMixedReality.fakeCameraFov, OVRMixedReality.fakeCameraAspect, cameraPose);
207 }
208 else
209 {
210 OVRPlugin.CameraExtrinsics extrinsics;
211 OVRPlugin.CameraIntrinsics intrinsics;
212 OVRPlugin.Posef calibrationRawPose;
213
214 // So far, only support 1 camera for MR and always use camera index 0
215 if (OVRPlugin.GetMixedRealityCameraInfo(0, out extrinsics, out intrinsics, out calibrationRawPose))
216 {
217 OVRPose cameraPose = cameraInTrackingSpace ? ComputeCameraTrackingSpacePose(extrinsics, calibrationRawPose) : ComputeCameraWorldSpacePose(extrinsics, calibrationRawPose);
218
219 float fovY = Mathf.Atan(intrinsics.FOVPort.UpTan) * Mathf.Rad2Deg * 2;
220 float aspect = intrinsics.FOVPort.LeftTan / intrinsics.FOVPort.UpTan;
221
222 RefreshCameraPoses(fovY, aspect, cameraPose);
223 }
224 else
225 {
226 Debug.LogWarning("Failed to get external camera information");
227 }
228 }
229
230 compositionCamera.GetComponent<OVRCameraFrameCompositionManager>().boundaryMeshMaskTexture = historyRecordArray[historyRecordCursorIndex].boundaryMeshMaskTexture;
231 HistoryRecord record = GetHistoryRecordForComposition();
232 UpdateCameraFramePlaneObject(mainCamera, compositionCamera, record.boundaryMeshMaskTexture);
233 OVRSandwichCompositionManager compositionManager = compositionCamera.gameObject.GetComponent<OVRSandwichCompositionManager>();
234 compositionManager.fgTexture = record.fgRenderTexture;
235 compositionManager.bgTexture = record.bgRenderTexture;
236
237 cameraProxyPlane.transform.position = fgCamera.transform.position + fgCamera.transform.forward * cameraFramePlaneDistance;
238 cameraProxyPlane.transform.LookAt(cameraProxyPlane.transform.position + fgCamera.transform.forward);
239 }
240
241 public override void Cleanup()
242 {
243 base.Cleanup();
244
245 Camera[] cameras = { fgCamera, bgCamera, compositionCamera };
246
247 foreach (Camera c in cameras)
248 {
249 OVRCompositionUtil.SafeDestroy(c.gameObject);
250 }
251
252 fgCamera = null;
253 bgCamera = null;
254 compositionCamera = null;
255
256 Debug.Log("SandwichComposition deactivated");
257 }
258
259 private RenderTextureFormat DesiredRenderTextureFormat(RenderTextureFormat originalFormat)
260 {
261 if (originalFormat == RenderTextureFormat.RGB565)
262 {
263 return RenderTextureFormat.ARGB1555;
264 }
265 else if (originalFormat == RenderTextureFormat.RGB111110Float)
266 {
267 return RenderTextureFormat.ARGBHalf;
268 }
269 else
270 {
271 return originalFormat;
272 }
273 }
274
275 protected void RefreshRenderTextures(Camera mainCamera)
276 {
277 int width = Screen.width;
278 int height = Screen.height;
279 RenderTextureFormat format = mainCamera.targetTexture ? DesiredRenderTextureFormat(mainCamera.targetTexture.format) : RenderTextureFormat.ARGB32;
280 int depth = mainCamera.targetTexture ? mainCamera.targetTexture.depth : 24;
281
282 Debug.Assert(fgCamera != null && bgCamera != null);
283
284 HistoryRecord record = historyRecordArray[historyRecordCursorIndex];
285
286 record.timestamp = frameRealtime;
287
288 if (record.fgRenderTexture == null || record.fgRenderTexture.width != width || record.fgRenderTexture.height != height || record.fgRenderTexture.format != format || record.fgRenderTexture.depth != depth)
289 {
290 record.fgRenderTexture = new RenderTexture(width, height, depth, format);
291 record.fgRenderTexture.name = "Sandwich FG " + historyRecordCursorIndex.ToString();
292 }
293 fgCamera.targetTexture = record.fgRenderTexture;
294
295 if (record.bgRenderTexture == null || record.bgRenderTexture.width != width || record.bgRenderTexture.height != height || record.bgRenderTexture.format != format || record.bgRenderTexture.depth != depth)
296 {
297 record.bgRenderTexture = new RenderTexture(width, height, depth, format);
298 record.bgRenderTexture.name = "Sandwich BG " + historyRecordCursorIndex.ToString();
299 }
300 bgCamera.targetTexture = record.bgRenderTexture;
301
302 if (OVRManager.instance.virtualGreenScreenType != OVRManager.VirtualGreenScreenType.Off)
303 {
304 if (record.boundaryMeshMaskTexture == null || record.boundaryMeshMaskTexture.width != width || record.boundaryMeshMaskTexture.height != height)
305 {
306 record.boundaryMeshMaskTexture = new RenderTexture(width, height, 0, RenderTextureFormat.R8);
307 record.boundaryMeshMaskTexture.name = "Boundary Mask " + historyRecordCursorIndex.ToString();
308 record.boundaryMeshMaskTexture.Create();
309 }
310 }
311 else
312 {
313 record.boundaryMeshMaskTexture = null;
314 }
315
316 Debug.Assert(fgCamera.targetTexture != null && bgCamera.targetTexture != null && (OVRManager.instance.virtualGreenScreenType == OVRManager.VirtualGreenScreenType.Off || record.boundaryMeshMaskTexture != null));
317 }
318
319 protected HistoryRecord GetHistoryRecordForComposition()
320 {
321 float expectedTime = frameRealtime - OVRManager.instance.sandwichCompositionRenderLatency;
322 int currIndex = historyRecordCursorIndex;
323 int prevIndex = currIndex - 1;
324 if (prevIndex < 0)
325 {
326 prevIndex = historyRecordCount - 1;
327 }
328 while (prevIndex != historyRecordCursorIndex)
329 {
330 if (historyRecordArray[prevIndex].timestamp <= expectedTime)
331 {
332 float timeToCurrIndex = historyRecordArray[currIndex].timestamp - expectedTime;
333 float timeToPrevIndex = expectedTime - historyRecordArray[prevIndex].timestamp;
334 return timeToCurrIndex <= timeToPrevIndex ? historyRecordArray[currIndex] : historyRecordArray[prevIndex];
335 }
336 currIndex = prevIndex;
337 prevIndex = currIndex - 1;
338 if (prevIndex < 0) prevIndex = historyRecordCount - 1;
339 }
340 // return the earliest frame
341 return historyRecordArray[currIndex];
342 }
343
344 protected void RefreshCameraPoses(float fovY, float aspect, OVRPose pose)
345 {
346 Camera[] cameras = { fgCamera, bgCamera, compositionCamera };
347 foreach (Camera c in cameras)
348 {
349 c.fieldOfView = fovY;
350 c.aspect = aspect;
351 if (cameraInTrackingSpace)
352 {
353 c.transform.FromOVRPose(pose, true);
354 }
355 else
356 {
357 c.transform.FromOVRPose(pose);
358 }
359 }
360 }
361
362 public class OVRSandwichCompositionManager : MonoBehaviour
363 {
364 public RenderTexture fgTexture;
365 public RenderTexture bgTexture;
366 public Material alphaBlendMaterial;
367
368 void Start()
369 {
370 Shader alphaBlendShader = Shader.Find("Oculus/UnlitTransparent");
371 if (alphaBlendShader == null)
372 {
373 Debug.LogError("Unable to create transparent shader");
374 return;
375 }
376 alphaBlendMaterial = new Material(alphaBlendShader);
377 }
378
379 private void OnPreRender()
380 {
381 if (fgTexture == null || bgTexture == null || alphaBlendMaterial == null)
382 {
383 Debug.LogError("OVRSandwichCompositionManager has not setup properly");
384 return;
385 }
386
387 Graphics.Blit(bgTexture, RenderTexture.active);
388 }
389
390 void OnPostRender()
391 {
392 if (fgTexture == null || bgTexture == null || alphaBlendMaterial == null)
393 {
394 Debug.LogError("OVRSandwichCompositionManager has not setup properly");
395 return;
396 }
397
398 Graphics.Blit(fgTexture, RenderTexture.active, alphaBlendMaterial);
399 }
400 }
401}
402
403#endif