LeapServiceProvider.cs
17.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
/******************************************************************************
* Copyright (C) Leap Motion, Inc. 2011-2017. *
* Leap Motion proprietary and confidential. *
* *
* Use subject to the terms of the Leap Motion SDK Agreement available at *
* https://developer.leapmotion.com/sdk_agreement, or another agreement *
* between Leap Motion and you, your company or other organization. *
******************************************************************************/
using UnityEngine;
#if UNITY_EDITOR
using UnityEditor;
#endif
using System;
using System.Collections;
using Leap.Unity.Attributes;
//using Leap.Unity.Graphing;
namespace Leap.Unity {
/**LeapServiceProvider creates a Controller and supplies Leap Hands and images */
public class LeapServiceProvider : LeapProvider {
/** Conversion factor for nanoseconds to seconds. */
protected const double NS_TO_S = 1e-6;
/** Conversion factor for seconds to nanoseconds. */
protected const double S_TO_NS = 1e6;
/** Transform Array for Precull Latching **/
protected const string HAND_ARRAY = "_LeapHandTransforms";
public enum FrameOptimizationMode {
None,
ReuseUpdateForPhysics,
ReusePhysicsForUpdate,
}
public enum PhysicsExtrapolationMode {
None,
Auto,
Manual
}
[Tooltip("Set true if the Leap Motion hardware is mounted on an HMD; otherwise, leave false.")]
[SerializeField]
protected bool _isHeadMounted = false;
[SerializeField]
protected LeapVRTemporalWarping _temporalWarping;
[Tooltip("When enabled, the provider will only calculate one leap frame instead of two.")]
[SerializeField]
protected FrameOptimizationMode _frameOptimization = FrameOptimizationMode.None;
[Tooltip("The mode to use when extrapolating physics.\n" +
" None - No extrapolation is used at all.\n" +
" Auto - Extrapolation is chosen based on the fixed timestep.\n" +
" Manual - Extrapolation time is chosen manually by the user.")]
[SerializeField]
protected PhysicsExtrapolationMode _physicsExtrapolation = PhysicsExtrapolationMode.Auto;
[Tooltip("The amount of time (in seconds) to extrapolate the phyiscs data by.")]
[SerializeField]
protected float _physicsExtrapolationTime = 1.0f / 90.0f;
[Header("[Experimental]")]
[Tooltip("Pass updated transform matrices to objects with materials using the VertexOffsetShader.")]
[SerializeField]
protected bool _updateHandInPrecull = false;
protected bool _useInterpolation = true;
//Extrapolate on Android to compensate for the latency introduced by its graphics pipeline
#if UNITY_ANDROID && !UNITY_EDITOR
protected int ExtrapolationAmount = 15;
protected int BounceAmount = 70;
#else
protected int ExtrapolationAmount = 0;
protected int BounceAmount = 0;
#endif
protected Controller leap_controller_;
protected bool manualUpdateHasBeenCalledSinceUpdate;
protected Vector3 warpedPosition;
protected Quaternion warpedRotation;
protected SmoothedFloat _fixedOffset = new SmoothedFloat();
protected SmoothedFloat _smoothedTrackingLatency = new SmoothedFloat();
protected long _unityToLeapOffset;
protected Frame _untransformedUpdateFrame;
protected Frame _transformedUpdateFrame;
protected Frame _untransformedFixedFrame;
protected Frame _transformedFixedFrame;
protected Matrix4x4[] _transformArray = new Matrix4x4[2];
[NonSerialized]
public long imageTimeStamp = 0;
public override Frame CurrentFrame {
get {
if (_frameOptimization == FrameOptimizationMode.ReusePhysicsForUpdate) {
return _transformedFixedFrame;
} else {
return _transformedUpdateFrame;
}
}
}
public override Frame CurrentFixedFrame {
get {
if (_frameOptimization == FrameOptimizationMode.ReuseUpdateForPhysics) {
return _transformedUpdateFrame;
} else {
return _transformedFixedFrame;
}
}
}
protected bool UseInterpolation {
get {
return _useInterpolation;
}
set {
_useInterpolation = value;
}
}
public bool UpdateHandInPrecull {
get {
return _updateHandInPrecull;
}
set {
resetTransforms();
_updateHandInPrecull = value;
}
}
public float CalculatePhysicsExtrapolation() {
switch (_physicsExtrapolation) {
case PhysicsExtrapolationMode.None:
return 0;
case PhysicsExtrapolationMode.Auto:
return Time.fixedDeltaTime;
case PhysicsExtrapolationMode.Manual:
return _physicsExtrapolationTime;
default:
throw new InvalidOperationException("Unexpected physics extrapolation mode: " + _physicsExtrapolation);
}
}
/** Returns the Leap Controller instance. */
public Controller GetLeapController() {
#if UNITY_EDITOR
//Null check to deal with hot reloading
if (leap_controller_ == null) {
createController();
}
#endif
return leap_controller_;
}
/** True, if the Leap Motion hardware is plugged in and this application is connected to the Leap Motion service. */
public bool IsConnected() {
return GetLeapController().IsConnected;
}
/** Returns information describing the device hardware. */
public LeapDeviceInfo GetDeviceInfo() {
return LeapDeviceInfo.GetLeapDeviceInfo();
}
public void ReTransformFrames() {
transformFrame(_untransformedUpdateFrame, _transformedUpdateFrame);
transformFrame(_untransformedFixedFrame, _transformedFixedFrame);
}
protected virtual void Reset() {
if (checkShouldEnableHeadMounted()) {
_isHeadMounted = true;
}
_temporalWarping = GetComponentInParent<LeapVRTemporalWarping>();
_frameOptimization = FrameOptimizationMode.None;
_updateHandInPrecull = false;
}
protected virtual void Awake() {
_fixedOffset.delay = 0.4f;
_smoothedTrackingLatency.SetBlend(0.99f, 0.0111f);
}
protected virtual void Start() {
checkShouldEnableHeadMounted();
createController();
_transformedUpdateFrame = new Frame();
_transformedFixedFrame = new Frame();
_untransformedUpdateFrame = new Frame();
_untransformedFixedFrame = new Frame();
}
protected virtual void Update() {
#if UNITY_EDITOR
if (EditorApplication.isCompiling) {
EditorApplication.isPlaying = false;
Debug.LogWarning("Unity hot reloading not currently supported. Stopping Editor Playback.");
return;
}
#endif
manualUpdateHasBeenCalledSinceUpdate = false;
_fixedOffset.Update(Time.time - Time.fixedTime, Time.deltaTime);
if (_frameOptimization == FrameOptimizationMode.ReusePhysicsForUpdate) {
DispatchUpdateFrameEvent(_transformedFixedFrame);
return;
}
if (_useInterpolation) {
#if !UNITY_ANDROID || UNITY_EDITOR
_smoothedTrackingLatency.value = Mathf.Min(_smoothedTrackingLatency.value, 30000f);
_smoothedTrackingLatency.Update((float)(leap_controller_.Now() - leap_controller_.FrameTimestamp()), Time.deltaTime);
#endif
long timestamp = CalculateInterpolationTime() + (ExtrapolationAmount * 1000);
_unityToLeapOffset = timestamp - (long)(Time.time * S_TO_NS);
leap_controller_.GetInterpolatedFrameFromTime(_untransformedUpdateFrame, timestamp, CalculateInterpolationTime() - (BounceAmount * 1000));
} else {
leap_controller_.Frame(_untransformedUpdateFrame);
}
imageTimeStamp = leap_controller_.FrameTimestamp();
if (_untransformedUpdateFrame != null) {
transformFrame(_untransformedUpdateFrame, _transformedUpdateFrame);
DispatchUpdateFrameEvent(_transformedUpdateFrame);
}
}
protected virtual void FixedUpdate() {
if (_frameOptimization == FrameOptimizationMode.ReuseUpdateForPhysics) {
DispatchFixedFrameEvent(_transformedUpdateFrame);
return;
}
if (_useInterpolation) {
long timestamp;
switch (_frameOptimization) {
case FrameOptimizationMode.None:
//By default we use Time.fixedTime to ensure that our hands are on the same timeline
//as Update. We add an extrapolation value to help compensate for latency.
float extrapolatedTime = Time.fixedTime + CalculatePhysicsExtrapolation();
timestamp = (long)(extrapolatedTime * S_TO_NS) + _unityToLeapOffset;
break;
case FrameOptimizationMode.ReusePhysicsForUpdate:
//If we are re-using physics frames for update, we don't even want to care about
//Time.fixedTime, just grab the most recent interpolated timestamp like we are
//in Update
timestamp = CalculateInterpolationTime() + (ExtrapolationAmount * 1000);
break;
default:
throw new InvalidOperationException("Unexpected frame optimization mode: " + _frameOptimization);
}
leap_controller_.GetInterpolatedFrame(_untransformedFixedFrame, timestamp);
} else {
leap_controller_.Frame(_untransformedFixedFrame);
}
if (_untransformedFixedFrame != null) {
transformFrame(_untransformedFixedFrame, _transformedFixedFrame);
DispatchFixedFrameEvent(_transformedFixedFrame);
}
}
long CalculateInterpolationTime(bool endOfFrame = false) {
#if UNITY_ANDROID && !UNITY_EDITOR
return leap_controller_.Now() - 16000;
#else
if (leap_controller_ != null) {
return leap_controller_.Now() - (long)_smoothedTrackingLatency.value + (_updateHandInPrecull && !endOfFrame ? (long)(Time.smoothDeltaTime * S_TO_NS / Time.timeScale) : 0);
} else {
return 0;
}
#endif
}
protected virtual void OnDestroy() {
destroyController();
}
protected virtual void OnApplicationPause(bool isPaused) {
if (leap_controller_ != null) {
if (isPaused) {
leap_controller_.StopConnection();
} else {
leap_controller_.StartConnection();
}
}
}
protected virtual void OnApplicationQuit() {
destroyController();
}
protected virtual void OnEnable() {
Camera.onPreCull -= LateUpdateHandTransforms;
Camera.onPreCull += LateUpdateHandTransforms;
resetTransforms();
}
protected virtual void OnDisable() {
Camera.onPreCull -= LateUpdateHandTransforms;
resetTransforms();
}
private bool checkShouldEnableHeadMounted() {
if (XRSupportUtil.IsXREnabled()) {
var parentCamera = GetComponentInParent<Camera>();
if (parentCamera != null && parentCamera.stereoTargetEye != StereoTargetEyeMask.None) {
if (!_isHeadMounted) {
if (Application.isPlaying) {
Debug.LogError("VR is enabled and the LeapServiceProvider is the child of a "
+ "camera targeting one or both stereo eyes; You should "
+ "check the isHeadMounted option on the LeapServiceProvider "
+ "if the Leap is mounted or attached to your VR headset!",
this);
}
return true;
}
}
}
return false;
}
/*
* Resets the Global Hand Transform Shader Matrices
*/
protected void resetTransforms() {
_transformArray[0] = Matrix4x4.identity;
_transformArray[1] = Matrix4x4.identity;
Shader.SetGlobalMatrixArray(HAND_ARRAY, _transformArray);
}
/*
* Initializes the Leap Motion policy flags.
* The POLICY_OPTIMIZE_HMD flag improves tracking for head-mounted devices.
*/
protected void initializeFlags() {
if (leap_controller_ == null) {
return;
}
//Optimize for top-down tracking if on head mounted display.
if (_isHeadMounted) {
leap_controller_.SetPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_HMD);
} else {
leap_controller_.ClearPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_HMD);
}
}
/** Create an instance of a Controller, initialize its policy flags
* and subscribe to connection event */
protected void createController() {
if (leap_controller_ != null) {
destroyController();
}
leap_controller_ = new Controller();
if (leap_controller_.IsConnected) {
initializeFlags();
} else {
leap_controller_.Device += onHandControllerConnect;
}
}
/** Calling this method stop the connection for the existing instance of a Controller,
* clears old policy flags and resets to null */
protected void destroyController() {
if (leap_controller_ != null) {
if (leap_controller_.IsConnected) {
leap_controller_.ClearPolicy(Controller.PolicyFlag.POLICY_OPTIMIZE_HMD);
}
leap_controller_.StopConnection();
leap_controller_ = null;
}
}
protected void onHandControllerConnect(object sender, LeapEventArgs args) {
initializeFlags();
leap_controller_.Device -= onHandControllerConnect;
}
protected void transformFrame(Frame source, Frame dest, bool resampleTemporalWarping = true) {
LeapTransform leapTransform;
if (_temporalWarping != null) {
if (resampleTemporalWarping) {
_temporalWarping.TryGetWarpedTransform(LeapVRTemporalWarping.WarpedAnchor.CENTER, out warpedPosition, out warpedRotation, source.Timestamp);
warpedRotation = warpedRotation * transform.localRotation;
}
leapTransform = new LeapTransform(warpedPosition.ToVector(), warpedRotation.ToLeapQuaternion(), transform.lossyScale.ToVector() * 1e-3f);
leapTransform.MirrorZ();
} else {
leapTransform = transform.GetLeapMatrix();
}
dest.CopyFrom(source).Transform(leapTransform);
}
protected void transformHands(ref LeapTransform LeftHand, ref LeapTransform RightHand) {
LeapTransform leapTransform;
if (_temporalWarping != null) {
leapTransform = new LeapTransform(warpedPosition.ToVector(), warpedRotation.ToLeapQuaternion(), transform.lossyScale.ToVector() * 1e-3f);
leapTransform.MirrorZ();
} else {
leapTransform = transform.GetLeapMatrix();
}
LeftHand = new LeapTransform(leapTransform.TransformPoint(LeftHand.translation), leapTransform.TransformQuaternion(LeftHand.rotation));
RightHand = new LeapTransform(leapTransform.TransformPoint(RightHand.translation), leapTransform.TransformQuaternion(RightHand.rotation));
}
public void LateUpdateHandTransforms(Camera camera) {
if (_updateHandInPrecull) {
#if UNITY_EDITOR
//Hard-coded name of the camera used to generate the pre-render view
if (camera.gameObject.name == "PreRenderCamera") {
return;
}
bool isScenePreviewCamera = camera.gameObject.hideFlags == HideFlags.HideAndDontSave;
if (isScenePreviewCamera) {
return;
}
#endif
if (Application.isPlaying && !manualUpdateHasBeenCalledSinceUpdate && leap_controller_ != null) {
manualUpdateHasBeenCalledSinceUpdate = true;
//Find the Left and/or Right Hand(s) to Latch
Hand leftHand = null, rightHand = null;
LeapTransform PrecullLeftHand = LeapTransform.Identity, PrecullRightHand = LeapTransform.Identity;
for (int i = 0; i < CurrentFrame.Hands.Count; i++) {
Hand updateHand = CurrentFrame.Hands[i];
if (updateHand.IsLeft && leftHand == null) {
leftHand = updateHand;
} else if (updateHand.IsRight && rightHand == null) {
rightHand = updateHand;
}
}
//Determine their new Transforms
leap_controller_.GetInterpolatedLeftRightTransform(CalculateInterpolationTime() + (ExtrapolationAmount * 1000), CalculateInterpolationTime() - (BounceAmount * 1000), (leftHand != null ? leftHand.Id : 0), (rightHand != null ? rightHand.Id : 0), out PrecullLeftHand, out PrecullRightHand);
bool LeftValid = PrecullLeftHand.translation != Vector.Zero; bool RightValid = PrecullRightHand.translation != Vector.Zero;
transformHands(ref PrecullLeftHand, ref PrecullRightHand);
//Calculate the Delta Transforms
if (rightHand != null && RightValid) {
_transformArray[0] =
Matrix4x4.TRS(PrecullRightHand.translation.ToVector3(), PrecullRightHand.rotation.ToQuaternion(), Vector3.one) *
Matrix4x4.Inverse(Matrix4x4.TRS(rightHand.PalmPosition.ToVector3(), rightHand.Rotation.ToQuaternion(), Vector3.one));
}
if (leftHand != null && LeftValid) {
_transformArray[1] =
Matrix4x4.TRS(PrecullLeftHand.translation.ToVector3(), PrecullLeftHand.rotation.ToQuaternion(), Vector3.one) *
Matrix4x4.Inverse(Matrix4x4.TRS(leftHand.PalmPosition.ToVector3(), leftHand.Rotation.ToQuaternion(), Vector3.one));
}
//Apply inside of the vertex shader
Shader.SetGlobalMatrixArray(HAND_ARRAY, _transformArray);
}
}
}
}
}