aframe-desktop-xr-hands - npm Package Compare versions

Comparing version 0.0.1 to 0.0.2

package.json

 {
   "name": "aframe-desktop-xr-hands",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "Desktop simulation of WebXR Hand Tracking using Mediapipe",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

@@ -120,2 +120,16 @@ # desktop-xr-hands
 
-`desktop-xr-hands` takes care of mapping Mediapipe reported positions, to WebXR API positions and orientations.
+`desktop-xr-hands` takes care of mapping Mediapipe reported positions, to WebXR API positions and orientations.
+
+
+
+The following parameters are set for each WebXR joint.  The values are hard-coded within the `desktop-xr-hands` component.
+
+| parameter                  | description                                                  |      |
+| -------------------------- | ------------------------------------------------------------ | ---- |
+| position1                  | The index of a mediapipe landmark to use for the position of this joint |      |
+| position2                  | The index of a second mediapipe landmark to use for the position of this joint |      |
+| weight                     | The weight to attribute to position2 in positioning the joint.  0 => use position1, 1 => use position2, 0.5 => use the midpoint. |      |
+| orientation start          | The index of a mediapipe landmark to use for the joint orientation.  This indicates the start point of the direction vector.  For example, the wrist orientation is a vector from point 0 (start) to point 9 (end) |      |
+| orientation end            | The index of a mediapipe landmark to use for the joint orientation.  This indicates the end point of the direction vector. |      |
+| perpendicular vector start | (optional) The index of a mediapipe landmark to use for the joint orientation.     This indicates the start point for a vector that the joint should be perpendicular to.  Specifying a perpendicular vector is important to get the correct twist around the main orientation. |      |
+| perpendicular vector end   | (optional) The index of a mediapipe landmark to use for the joint orientation.     This indicates the end point for a vector that the joint should be perpendicular to. |      |

src/desktop-xr-hands.js

@@ -56,3 +56,3 @@ require('aframe-screen-display')
 
-// [position1, position2, weight, orientation start, orientation end]
+// [position1, position2, weight, orientation start, orientation end, perpendicular vector start, perpendicular vector end]
 // See Mediapipe landmarks here:
@@ -64,32 +64,32 @@ // https://developers.google.com/mediapipe/solutions/vision/hand_landmarker
 const xrJointMPMappings = [
-  [0, 0, 1, 0, 9], // "wrist"
+  [0, 0, 1, 0, 9, 17, 5], // "wrist"
 
-  [0, 1, 0.6, 0, 1], // "thumb-metacarpal"
-  [2, 2, 1, 1, 2], // "thumb-phalanx-proximal"
-  [3, 3, 1, 2, 3], // "thumb-phalanx-distal"
-  [4, 4, 1, 3, 4], // "thumb-tip"
+  [0, 1, 0.6, 0, 1, 0, 9], // "thumb-metacarpal"
+  [2, 2, 1, 1, 2, 1, 9], // "thumb-phalanx-proximal"
+  [3, 3, 1, 2, 3, 1, 9], // "thumb-phalanx-distal"
+  [4, 4, 1, 3, 4, 1, 9], // "thumb-tip"
 
-  [1, 5, 0.3, 1, 5], // "index-finger-metacarpal"
-  [5, 5, 1, 5, 6], // "index-finger-phalanx-proximal"
-  [6, 6, 1, 6, 7], // "index-finger-phalanx-intermediate"
-  [7, 7, 1, 6, 7], // "index-finger-phalanx-distal"
-  [8, 8, 1, 7, 8], // "index-finger-tip"
+  [1, 5, 0.3, 1, 5, 9, 5], // "index-finger-metacarpal"
+  [5, 5, 1, 5, 6, 9, 5], // "index-finger-phalanx-proximal"
+  [6, 6, 1, 6, 7, 9, 5], // "index-finger-phalanx-intermediate"
+  [7, 7, 1, 6, 7, 9, 5], // "index-finger-phalanx-distal"
+  [8, 8, 1, 7, 8, 9, 5], // "index-finger-tip"
 
-  [0, 9, 0.3, 0, 9], // "middle-finger-metacarpal"
-  [9, 9, 1, 9, 10], // "middle-finger-phalanx-proximal"
-  [10, 10, 1, 10, 11], // "middle-finger-phalanx-intermediate"
-  [11, 11, 1, 10, 11], // "middle-finger-phalanx-distal"
-  [12, 12, 1, 11, 12], // "middle-finger-tip"
+  [0, 9, 0.3, 0, 9, 13, 5], // "middle-finger-metacarpal"
+  [9, 9, 1, 9, 10, 13, 5], // "middle-finger-phalanx-proximal"
+  [10, 10, 1, 10, 11, 13, 5], // "middle-finger-phalanx-intermediate"
+  [11, 11, 1, 10, 11, 13, 5], // "middle-finger-phalanx-distal"
+  [12, 12, 1, 11, 12, 13, 5], // "middle-finger-tip"
 
-  [0, 13, 0.3, 0, 13], // "ring-finger-metacarpal"
-  [13, 13, 1, 13, 14], // "ring-finger-phalanx-proximal"
-  [14, 14, 1, 14, 15], // "ring-finger-phalanx-intermediate"
-  [15, 15, 1, 14, 15], // "ring-finger-phalanx-distal"
-  [16, 16, 1, 15, 16], // "ring-finger-tip"
+  [0, 13, 0.3, 0, 13, 17, 9], // "ring-finger-metacarpal"
+  [13, 13, 1, 13, 14, 17, 9], // "ring-finger-phalanx-proximal"
+  [14, 14, 1, 14, 15, 17, 9], // "ring-finger-phalanx-intermediate"
+  [15, 15, 1, 14, 15, 17, 9], // "ring-finger-phalanx-distal"
+  [16, 16, 1, 15, 16, 17, 9], // "ring-finger-tip"
 
-  [0, 17, 0.3, 0, 17], // "pinky-finger-metacarpal"
-  [17, 17, 1, 17, 18], // "pinky-finger-phalanx-proximal"
-  [18, 18, 1, 18, 19], // "pinky-finger-phalanx-intermediate"
-  [19, 19, 1, 18, 19], // "pinky-finger-phalanx-distal"
-  [20, 20, 1, 19, 20], // "pinky-finger-tip"
+  [0, 17, 0.3, 0, 17, 17, 13], // "pinky-finger-metacarpal"
+  [17, 17, 1, 17, 18, 17, 13], // "pinky-finger-phalanx-proximal"
+  [18, 18, 1, 18, 19, 17, 13], // "pinky-finger-phalanx-intermediate"
+  [19, 19, 1, 18, 19, 17, 13], // "pinky-finger-phalanx-distal"
+  [20, 20, 1, 19, 20, 17, 13], // "pinky-finger-tip"
 ]
@@ -101,3 +101,6 @@
 const _zAxis = new THREE.Vector3(0, 0, -1)
+const _xAxis = new THREE.Vector3()
+const _perpDirection = new THREE.Vector3()
 const _position = new THREE.Vector3()
+const _twist = new THREE.Quaternion()
 const _orientation = new THREE.Quaternion()
@@ -371,3 +374,9 @@ const _scale = new THREE.Vector3(1, 1, 1)
 
-    const [posIndex, pos2Index, weight, startIndex, endIndex] = xrJointMPMappings[jointIndex]
+    const [posIndex,
+           pos2Index,
+           weight,
+           startIndex,
+           endIndex,
+           perpStartIndex,
+           perpEndIndex] = xrJointMPMappings[jointIndex]
 
@@ -381,2 +390,20 @@ // determine orientation
 
+    // add twist, if perpendicular vector specified.
+    if (perpStartIndex !== undefined &&
+        perpEndIndex  !== undefined ) {
+      const xDirection = (hand === this.leftHand) ? 1 : -1
+      _xAxis.set(xDirection, 0, 0)
+      _xAxis.applyQuaternion(_orientation)
+      
+      const perpStart = worldLandmarks[perpStartIndex]
+      const perpEnd = worldLandmarks[perpEndIndex]
+      _perpDirection.subVectors(perpStart, perpEnd)
+      // perpendicular direction must be orthogonal to _direction, so that we only twist around the transformed z axis.
+      _perpDirection.projectOnPlane(_direction) 
+      _perpDirection.normalize()
+
+      _twist.setFromUnitVectors(_perpDirection, _xAxis)
+      _orientation.premultiply(_twist)
+    }
+
     // determine position
@@ -383,0 +410,0 @@ _position.copy(worldLandmarks[posIndex])

src/hand-landmarker.js

@@ -148,5 +148,8 @@ const {FilesetResolver, HandLandmarker} = require('@mediapipe/tasks-vision')
   
-    // Call this function again to keep predicting when the browser is ready
+    // Call this function again to keep predicting when the browser is ready.
+    // include a brief pause to ensure CPU is not completely swamped by landmarking processing.
     if (this.handLandmarker) {
-      window.requestAnimationFrame(this.predictWebcam);
+      setTimeout(() => {
+        window.requestAnimationFrame(this.predictWebcam);
+      }, 10)
     }
@@ -153,0 +156,0 @@ },

No alert changes

Improved metrics

Total package byte prevSize

5073628

increased by0.23%

Number of package files

16

increased by6.67%

Lines of code

8179

increased by0.68%

Number of lines in readme file

134

increased by11.67%

No dependency changes