Enable head tracking for AR avatar example (#130)

* Enable head tracking for AR avatar example * Tweaks
playcanvas · Feb 10, 2025 · 5d9e81b · 5d9e81b
1 parent bf15d11
commit 5d9e81b
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 4 deletions.
diff --git a/examples/ar-avatar.html b/examples/ar-avatar.html
@@ -25,7 +25,7 @@
             <!-- Scene -->
             <pc-scene>
                 <!-- Camera -->
-                <pc-entity name="camera" position="0 0 1">
+                <pc-entity name="camera">
                     <pc-camera clear-color="0 0 0 0" fov="80"></pc-camera>
                     <pc-scripts>
                         <pc-script name="cameraFeed"></pc-script>
@@ -37,7 +37,7 @@
                     <pc-light></pc-light>
                 </pc-entity>
                 <!-- Raccoon Avatar -->
-                <pc-entity name="raccoon" scale="-1 1 1">
+                <pc-entity name="raccoon" scale="100 100 100">
                     <pc-model asset="raccoon"></pc-model>
                     <pc-scripts>
                         <pc-script name="morphUpdate"></pc-script>

diff --git a/examples/assets/scripts/camera-feed.mjs b/examples/assets/scripts/camera-feed.mjs
@@ -13,7 +13,7 @@ export class CameraFeed extends Script {
      * @type {boolean}
      * @attribute
      */
-    mirror = true;
+    mirror = false;
 
     /**
      * @type {HTMLVideoElement|null}

diff --git a/examples/assets/scripts/face-landmarks.mjs b/examples/assets/scripts/face-landmarks.mjs
@@ -1,5 +1,5 @@
 import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
-import { Script } from 'playcanvas';
+import { Mat4, Script } from 'playcanvas';
 
 export class FaceLandmarks extends Script {
     /** @type {FaceLandmarker} */
@@ -27,6 +27,16 @@ export class FaceLandmarks extends Script {
             if (video && video.readyState >= HTMLMediaElement.HAVE_ENOUGH_DATA) {
                 const detections = this.faceLandmarker.detectForVideo(video, Date.now());
                 if (detections && detections.faceBlendshapes) {
+                    if (detections.facialTransformationMatrixes.length > 0) {
+                        const { data } = detections.facialTransformationMatrixes[0];
+                        const matrix = new Mat4();
+                        matrix.set(data).invert();
+                        const position = matrix.getTranslation();
+                        const rotation = matrix.getEulerAngles();
+                        this.entity.setPosition(position);
+                        this.entity.setEulerAngles(rotation);
+                    }
+
                     if (detections.faceBlendshapes.length > 0) {
                         const { categories } = detections.faceBlendshapes[0];
                         this.app.fire('face:blendshapes', categories);