fix(client): 🐛 Fix controller angular velocity for some devices; show…

… velocities in lobby (#2608)

* fix(client): 🐛 Fix controller angular velocity for some devices; show velocities in lobby

* Draw velocities only in debug builds

* Add push constants consts and bound checks

alvr/client_core/resources/lobby_line.wgsl

@@ -1,11 +1,15 @@
-var<push_constant> transform: mat4x4f;
+struct PushConstant {
+    transform: mat4x4f,
+    color: u32,
+}
+var<push_constant> pc: PushConstant;
 
 @vertex
 fn vertex_main(@builtin(vertex_index) vertex_index: u32) -> @builtin(position) vec4f {
-    return transform * vec4f(0.0, 0.0, -f32(vertex_index), 1.0);
+    return pc.transform * vec4f(0.0, 0.0, -f32(vertex_index), 1.0);
 }
 
 @fragment
 fn fragment_main() -> @location(0) vec4f {
-    return vec4f(1.0);
+    return unpack4x8unorm(pc.color);
 }

alvr/client_core/src/c_api.rs

@@ -820,6 +820,7 @@ pub unsafe extern "C" fn alvr_render_lobby_opengl(
                 [(None, None), (None, None)],
                 None,
                 render_background,
+                false,
             );
         }
     });

alvr/client_core/src/graphics/lobby.rs

@@ -1,13 +1,13 @@
-use super::{GraphicsContext, SDR_FORMAT};
+use super::{GraphicsContext, MAX_PUSH_CONSTANTS_SIZE, SDR_FORMAT};
 use alvr_common::{
     glam::{IVec2, Mat4, Quat, UVec2, Vec3},
-    Fov, Pose,
+    DeviceMotion, Fov, Pose,
 };
 use glyph_brush_layout::{
     ab_glyph::{Font, FontRef, ScaleFont},
     FontId, GlyphPositioner, HorizontalAlign, Layout, SectionGeometry, SectionText, VerticalAlign,
 };
-use std::{f32::consts::FRAC_PI_2, rc::Rc};
+use std::{f32::consts::FRAC_PI_2, mem, rc::Rc};
 use wgpu::{
     include_wgsl, BindGroup, BindGroupDescriptor, BindGroupEntry, BindGroupLayout,
     BindGroupLayoutDescriptor, BindGroupLayoutEntry, BindingResource, BindingType, BlendComponent,
@@ -20,6 +20,25 @@ use wgpu::{
     TextureSampleType, TextureView, TextureViewDimension, VertexState,
 };
 
+const TRANSFORM_CONST_SIZE: u32 = mem::size_of::<Mat4>() as u32;
+const OBJECT_TYPE_CONST_SIZE: u32 = mem::size_of::<u32>() as u32;
+const FLOOR_SIDE_CONST_SIZE: u32 = mem::size_of::<f32>() as u32;
+const COLOR_CONST_SIZE: u32 = mem::size_of::<u32>() as u32;
+
+const QUAD_PUSH_CONTANTS_SIZE: u32 =
+    TRANSFORM_CONST_SIZE + OBJECT_TYPE_CONST_SIZE + FLOOR_SIDE_CONST_SIZE;
+const LINE_PUSH_CONTANTS_SIZE: u32 = TRANSFORM_CONST_SIZE + COLOR_CONST_SIZE;
+const _: () = assert!(
+    QUAD_PUSH_CONTANTS_SIZE <= MAX_PUSH_CONSTANTS_SIZE
+        && LINE_PUSH_CONTANTS_SIZE <= MAX_PUSH_CONSTANTS_SIZE,
+    "Push constants size exceeds the maximum size"
+);
+
+const TRANSFORM_CONST_OFFSET: u32 = 0;
+const OBJECT_TYPE_CONST_OFFSET: u32 = TRANSFORM_CONST_SIZE;
+const FLOOR_SIDE_CONST_OFFSET: u32 = OBJECT_TYPE_CONST_OFFSET + OBJECT_TYPE_CONST_SIZE;
+const COLOR_CONST_OFFSET: u32 = TRANSFORM_CONST_SIZE;
+
 const FLOOR_SIDE: f32 = 300.0;
 const HUD_DIST: f32 = 5.0;
 const HUD_SIDE: f32 = 3.5;
@@ -213,7 +232,7 @@ impl LobbyRenderer {
             device,
             "lobby_quad",
             &[&bind_group_layout],
-            72,
+            QUAD_PUSH_CONTANTS_SIZE,
             include_wgsl!("../../resources/lobby_quad.wgsl"),
             PrimitiveTopology::TriangleStrip,
         );
@@ -222,7 +241,7 @@ impl LobbyRenderer {
             device,
             "lobby_line",
             &[],
-            64,
+            LINE_PUSH_CONTANTS_SIZE,
             include_wgsl!("../../resources/lobby_line.wgsl"),
             PrimitiveTopology::LineList,
         );
@@ -352,9 +371,10 @@ impl LobbyRenderer {
     pub fn render(
         &self,
         view_params: [LobbyViewParams; 2],
-        hand_data: [(Option<Pose>, Option<[Pose; 26]>); 2],
+        hand_data: [(Option<DeviceMotion>, Option<[Pose; 26]>); 2],
         body_skeleton_fb: Option<Vec<Option<Pose>>>,
         render_background: bool,
+        show_velocities: bool,
     ) {
         let mut encoder = self
             .context
@@ -401,7 +421,11 @@ impl LobbyRenderer {
                     .iter()
                     .flat_map(|v| v.to_le_bytes())
                     .collect::<Vec<u8>>();
-                pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 0, &data);
+                pass.set_push_constants(
+                    ShaderStages::VERTEX_FRAGMENT,
+                    TRANSFORM_CONST_OFFSET,
+                    &data,
+                );
                 pass.draw(0..vertices_count, 0..1);
             }
 
@@ -413,10 +437,14 @@ impl LobbyRenderer {
 
             if render_background {
                 // Render ground
-                pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 64, &0_u32.to_le_bytes());
                 pass.set_push_constants(
                     ShaderStages::VERTEX_FRAGMENT,
-                    68,
+                    OBJECT_TYPE_CONST_OFFSET,
+                    &0_u32.to_le_bytes(),
+                );
+                pass.set_push_constants(
+                    ShaderStages::VERTEX_FRAGMENT,
+                    FLOOR_SIDE_CONST_OFFSET,
                     &FLOOR_SIDE.to_le_bytes(),
                 );
                 let transform = view_proj
@@ -426,7 +454,11 @@ impl LobbyRenderer {
             }
 
             // Render HUD
-            pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 64, &1_u32.to_le_bytes());
+            pass.set_push_constants(
+                ShaderStages::VERTEX_FRAGMENT,
+                OBJECT_TYPE_CONST_OFFSET,
+                &1_u32.to_le_bytes(),
+            );
             for i in 0..4 {
                 let transform = Mat4::from_rotation_y(FRAC_PI_2 * i as f32)
                     * Mat4::from_translation(Vec3::new(0.0, HUD_SIDE / 2.0, -HUD_DIST))
@@ -436,8 +468,14 @@ impl LobbyRenderer {
 
             // Render hands and body skeleton
             pass.set_pipeline(&self.line_pipeline);
-            for (maybe_pose, maybe_skeleton) in &hand_data {
+            for (maybe_motion, maybe_skeleton) in &hand_data {
                 if let Some(skeleton) = maybe_skeleton {
+                    pass.set_push_constants(
+                        ShaderStages::VERTEX_FRAGMENT,
+                        COLOR_CONST_OFFSET,
+                        &[255, 255, 255, 255],
+                    );
+
                     for (joint1_idx, joint2_idx) in HAND_SKELETON_BONES {
                         let j1_pose = skeleton[joint1_idx];
                         let j2_pose = skeleton[joint2_idx];
@@ -451,25 +489,59 @@ impl LobbyRenderer {
                     }
                 }
 
-                if let Some(pose) = maybe_pose {
+                if let Some(motion) = maybe_motion {
                     let hand_transform = Mat4::from_scale_rotation_translation(
                         Vec3::ONE * 0.2,
-                        pose.orientation,
-                        pose.position,
+                        motion.pose.orientation,
+                        motion.pose.position,
                     );
 
+                    // Draw crossair
                     let segment_rotations = [
                         Mat4::IDENTITY,
                         Mat4::from_rotation_y(FRAC_PI_2),
                         Mat4::from_rotation_x(FRAC_PI_2),
                     ];
+                    pass.set_push_constants(
+                        ShaderStages::VERTEX_FRAGMENT,
+                        COLOR_CONST_OFFSET,
+                        &[255, 255, 255, 255],
+                    );
                     for rot in &segment_rotations {
                         let transform = hand_transform
                             * *rot
                             * Mat4::from_scale(Vec3::ONE * 0.5)
                             * Mat4::from_translation(Vec3::Z * 0.5);
                         transform_draw(&mut pass, view_proj * transform, 2);
                     }
+
+                    if show_velocities {
+                        // Draw linear velocity
+                        let transform = Mat4::from_scale_rotation_translation(
+                            Vec3::ONE * motion.linear_velocity.length() * 0.2,
+                            Quat::from_rotation_arc(-Vec3::Z, motion.linear_velocity.normalize()),
+                            motion.pose.position,
+                        );
+                        pass.set_push_constants(
+                            ShaderStages::VERTEX_FRAGMENT,
+                            COLOR_CONST_OFFSET,
+                            &[255, 0, 0, 255],
+                        );
+                        transform_draw(&mut pass, view_proj * transform, 2);
+
+                        // Draw angular velocity
+                        let transform = Mat4::from_scale_rotation_translation(
+                            Vec3::ONE * motion.angular_velocity.length() * 0.01,
+                            Quat::from_rotation_arc(-Vec3::Z, motion.angular_velocity.normalize()),
+                            motion.pose.position,
+                        );
+                        pass.set_push_constants(
+                            ShaderStages::VERTEX_FRAGMENT,
+                            COLOR_CONST_OFFSET,
+                            &[0, 255, 0, 255],
+                        );
+                        transform_draw(&mut pass, view_proj * transform, 2);
+                    }
                 }
             }
             if let Some(skeleton) = &body_skeleton_fb {

alvr/client_core/src/graphics/mod.rs

@@ -21,6 +21,7 @@ use wgpu::{
 pub const SDR_FORMAT: TextureFormat = TextureFormat::Rgba8Unorm;
 pub const SDR_FORMAT_GL: u32 = gl::RGBA8;
 pub const GL_TEXTURE_EXTERNAL_OES: u32 = 0x8D65;
+pub const MAX_PUSH_CONSTANTS_SIZE: u32 = 72;
 
 type CreateImageFn = unsafe extern "C" fn(
     egl::EGLDisplay,
@@ -241,7 +242,7 @@ impl GraphicsContext {
                 label: None,
                 required_features: Features::PUSH_CONSTANTS,
                 required_limits: Limits {
-                    max_push_constant_size: 72,
+                    max_push_constant_size: MAX_PUSH_CONSTANTS_SIZE,
                     ..adapter.limits()
                 },
             },

alvr/client_core/src/graphics/stream.rs

@@ -1,10 +1,10 @@
-use super::{staging::StagingRenderer, GraphicsContext};
+use super::{staging::StagingRenderer, GraphicsContext, MAX_PUSH_CONSTANTS_SIZE};
 use alvr_common::{
     glam::{self, Mat4, Quat, UVec2, Vec3},
     Fov,
 };
 use alvr_session::{FoveatedEncodingConfig, PassthroughMode};
-use std::{collections::HashMap, ffi::c_void, iter, rc::Rc};
+use std::{collections::HashMap, ffi::c_void, iter, mem, rc::Rc};
 use wgpu::{
     hal::{api, gles},
     include_wgsl, BindGroup, BindGroupDescriptor, BindGroupEntry, BindGroupLayoutDescriptor,
@@ -16,6 +16,18 @@ use wgpu::{
     VertexState,
 };
 
+const TRANSFORM_CONST_SIZE: u32 = mem::size_of::<Mat4>() as u32;
+const VIEW_INDEX_CONST_SIZE: u32 = mem::size_of::<u32>() as u32;
+
+const PUSH_CONSTANTS_SIZE: u32 = TRANSFORM_CONST_SIZE + VIEW_INDEX_CONST_SIZE;
+const _: () = assert!(
+    PUSH_CONSTANTS_SIZE <= MAX_PUSH_CONSTANTS_SIZE,
+    "Push constants size exceeds the maximum size"
+);
+
+const TRANSFORM_CONST_OFFSET: u32 = 0;
+const VIEW_INDEX_CONST_OFFSET: u32 = TRANSFORM_CONST_SIZE;
+
 pub struct StreamViewParams {
     pub swapchain_index: u32,
     pub reprojection_rotation: Quat,
@@ -113,7 +125,7 @@ impl StreamRenderer {
                 bind_group_layouts: &[&bind_group_layout],
                 push_constant_ranges: &[PushConstantRange {
                     stages: ShaderStages::VERTEX_FRAGMENT,
-                    range: 0..68,
+                    range: 0..PUSH_CONSTANTS_SIZE,
                 }],
             })),
             vertex: VertexState {
@@ -265,10 +277,14 @@ impl StreamRenderer {
                 .collect::<Vec<u8>>();
 
             render_pass.set_pipeline(&self.pipeline);
-            render_pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 0, &transform_bytes);
             render_pass.set_push_constants(
                 ShaderStages::VERTEX_FRAGMENT,
-                64,
+                TRANSFORM_CONST_OFFSET,
+                &transform_bytes,
+            );
+            render_pass.set_push_constants(
+                ShaderStages::VERTEX_FRAGMENT,
+                VIEW_INDEX_CONST_OFFSET,
                 &(view_idx as u32).to_le_bytes(),
             );
             render_pass.set_bind_group(0, &self.views_objects[view_idx].bind_group, &[]);

alvr/client_openxr/src/interaction.rs

@@ -548,6 +548,7 @@ pub fn get_head_data(
 
 pub fn get_hand_data(
     xr_session: &xr::Session<xr::OpenGlEs>,
+    platform: Platform,
     reference_space: &xr::Space,
     time: xr::Time,
     hand_source: &HandInteraction,
@@ -574,10 +575,18 @@ pub fn get_hand_data(
                 last_controller_pose.position = crate::from_xr_vec3(location.pose.position);
             }
 
+            let linear_velocity = crate::from_xr_vec3(velocity.linear_velocity);
+            let mut angular_velocity = crate::from_xr_vec3(velocity.angular_velocity);
+
+            // Some headsets use wrong frame of reference
+            if matches!(platform, Platform::PicoNeo3 | Platform::Pico4) || platform.is_vive() {
+                angular_velocity = last_controller_pose.orientation * angular_velocity;
+            };
+
             Some(DeviceMotion {
                 pose: *last_controller_pose,
-                linear_velocity: crate::from_xr_vec3(velocity.linear_velocity),
-                angular_velocity: crate::from_xr_vec3(velocity.angular_velocity),
+                linear_velocity,
+                angular_velocity,
             })
         } else {
             None

alvr/client_openxr/src/lib.rs

@@ -272,6 +272,7 @@ pub fn entry_point() {
             xr_session.clone(),
             Rc::clone(&graphics_context),
             Arc::clone(&interaction_context),
+            platform,
             default_view_resolution,
             &last_lobby_message,
         );

alvr/client_openxr/src/lobby.rs

@@ -4,13 +4,15 @@ use crate::{
 };
 use alvr_client_core::graphics::{GraphicsContext, LobbyRenderer, LobbyViewParams, SDR_FORMAT_GL};
 use alvr_common::{glam::UVec2, parking_lot::RwLock, Pose};
+use alvr_system_info::Platform;
 use openxr as xr;
 use std::{rc::Rc, sync::Arc};
 
 // todo: add interaction?
 pub struct Lobby {
     xr_session: xr::Session<xr::OpenGlEs>,
     interaction_ctx: Arc<RwLock<InteractionContext>>,
+    platform: Platform,
     reference_space: xr::Space,
     swapchains: [xr::Swapchain<xr::OpenGlEs>; 2],
     view_resolution: UVec2,
@@ -23,6 +25,7 @@ impl Lobby {
         xr_session: xr::Session<xr::OpenGlEs>,
         gfx_ctx: Rc<GraphicsContext>,
         interaction_ctx: Arc<RwLock<InteractionContext>>,
+        platform: Platform,
         view_resolution: UVec2,
         initial_hud_message: &str,
     ) -> Self {
@@ -64,6 +67,7 @@ impl Lobby {
         Self {
             xr_session,
             interaction_ctx,
+            platform,
             reference_space,
             swapchains,
             view_resolution,
@@ -102,6 +106,7 @@ impl Lobby {
             .ok();
         let left_hand_data = interaction::get_hand_data(
             &self.xr_session,
+            self.platform,
             &self.reference_space,
             predicted_display_time,
             &self.interaction_ctx.read().hands_interaction[0],
@@ -110,6 +115,7 @@ impl Lobby {
         );
         let right_hand_data = interaction::get_hand_data(
             &self.xr_session,
+            self.platform,
             &self.reference_space,
             predicted_display_time,
             &self.interaction_ctx.read().hands_interaction[1],
@@ -155,12 +161,10 @@ impl Lobby {
                     swapchain_index: right_swapchain_idx,
                 },
             ],
-            [
-                (left_hand_data.0.map(|dm| dm.pose), left_hand_data.1),
-                (right_hand_data.0.map(|dm| dm.pose), right_hand_data.1),
-            ],
+            [left_hand_data, right_hand_data],
             body_skeleton_fb,
             false,
+            cfg!(debug_assertions),
         );
 
         self.swapchains[0].release_image().unwrap();