Add HDR support (#2030)

alvr-org · Mar 22, 2024 · 68ea265 · 68ea265
1 parent 005c4c7
commit 68ea265
Show file tree

Hide file tree

Showing 31 changed files with 659 additions and 56 deletions.
diff --git a/alvr/client_core/cpp/bindings.h b/alvr/client_core/cpp/bindings.h
@@ -23,6 +23,8 @@ struct FfiStreamConfig {
     float foveationEdgeRatioX;
     float foveationEdgeRatioY;
     unsigned int enableSrgbCorrection;
+    unsigned int fixLimitedRange;
+    float encodingGamma;
 };
 
 // gltf_model.h
@@ -37,7 +39,8 @@ extern "C" void destroyGraphicsNative();
 extern "C" void prepareLobbyRoom(int viewWidth,
                                  int viewHeight,
                                  const unsigned int *swapchainTextures[2],
-                                 int swapchainLength);
+                                 int swapchainLength,
+                                 bool enable_srgb_correction);
 extern "C" void destroyRenderers();
 extern "C" void streamStartNative(FfiStreamConfig config);
 extern "C" void updateLobbyHudTexture(const unsigned char *data);

diff --git a/alvr/client_core/cpp/graphics.cpp b/alvr/client_core/cpp/graphics.cpp
@@ -322,7 +322,7 @@ void ovrFramebuffer_Create(ovrFramebuffer *frameBuffer,
     for (int i = 0; i < textures.size(); i++) {
         auto glRenderTarget = textures[i];
         frameBuffer->renderTargets.push_back(std::make_unique<gl_render_utils::Texture>(
-            true, glRenderTarget, false, width, height, GL_SRGB8_ALPHA8, GL_RGBA));
+            true, glRenderTarget, false, width, height, GL_RGBA16F, GL_RGBA));
         frameBuffer->renderStates.push_back(
             std::make_unique<gl_render_utils::RenderState>(frameBuffer->renderTargets[i].get()));
     }
@@ -580,19 +580,21 @@ void ovrRenderer_Create(ovrRenderer *renderer,
                         std::vector<GLuint> textures[2],
                         FFRData ffrData,
                         bool isLobby,
-                        bool enableSrgbCorrection) {
+                        bool enableSrgbCorrection,
+                        bool fixLimitedRange,
+                        float encodingGamma) {
     if (!isLobby) {
         renderer->srgbCorrectionPass = std::make_unique<SrgbCorrectionPass>(streamTexture);
         renderer->enableFFE = ffrData.enabled;
         if (renderer->enableFFE) {
             FoveationVars fv = CalculateFoveationVars(ffrData);
             renderer->srgbCorrectionPass->Initialize(
-                fv.optimizedEyeWidth, fv.optimizedEyeHeight, !enableSrgbCorrection);
+                fv.optimizedEyeWidth, fv.optimizedEyeHeight, !enableSrgbCorrection, fixLimitedRange, encodingGamma);
             renderer->ffr = std::make_unique<FFR>(renderer->srgbCorrectionPass->GetOutputTexture());
             renderer->ffr->Initialize(fv);
             renderer->streamRenderTexture = renderer->ffr->GetOutputTexture()->GetGLTexture();
         } else {
-            renderer->srgbCorrectionPass->Initialize(width, height, !enableSrgbCorrection);
+            renderer->srgbCorrectionPass->Initialize(width, height, !enableSrgbCorrection, fixLimitedRange, encodingGamma);
             renderer->streamRenderTexture =
                 renderer->srgbCorrectionPass->GetOutputTexture()->GetGLTexture();
         }
@@ -793,7 +795,8 @@ void destroyGraphicsNative() {
 void prepareLobbyRoom(int viewWidth,
                       int viewHeight,
                       const unsigned int *swapchainTextures[2],
-                      int swapchainLength) {
+                      int swapchainLength,
+                      bool enable_srgb_correction) {
     for (int eye = 0; eye < 2; eye++) {
         g_ctx.lobbySwapchainTextures[eye].clear();
 
@@ -811,7 +814,9 @@ void prepareLobbyRoom(int viewWidth,
                        g_ctx.lobbySwapchainTextures,
                        {false},
                        true,
-                       false);
+                       enable_srgb_correction,
+                       false,
+                       1.0);
 }
 
 // on pause
@@ -857,7 +862,9 @@ void streamStartNative(FfiStreamConfig config) {
                         config.foveationEdgeRatioX,
                         config.foveationEdgeRatioY},
                        false,
-                       config.enableSrgbCorrection);
+                       config.enableSrgbCorrection,
+                       config.fixLimitedRange,
+                       config.encodingGamma);
 }
 
 void updateLobbyHudTexture(const unsigned char *data) {

diff --git a/alvr/client_core/cpp/srgb_correction_pass.cpp b/alvr/client_core/cpp/srgb_correction_pass.cpp
@@ -31,25 +31,39 @@ const string CORRECTION_FRAGMENT_SHADER = R"glsl(
         {
             color = texture(tex0, uv);
 
+#ifdef LIMITED_RANGE_BUG
             // For some reason, the encoder shifts full-range color into the negatives and over one.
             color.rgb = LIMITED_MIN + ((LIMITED_MAX - LIMITED_MIN) * color.rgb);
+#endif
 #ifdef SRGB_CORRECTION
             vec3 condition = vec3(color.r < THRESHOLD, color.g < THRESHOLD, color.b < THRESHOLD);
             vec3 lowValues = color.rgb * DIV12;
             vec3 highValues = pow((color.rgb + 0.055) * DIV1, GAMMA);
             color.rgb = condition * lowValues + (1.0 - condition) * highValues;
+#endif
+#ifdef ENCODING_GAMMA
+            vec3 enc_condition = vec3(color.r < 0.0, color.g < 0.0, color.b < 0.0);
+            vec3 enc_lowValues = color.rgb;
+            vec3 enc_highValues = pow(color.rgb, vec3(ENCODING_GAMMA));
+            color.rgb = enc_condition * enc_lowValues + (1.0 - enc_condition) * enc_highValues;
 #endif
         }
     )glsl";
 } // namespace
 
 SrgbCorrectionPass::SrgbCorrectionPass(Texture *inputSurface) : mInputSurface(inputSurface) {}
 
-void SrgbCorrectionPass::Initialize(uint32_t width, uint32_t height, bool passthrough) {
+void SrgbCorrectionPass::Initialize(uint32_t width, uint32_t height, bool passthrough, bool fixLimitedRange, float encodingGamma) {
     mOutputTexture.reset(new Texture(false, 0, false, width * 2, height));
     mOutputTextureState = make_unique<RenderState>(mOutputTexture.get());
 
     string defines = passthrough ? "" : "#define SRGB_CORRECTION";
+    if (fixLimitedRange) {
+        defines += "\n#define LIMITED_RANGE_BUG";
+    }
+    if (encodingGamma != 1.0) {
+        defines += "\n#define ENCODING_GAMMA (" + std::to_string(encodingGamma) + ")";
+    }
 
     auto fragmentShader = CORRECTION_FRAGMENT_SHADER_HEADER + "\n" + defines + "\n" + CORRECTION_FRAGMENT_SHADER;
     mStagingPipeline = unique_ptr<RenderPipeline>(

diff --git a/alvr/client_core/cpp/srgb_correction_pass.h b/alvr/client_core/cpp/srgb_correction_pass.h
@@ -9,7 +9,7 @@ class SrgbCorrectionPass {
   public:
     SrgbCorrectionPass(gl_render_utils::Texture *inputSurface);
 
-    void Initialize(uint32_t width, uint32_t height, bool passthrough);
+    void Initialize(uint32_t width, uint32_t height, bool passthrough, bool fixLimitedRange, float encodingGamma);
 
     void Render() const;
 

diff --git a/alvr/client_core/src/c_api.rs b/alvr/client_core/src/c_api.rs
@@ -658,10 +658,12 @@ pub unsafe extern "C" fn alvr_resume_opengl(
     preferred_view_height: u32,
     swapchain_textures: *mut *const u32,
     swapchain_length: u32,
+    enable_srgb_correction: bool,
 ) {
     opengl::initialize_lobby(
         UVec2::new(preferred_view_width, preferred_view_height),
         convert_swapchain_array(swapchain_textures, swapchain_length),
+        enable_srgb_correction,
     );
 }
 
@@ -690,7 +692,14 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) {
         edge_ratio_y: config.foveation_edge_ratio_y,
     });
 
-    opengl::start_stream(view_resolution, swapchain_textures, foveated_encoding, true);
+    opengl::start_stream(
+        view_resolution,
+        swapchain_textures,
+        foveated_encoding,
+        true,
+        false, // TODO: limited range fix config
+        1.0,   // TODO: encoding gamma config
+    );
 }
 
 #[no_mangle]

diff --git a/alvr/client_core/src/opengl.rs b/alvr/client_core/src/opengl.rs
@@ -42,7 +42,11 @@ pub fn destroy() {
     }
 }
 
-pub fn initialize_lobby(preferred_view_resolution: UVec2, swapchain_textures: [Vec<u32>; 2]) {
+pub fn initialize_lobby(
+    preferred_view_resolution: UVec2,
+    swapchain_textures: [Vec<u32>; 2],
+    enable_srgb_correction: bool,
+) {
     #[cfg(target_os = "android")]
     unsafe {
         let swapchain_length = swapchain_textures[0].len();
@@ -56,6 +60,7 @@ pub fn initialize_lobby(preferred_view_resolution: UVec2, swapchain_textures: [V
             preferred_view_resolution.y as _,
             swapchain_textures.as_mut_ptr(),
             swapchain_length as _,
+            enable_srgb_correction,
         );
     }
 }
@@ -72,6 +77,8 @@ pub fn start_stream(
     swapchain_textures: [Vec<u32>; 2],
     foveated_encoding: Option<FoveatedEncodingConfig>,
     enable_srgb_correction: bool,
+    fix_limited_range: bool,
+    encoding_gamma: f32,
 ) {
     #[cfg(target_os = "android")]
     unsafe {
@@ -109,6 +116,8 @@ pub fn start_stream(
                 .map(|f| f.edge_ratio_y)
                 .unwrap_or_default(),
             enableSrgbCorrection: enable_srgb_correction as u32,
+            fixLimitedRange: fix_limited_range as u32,
+            encodingGamma: encoding_gamma,
         };
 
         streamStartNative(config);

diff --git a/alvr/client_openxr/src/graphics.rs b/alvr/client_openxr/src/graphics.rs
@@ -101,7 +101,7 @@ pub fn create_swapchain(
     let swapchain_info = xr::SwapchainCreateInfo {
         create_flags: xr::SwapchainCreateFlags::EMPTY,
         usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT | xr::SwapchainUsageFlags::SAMPLED,
-        format: glow::SRGB8_ALPHA8,
+        format: glow::RGBA16F,
         sample_count: 1,
         width: resolution.x,
         height: resolution.y,

diff --git a/alvr/client_openxr/src/lib.rs b/alvr/client_openxr/src/lib.rs
@@ -20,6 +20,7 @@ use std::{
     time::{Duration, Instant},
 };
 use stream::StreamContext;
+use xr::ColorSpaceFB;
 
 const DECODER_MAX_TIMEOUT_MULTIPLIER: f32 = 0.8;
 
@@ -188,6 +189,10 @@ pub fn entry_point() {
             vec![90.0]
         };
 
+        if exts.fb_color_space {
+            xr_session.set_color_space(ColorSpaceFB::P3).unwrap();
+        }
+
         let capabilities = ClientCapabilities {
             default_view_resolution,
             external_decoder: false,

diff --git a/alvr/client_openxr/src/lobby.rs b/alvr/client_openxr/src/lobby.rs
@@ -39,6 +39,7 @@ impl Lobby {
                     .map(|i| *i as _)
                     .collect(),
             ],
+            false, // TODO: correct lobby sRGB for some headsets
         );
 
         Self {

diff --git a/alvr/client_openxr/src/stream.rs b/alvr/client_openxr/src/stream.rs
@@ -11,7 +11,7 @@ use alvr_common::{
 };
 use alvr_packets::{FaceData, NegotiatedStreamingConfig, Tracking};
 use alvr_session::{
-    BodyTrackingSourcesConfig, ClientsideFoveationConfig, ClientsideFoveationMode,
+    BodyTrackingSourcesConfig, ClientsideFoveationConfig, ClientsideFoveationMode, EncoderConfig,
     FaceTrackingSourcesConfig, FoveatedEncodingConfig, Settings,
 };
 use openxr as xr;
@@ -33,6 +33,7 @@ pub struct StreamConfig {
     pub refresh_rate_hint: f32,
     pub foveated_encoding_config: Option<FoveatedEncodingConfig>,
     pub clientside_foveation_config: Option<ClientsideFoveationConfig>,
+    pub encoder_config: EncoderConfig,
     pub face_sources_config: Option<FaceTrackingSourcesConfig>,
     pub body_sources_config: Option<BodyTrackingSourcesConfig>,
 }
@@ -47,6 +48,7 @@ impl StreamConfig {
                 .then(|| settings.video.foveated_encoding.as_option().cloned())
                 .flatten(),
             clientside_foveation_config: settings.video.clientside_foveation.as_option().cloned(),
+            encoder_config: settings.video.encoder_config.clone(),
             face_sources_config: settings
                 .headset
                 .face_tracking
@@ -187,7 +189,11 @@ impl StreamContext {
                     .collect(),
             ],
             config.foveated_encoding_config.clone(),
-            platform != Platform::Lynx,
+            platform != Platform::Lynx
+                && platform != Platform::Pico4
+                && platform != Platform::PicoNeo3,
+            config.encoder_config.enable_hdr != true,
+            config.encoder_config.encoding_gamma,
         );
 
         core_ctx.send_playspace(

diff --git a/alvr/server/cpp/alvr_server/Settings.cpp b/alvr/server/cpp/alvr_server/Settings.cpp
@@ -66,6 +66,10 @@ void Settings::Load() {
         m_entropyCoding = (uint32_t)config.get("entropy_coding").get<int64_t>();
         m_use10bitEncoder = config.get("use_10bit_encoder").get<bool>();
         m_useFullRangeEncoding = config.get("use_full_range_encoding").get<bool>();
+        m_encodingGamma = config.get("encoding_gamma").get<double>();
+        m_enableHdr = config.get("enable_hdr").get<bool>();
+        m_forceHdrSrgbCorrection = config.get("force_hdr_srgb_correction").get<bool>();
+        m_clampHdrExtendedRange = config.get("clamp_hdr_extended_range").get<bool>();
         m_enablePreAnalysis = config.get("enable_pre_analysis").get<bool>();
         m_enableVbaq = config.get("enable_vbaq").get<bool>();
         m_enableHmqb = config.get("enable_hmqb").get<bool>();

diff --git a/alvr/server/cpp/alvr_server/Settings.h b/alvr/server/cpp/alvr_server/Settings.h
@@ -43,6 +43,10 @@ class Settings {
     int m_h264Profile;
     bool m_use10bitEncoder;
     bool m_useFullRangeEncoding;
+    double m_encodingGamma;
+    bool m_enableHdr;
+    bool m_forceHdrSrgbCorrection;
+    bool m_clampHdrExtendedRange;
     bool m_enablePreAnalysis;
     bool m_enableVbaq;
     bool m_enableHmqb;

diff --git a/alvr/server/cpp/alvr_server/alvr_server.cpp b/alvr/server/cpp/alvr_server/alvr_server.cpp
@@ -255,6 +255,8 @@ const unsigned char *COMPRESS_AXIS_ALIGNED_CSO_PTR;
 unsigned int COMPRESS_AXIS_ALIGNED_CSO_LEN;
 const unsigned char *COLOR_CORRECTION_CSO_PTR;
 unsigned int COLOR_CORRECTION_CSO_LEN;
+const unsigned char *RGBTOYUV420_CSO_PTR;
+unsigned int RGBTOYUV420_CSO_LEN;
 
 const unsigned char *QUAD_SHADER_COMP_SPV_PTR;
 unsigned int QUAD_SHADER_COMP_SPV_LEN;

diff --git a/alvr/server/cpp/alvr_server/bindings.h b/alvr/server/cpp/alvr_server/bindings.h
@@ -94,6 +94,9 @@ extern "C" const unsigned char *COMPRESS_AXIS_ALIGNED_CSO_PTR;
 extern "C" unsigned int COMPRESS_AXIS_ALIGNED_CSO_LEN;
 extern "C" const unsigned char *COLOR_CORRECTION_CSO_PTR;
 extern "C" unsigned int COLOR_CORRECTION_CSO_LEN;
+extern "C" const unsigned char *RGBTOYUV420_CSO_PTR;
+extern "C" unsigned int RGBTOYUV420_CSO_LEN;
+
 
 extern "C" const unsigned char *QUAD_SHADER_COMP_SPV_PTR;
 extern "C" unsigned int QUAD_SHADER_COMP_SPV_LEN;
-Original file line number
+Diff line change
@@ Expand Up / @@ -39,6 +39,7 @@ impl Lobby { @@
                         .map(|i| *i as _)
                         .collect(),
                 ],
+                false, // TODO: correct lobby sRGB for some headsets
             );
             Self {
@@ Expand Down @@