From 64e1d09fb4a1f2f7bb1e96552a68ce50f2d621ba Mon Sep 17 00:00:00 2001 From: Stephan Vedder Date: Fri, 3 Jan 2025 16:56:10 +0100 Subject: [PATCH] feat(rendered): Support rendered resources --- Cargo.lock | 1 + Cargo.toml | 7 +- src/api/wado/routes.rs | 65 ++++++++++-- src/api/wado/service.rs | 205 ++++++++++++++++++++++++++++++++++++-- src/backend/dimse/wado.rs | 102 ++++++++++++++++++- src/backend/s3/wado.rs | 9 +- 6 files changed, 371 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb5b918..9573c1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1106,6 +1106,7 @@ dependencies = [ "config", "dicom", "dicom-json", + "dicom-pixeldata", "futures", "mime", "multer", diff --git a/Cargo.toml b/Cargo.toml index 8fe12d9..130a44b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,11 @@ version = "0.3.0-beta.1" description = "A robust DICOMweb server with swappable backend" edition = "2021" rust-version = "1.74.0" -categories = ["multimedia", "network-programming", "web-programming::http-server"] +categories = [ + "multimedia", + "network-programming", + "web-programming::http-server", +] keywords = ["dicom", "dicomweb", "healthcare", "medical"] repository = "https://github.com/UMEssen/DICOM-RST" license = "MIT" @@ -20,6 +24,7 @@ s3 = [] # DICOM processing dicom = "0.8.0" dicom-json = "0.8.0" +dicom-pixeldata = { version = "0.8.0", features = ["image"] } sentry = { version = "0.35.0", features = ["tracing"] } # Serialization diff --git a/src/api/wado/routes.rs b/src/api/wado/routes.rs index 36b9357..8058d82 100644 --- a/src/api/wado/routes.rs +++ b/src/api/wado/routes.rs @@ -1,4 +1,4 @@ -use crate::api::wado::RetrieveInstanceRequest; +use crate::api::wado::{RenderedRequest, RetrieveInstanceRequest}; use crate::backend::dimse::wado::DicomMultipartStream; use crate::backend::ServiceProvider; use crate::types::UI; @@ -9,6 +9,7 @@ use axum::http::{Response, StatusCode}; use axum::response::IntoResponse; use axum::routing::get; use axum::Router; +use dicom_pixeldata::image::ImageFormat; use futures::{StreamExt, TryStreamExt}; use std::pin::Pin; use tracing::{error, instrument}; @@ -96,6 +97,45 @@ async fn instance_resource( } } +async fn rendered_resource( + provider: ServiceProvider, + request: RenderedRequest, +) -> impl IntoResponse { + if let Some(wado) = provider.wado { + let response = wado.render(request).await; + + match response { + Ok(response) => { + let image = response.image; + + // Write the image to a buffer (JPEG) + let mut img_buf = Vec::new(); + if let Err(err) = + image.write_to(&mut std::io::Cursor::new(&mut img_buf), ImageFormat::Jpeg) + { + error!("{err:?}"); + return (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } + + Response::builder() + .header(CONTENT_TYPE, "image/jpeg") + .body(Body::from(img_buf)) + .unwrap() + } + Err(err) => { + error!("{err:?}"); + (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + } + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + "WADO-RS endpoint is disabled", + ) + .into_response() + } +} + #[instrument(skip_all)] async fn study_instances( provider: ServiceProvider, @@ -132,20 +172,27 @@ async fn instance_metadata() -> impl IntoResponse { StatusCode::NOT_IMPLEMENTED } -async fn rendered_study() -> impl IntoResponse { - StatusCode::NOT_IMPLEMENTED +#[instrument(skip_all)] +async fn rendered_study(provider: ServiceProvider, request: RenderedRequest) -> impl IntoResponse { + rendered_resource(provider, request).await } -async fn rendered_series() -> impl IntoResponse { - StatusCode::NOT_IMPLEMENTED +#[instrument(skip_all)] +async fn rendered_series(provider: ServiceProvider, request: RenderedRequest) -> impl IntoResponse { + rendered_resource(provider, request).await } -async fn rendered_instance() -> impl IntoResponse { - StatusCode::NOT_IMPLEMENTED +#[instrument(skip_all)] +async fn rendered_instance( + provider: ServiceProvider, + request: RenderedRequest, +) -> impl IntoResponse { + rendered_resource(provider, request).await } -async fn rendered_frames() -> impl IntoResponse { - StatusCode::NOT_IMPLEMENTED +#[instrument(skip_all)] +async fn rendered_frames(provider: ServiceProvider, request: RenderedRequest) -> impl IntoResponse { + rendered_resource(provider, request).await } async fn study_thumbnail() -> impl IntoResponse { diff --git a/src/api/wado/service.rs b/src/api/wado/service.rs index 1195116..82ac067 100644 --- a/src/api/wado/service.rs +++ b/src/api/wado/service.rs @@ -7,9 +7,11 @@ use axum::extract::{FromRef, FromRequestParts, Path, Query}; use axum::http::request::Parts; use axum::response::{IntoResponse, Response}; use dicom::object::{FileDicomObject, InMemDicomObject}; +use dicom_pixeldata::image::DynamicImage; use futures::stream::BoxStream; -use serde::{Deserialize, Serialize}; -use std::fmt::Debug; +use serde::de::{Error, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; +use std::fmt::{Debug, Formatter}; use std::num::ParseIntError; use std::str::FromStr; use std::sync::Arc; @@ -21,6 +23,8 @@ pub trait WadoService: Send + Sync { &self, request: RetrieveInstanceRequest, ) -> Result; + + async fn render(&self, request: RenderedRequest) -> Result; } #[derive(Debug, Error)] @@ -30,6 +34,7 @@ pub enum RetrieveError { } pub type RetrieveInstanceRequest = RetrieveRequest; +pub type RenderedRequest = RetrieveRequest; pub struct RetrieveRequest { pub query: ResourceQuery, @@ -64,10 +69,41 @@ where } } +#[async_trait] +impl FromRequestParts for RenderedRequest +where + AppState: FromRef, + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + let Path(query): Path = Path::from_request_parts(parts, state) + .await + .map_err(PathRejection::into_response)?; + + let Query(parameters): Query = + Query::from_request_parts(parts, state) + .await + .map_err(QueryRejection::into_response)?; + + Ok(Self { + query, + parameters, + // TODO: currently unused + headers: RequestHeaderFields::default(), + }) + } +} + pub struct InstanceResponse { pub stream: BoxStream<'static, Result>, MoveError>>, } +pub struct RenderedResponse { + pub image: DynamicImage, +} + #[derive(Debug, Deserialize)] pub struct ResourceQuery { #[serde(rename = "aet")] @@ -108,7 +144,7 @@ pub struct MetadataQueryParameters { pub charset: Option, } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Deserialize)] pub struct ImageQuality(u8); impl ImageQuality { @@ -156,9 +192,73 @@ pub enum ImageAnnotation { Technique, } +/// Controls the viewport scaling of the images or video +/// +/// https://dicom.nema.org/medical/dicom/current/output/chtml/part18/sect_8.3.5.html#sect_8.3.5.1.3 +#[derive(Debug, Deserialize, PartialEq)] +pub struct Viewport { + /// Width of the viewport in pixels. + pub viewport_width: u32, + /// Height of the viewport in pixels + pub viewport_height: u32, + /// Offset of the top-left corner of the viewport from the top-left corner of the image in pixels along the horizontal axis. + pub source_xpos: Option, + /// Offset of the top-left corner of the viewport from the top-left corner of the image in pixels along the vertical axis. + pub source_ypos: Option, + /// Width of the source region to use in pixels. + pub source_width: Option, + /// Height of the source region to use in pixels. + pub source_height: Option, +} + +struct ViewportVisitor; + +impl<'a> Visitor<'a> for ViewportVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + write!(formatter, "a value of ") + } + + fn visit_str(self, v: &str) -> Result + where + E: Error, + { + let values = v.split(',').collect::>(); + match values.len() { + 2 => Ok(Some(Viewport { + viewport_width: values[0].parse().map_err(E::custom)?, + viewport_height: values[1].parse().map_err(E::custom)?, + source_xpos: None, + source_ypos: None, + source_width: None, + source_height: None, + })), + 6 => Ok(Some(Viewport { + viewport_width: values[0].parse().map_err(E::custom)?, + viewport_height: values[1].parse().map_err(E::custom)?, + source_xpos: Some(values[2].parse().map_err(E::custom)?), + source_ypos: Some(values[3].parse().map_err(E::custom)?), + source_width: Some(values[4].parse().map_err(E::custom)?), + source_height: Some(values[5].parse().map_err(E::custom)?), + })), + _ => Err(E::custom("expected 2 or 6 comma-separated values")), + } + } +} + +// See [`ViewportVisitor`]. +fn deserialize_viewport<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + deserializer.deserialize_any(ViewportVisitor) +} + /// Controls the windowing of the images or video as defined in Section C.8.11.3.1.5 in PS3.3. /// /// +#[derive(Debug, Deserialize, PartialEq)] pub struct Window { /// Decimal number containing the window-center value. pub center: f64, @@ -168,7 +268,46 @@ pub struct Window { pub function: VoiLutFunction, } +/// Custom deserialization visitor for repeated `includefield` query parameters. +/// It collects all `includefield` parameters in [`crate::dicomweb::qido::IncludeField::List`]. +/// If at least one `includefield` parameter has the value `all`, +/// [`crate::dicomweb::qido::IncludeField::All`] is returned instead. +struct WindowVisitor; + +impl<'a> Visitor<'a> for WindowVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + write!(formatter, "a value of <{{attribute}}* | all>") + } + + fn visit_str(self, v: &str) -> Result + where + E: Error, + { + let values = v.split(',').collect::>(); + if values.len() != 3 { + return Err(E::custom("expected 3 comma-separated values")); + } + + Ok(Some(Window { + center: values[0].parse().map_err(E::custom)?, + width: values[1].parse().map_err(E::custom)?, + function: values[2].parse().map_err(E::custom)?, + })) + } +} + +/// See [`WindowVisitor`]. +fn deserialize_window<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + deserializer.deserialize_any(WindowVisitor) +} + /// +#[derive(Debug, Deserialize, PartialEq)] pub enum VoiLutFunction { /// Linear, @@ -184,6 +323,25 @@ impl Default for VoiLutFunction { } } +#[derive(Debug, Error)] +pub enum ParseVoiLutFunctionError { + #[error("Unknown VOI LUT function: {function}")] + UnknownFunction { function: String }, +} + +impl FromStr for VoiLutFunction { + type Err = ParseVoiLutFunctionError; + + fn from_str(s: &str) -> Result { + match s { + "LINEAR" => Ok(Self::Linear), + "LINEAR_EXACT" => Ok(Self::LinearExact), + "SIGMOID" => Ok(Self::Sigmoid), + _ => Err(ParseVoiLutFunctionError::UnknownFunction { function: s.into() }), + } + } +} + /// Specifies the inclusion of an ICC Profile in the rendered images. /// /// @@ -218,13 +376,15 @@ impl ImageAnnotation { } } -#[derive(Debug, Default)] +#[derive(Debug, Default, Deserialize, PartialEq)] pub struct RenderedQueryParameters { pub accept: Option, pub annotation: Option, pub quality: Option, - pub viewport: Option, - pub window: Option, + #[serde(deserialize_with = "deserialize_viewport", default)] + pub viewport: Option, + #[serde(deserialize_with = "deserialize_window", default)] + pub window: Option, pub iccprofile: Option, } @@ -236,6 +396,9 @@ pub struct ThumbnailQueryParameters { #[cfg(test)] mod tests { + use axum::extract::Query; + use axum::http::Uri; + use super::*; #[test] @@ -259,4 +422,34 @@ mod tests { ImageQuality::new(0).unwrap() ); } + + #[test] + fn parse_rendered_query_params() { + let uri = + Uri::from_static("http://test?window=100,200,SIGMOID&viewport=100,100,0,0,100,100"); + let Query(params) = Query::::try_from_uri(&uri).unwrap(); + + assert_eq!( + params, + RenderedQueryParameters { + accept: None, + annotation: None, + quality: None, + viewport: Some(Viewport { + viewport_width: 100, + viewport_height: 100, + source_xpos: Some(0), + source_ypos: Some(0), + source_width: Some(100), + source_height: Some(100), + }), + window: Some(Window { + center: 100.0, + width: 200.0, + function: VoiLutFunction::Sigmoid, + }), + iccprofile: None, + } + ); + } } diff --git a/src/backend/dimse/wado.rs b/src/backend/dimse/wado.rs index f16e2bf..3c1114e 100644 --- a/src/backend/dimse/wado.rs +++ b/src/backend/dimse/wado.rs @@ -1,4 +1,7 @@ -use crate::api::wado::{InstanceResponse, RetrieveError, RetrieveInstanceRequest, WadoService}; +use crate::api::wado::{ + InstanceResponse, RenderedRequest, RenderedResponse, RetrieveError, RetrieveInstanceRequest, + WadoService, +}; use crate::backend::dimse::association; use crate::backend::dimse::cmove::movescu::{MoveError, MoveServiceClassUser}; use crate::backend::dimse::cmove::{ @@ -14,6 +17,8 @@ use async_trait::async_trait; use dicom::core::VR; use dicom::dictionary_std::tags; use dicom::object::{FileDicomObject, InMemDicomObject}; +use dicom_pixeldata::image::{self, DynamicImage}; +use dicom_pixeldata::{ConvertOptions, PixelDecoder, VoiLutOption, WindowLevel}; use futures::stream::BoxStream; use futures::{Stream, StreamExt}; use pin_project::pin_project; @@ -65,6 +70,101 @@ impl WadoService for DimseWadoService { stream: stream.boxed(), }) } + + async fn render(&self, request: RenderedRequest) -> Result { + if self.config.receivers.len() > 1 { + warn!("Multiple receivers are not supported yet."); + } + + let storescp_aet = self + .config + .receivers + .first() // TODO + .ok_or_else(|| RetrieveError::Backend { + source: anyhow::Error::new(DimseRetrieveError::MissingReceiver { + aet: request.query.aet.clone(), + }), + })?; + + let mut stream = self + .retrieve_instances( + &request.query.aet, + storescp_aet, + Self::create_identifier(Some(&request.query.study_instance_uid), None, None), + ) + .await; + + while let Some(result) = stream.next().await { + match result { + Ok(dicom_file) => { + trace!( + "Rendering {}", + dicom_file.meta().media_storage_sop_instance_uid() + ); + + let pixel_data = + dicom_file + .decode_pixel_data() + .map_err(|e| RetrieveError::Backend { + source: anyhow::anyhow!("Failed to decode pixel data"), + })?; + + // Convert the pixel data to an image + let options = match &request.parameters.window { + Some(windowing) => ConvertOptions::new() + .with_voi_lut(VoiLutOption::Custom(WindowLevel { + center: windowing.center, + width: windowing.width, + })) + .force_8bit(), + None => ConvertOptions::default().force_8bit(), + }; + let image = pixel_data + .to_dynamic_image_with_options(0, &options) + .map_err(|e| { + error!("Failed to convert pixel data to image: {}", e); + RetrieveError::Backend { + source: anyhow::anyhow!("Failed to decode pixel data"), + } + })?; + let rescaled = match request.parameters.viewport { + Some(viewport) => { + // 1. Crop our image to the source rectangle + // 2. Scale the cropped image to the viewport size + // 3. Center the scaled image on a new canvas of the viewport size + let scaled = image + .crop_imm( + viewport.source_xpos.unwrap_or(0), + viewport.source_ypos.unwrap_or(0), + viewport.source_width.unwrap_or(image.width()), + viewport.source_height.unwrap_or(image.height()), + ) + .thumbnail(viewport.viewport_width, viewport.viewport_height); + let mut canvas = DynamicImage::new( + viewport.viewport_width, + viewport.viewport_height, + scaled.color(), + ); + let dx = (canvas.width() - scaled.width()) / 2; + let dy = (canvas.height() - scaled.height()) / 2; + image::imageops::overlay(&mut canvas, &scaled, dx as i64, dy as i64); + canvas + } + None => image, + }; + + return Ok(RenderedResponse { image: rescaled }); + } + Err(err) => { + error!("{:?}", err); + } + } + } + + Err(RetrieveError::Backend { + source: anyhow::anyhow!("No renderable instance found"), + }) + } } #[derive(Debug, Error)] diff --git a/src/backend/s3/wado.rs b/src/backend/s3/wado.rs index af5dca3..68a2742 100644 --- a/src/backend/s3/wado.rs +++ b/src/backend/s3/wado.rs @@ -1,4 +1,7 @@ -use crate::api::wado::{InstanceResponse, RetrieveError, RetrieveInstanceRequest, WadoService}; +use crate::api::wado::{ + InstanceResponse, RenderedRequest, RenderedResponse, RetrieveError, RetrieveInstanceRequest, + WadoService, +}; use crate::backend::dimse::cmove::movescu::MoveError; use crate::config::{S3Config, S3EndpointStyle}; use async_trait::async_trait; @@ -115,4 +118,8 @@ impl WadoService for S3WadoService { stream: stream.boxed(), }) } + + async fn render(&self, _request: RenderedRequest) -> Result { + unimplemented!() + } }