diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py
index 07a5bbadee6c..22adb1631d41 100644
--- a/vllm/inputs/preprocess.py
+++ b/vllm/inputs/preprocess.py
@@ -386,7 +386,8 @@ def _process_encoder_decoder_prompt(
                     decoder_input,
                     request_id=request_id,
                 )
-            mm_processor_kwargs = prompt["mm_processor_kwargs"]
+            # Handle this carefully in case it was directly initialized by user
+            mm_processor_kwargs = prompt.get("mm_processor_kwargs", {})
         else:
             encoder_comps = self._extract_prompt_components(
                 prompt,