use IP-Adapter masking RuntimeError: mat1 and mat2 shapes cannot be multiplied (1028x1664 and 1280x1280)

[1272870698]

code is

from diffusers import StableDiffusionXLPipeline 
import torch


from diffusers.image_processor import IPAdapterMaskProcessor
from diffusers.utils import load_image

from diffusers import AutoPipelineForText2Image,StableDiffusionXLPipeline
from diffusers.utils import load_image
import torch
local_xl_2='F:\\SD\\sd-webui-aki-v4.5\\models\\Stable-diffusion\\XXMix_9realisticSDXL_v1.safetensors'

xl_pipeline=StableDiffusionXLPipeline.from_single_file(local_xl_2,torch_dtype=torch.float16).to('cuda')

pipeline = AutoPipelineForText2Image.from_pipe(xl_pipeline, torch_dtype=torch.float16).to("cuda")


mask1 = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/ip_mask_mask1.png")
mask2 = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/ip_mask_mask2.png")

mask1.save('ip_mask_mask1.png')
mask2.save('ip_mask_mask2.png')

output_height = 1024
output_width = 1024

processor = IPAdapterMaskProcessor()
masks = processor.preprocess([mask1, mask2], height=output_height, width=output_width)



pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models",cache_dir=cache_dir, weight_name=["ip-adapter-plus-face_sdxl_vit-h.safetensors"])
pipeline.set_ip_adapter_scale([[0.7, 0.7]])  # one scale for each image-mask pair

face_image1 = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/ip_mask_girl1.png")
face_image2 = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/ip_mask_girl2.png")




ip_images = [[face_image1, face_image2]]

masks = [masks.reshape(1, masks.shape[0], masks.shape[2], masks.shape[3])]

generator = torch.Generator(device="cuda").manual_seed(0)
num_images = 1



image = pipeline(
    prompt="2 girls",
    ip_adapter_image=ip_images,
    negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
    num_inference_steps=20,
    num_images_per_prompt=num_images,
    generator=generator,
    cross_attention_kwargs={"ip_adapter_masks": masks}
).images[0]

error is

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[3], line 52
     47 generator = torch.Generator(device="cuda").manual_seed(0)
     48 num_images = 1
---> 52 image = pipeline(
     53     prompt="2 girls",
     54     ip_adapter_image=ip_images,
     55     negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
     56     num_inference_steps=20,
     57     num_images_per_prompt=num_images,
     58     generator=generator,
     59     cross_attention_kwargs={"ip_adapter_masks": masks}
     60 ).images[0]

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\utils\_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\diffusers\pipelines\stable_diffusion_xl\pipeline_stable_diffusion_xl.py:1199, in StableDiffusionXLPipeline.__call__(self, prompt, prompt_2, height, width, num_inference_steps, timesteps, sigmas, denoising_end, guidance_scale, negative_prompt, negative_prompt_2, num_images_per_prompt, eta, generator, latents, prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds, ip_adapter_image, ip_adapter_image_embeds, output_type, return_dict, cross_attention_kwargs, guidance_rescale, original_size, crops_coords_top_left, target_size, negative_original_size, negative_crops_coords_top_left, negative_target_size, clip_skip, callback_on_step_end, callback_on_step_end_tensor_inputs, **kwargs)
   1197 if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
   1198     added_cond_kwargs["image_embeds"] = image_embeds
-> 1199 noise_pred = self.unet(
   1200     latent_model_input,
   1201     t,
   1202     encoder_hidden_states=prompt_embeds,
   1203     timestep_cond=timestep_cond,
   1204     cross_attention_kwargs=self.cross_attention_kwargs,
   1205     added_cond_kwargs=added_cond_kwargs,
   1206     return_dict=False,
   1207 )[0]
   1209 # perform guidance
   1210 if self.do_classifier_free_guidance:

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs)
   1530     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1531 else:
-> 1532     return self._call_impl(*args, **kwargs)

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1541, in Module._call_impl(self, *args, **kwargs)
   1536 # If we don't have any hooks, we want to skip the rest of the logic in
   1537 # this function, and just call forward.
   1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1539         or _global_backward_pre_hooks or _global_backward_hooks
   1540         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1541     return forward_call(*args, **kwargs)
   1543 try:
   1544     result = None

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\diffusers\models\unets\unet_2d_condition.py:1164, in UNet2DConditionModel.forward(self, sample, timestep, encoder_hidden_states, class_labels, timestep_cond, attention_mask, cross_attention_kwargs, added_cond_kwargs, down_block_additional_residuals, mid_block_additional_residual, down_intrablock_additional_residuals, encoder_attention_mask, return_dict)
   1161 if self.time_embed_act is not None:
   1162     emb = self.time_embed_act(emb)
-> 1164 encoder_hidden_states = self.process_encoder_hidden_states(
   1165     encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs
   1166 )
   1168 # 2. pre-process
   1169 sample = self.conv_in(sample)

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\diffusers\models\unets\unet_2d_condition.py:1035, in UNet2DConditionModel.process_encoder_hidden_states(self, encoder_hidden_states, added_cond_kwargs)
   1032         encoder_hidden_states = self.text_encoder_hid_proj(encoder_hidden_states)
   1034     image_embeds = added_cond_kwargs.get("image_embeds")
-> 1035     image_embeds = self.encoder_hid_proj(image_embeds)
   1036     encoder_hidden_states = (encoder_hidden_states, image_embeds)
   1037 return encoder_hidden_states

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs)
   1530     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1531 else:
-> 1532     return self._call_impl(*args, **kwargs)

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1541, in Module._call_impl(self, *args, **kwargs)
   1536 # If we don't have any hooks, we want to skip the rest of the logic in
   1537 # this function, and just call forward.
   1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1539         or _global_backward_pre_hooks or _global_backward_hooks
   1540         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1541     return forward_call(*args, **kwargs)
   1543 try:
   1544     result = None

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\diffusers\models\embeddings.py:1703, in MultiIPAdapterImageProjection.forward(self, image_embeds)
   1701 batch_size, num_images = image_embed.shape[0], image_embed.shape[1]
   1702 image_embed = image_embed.reshape((batch_size * num_images,) + image_embed.shape[2:])
-> 1703 image_embed = image_projection_layer(image_embed)
   1704 image_embed = image_embed.reshape((batch_size, num_images) + image_embed.shape[1:])
   1706 projected_image_embeds.append(image_embed)

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs)
   1530     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1531 else:
-> 1532     return self._call_impl(*args, **kwargs)

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1541, in Module._call_impl(self, *args, **kwargs)
   1536 # If we don't have any hooks, we want to skip the rest of the logic in
   1537 # this function, and just call forward.
   1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1539         or _global_backward_pre_hooks or _global_backward_hooks
   1540         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1541     return forward_call(*args, **kwargs)
   1543 try:
   1544     result = None

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\diffusers\models\embeddings.py:1586, in IPAdapterPlusImageProjection.forward(self, x)
   1577 """Forward pass.
   1578 
   1579 Args:
   (...)
   1582     torch.Tensor: Output Tensor.
   1583 """
   1584 latents = self.latents.repeat(x.size(0), 1, 1)
-> 1586 x = self.proj_in(x)
   1588 for block in self.layers:
   1589     residual = latents

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1532, in Module._wrapped_call_impl(self, *args, **kwargs)
   1530     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1531 else:
-> 1532     return self._call_impl(*args, **kwargs)

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\module.py:1541, in Module._call_impl(self, *args, **kwargs)
   1536 # If we don't have any hooks, we want to skip the rest of the logic in
   1537 # this function, and just call forward.
   1538 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1539         or _global_backward_pre_hooks or _global_backward_hooks
   1540         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1541     return forward_call(*args, **kwargs)
   1543 try:
   1544     result = None

File D:\softs\anaconda3\envs\unsloth_env_v2\lib\site-packages\torch\nn\modules\linear.py:116, in Linear.forward(self, input)
    115 def forward(self, input: Tensor) -> Tensor:
--> 116     return F.linear(input, self.weight, self.bias)

RuntimeError: mat1 and mat2 shapes cannot be multiplied (1028x1664 and 1280x1280)

pelase help me

[asomoza]

Hi, your code is not formatted so it's hard to read, but probably your error is because you're not changing the image encoder, if you use the vit-h IPAs you need to use the corresponding image encoder:

pipeline.load_ip_adapter(
    "h94/IP-Adapter",
    subfolder="sdxl_models",
    cache_dir=cache_dir,
    weight_name=["ip-adapter-plus-face_sdxl_vit-h.safetensors"],
    image_encoder_folder="models/image_encoder",
)

[1272870698]

I change code is

pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models",cache_dir=cache_dir,
                         image_encoder_folder="sdxl_models/image_encoder", 
                         weight_name=["ip-adapter-plus-face_sdxl_vit-h.safetensors"])

The same problem exists

[asomoza]

you're not changing the image encoder, the sdxl_models/image_encoder is the same one, you'll need to use the one in the models, the same one as SD 1.5 which is in models/image_encoder

[1272870698]

thank you , its ok . but why image encoder must use SD 1.5 -.-!

[asomoza]

It's not using SD 1.5, is just the same one, the image encoder is not part of the main difussion model, it's just used to extract the features for the images.

You can read the original IP Adapter repo, they tried using the big one which is the one in the sdxl_model directory but they found that it makes not difference apart from using more VRAM so for the later ones they just used the smaller image encoder.

Switch to CLIP-ViT-H: we trained the new IP-Adapter with OpenCLIP-ViT-H-14 instead of OpenCLIP-ViT-bigG-14. Although ViT-bigG is much larger than ViT-H, our experimental results did not find a significant difference, and the smaller model can reduce the memory usage in the inference phase.

[J4BEZ]

@asomoza Thanks for sharing your solution!
I was facing the same problem, and solved it thanks to your solution!

Hope you have a peaceful day!