paddleMix的flux工作管线FluxPipeline加载lora的功能有bug

[eleven-monkey]

下面是我的代码：
`import os
os.environ["USE_PEFT_BACKEND"] = "True" # 启用 PEFT 后端
import paddle

from ppdiffusers import FluxPipeline

pipe = FluxPipeline.from_pretrained(
"black-forest-labs/FLUX.1-dev", paddle_dtype=paddle.float16, low_cpu_mem_usage=True, map_location="cpu",
)
pipe.load_lora_weights("/home/aistudio/lora.safetensors",from_diffusers=True)

prompt = "Battle-scarred armored mecha warrior with plasma-edged blades, mid-leap through debris fields near a turquoise gas giant swirling with storm vortices. Modular space station rotates behind, its hexagonal docks ablaze with ion thrusters. Fractal energy beams pierce crystalline asteroid fragments, casting prismatic flares across reflective exoskeletons. Volumetric planetary glow illuminates nebula-dusted cosmic void, with lens flare accents and hyperrealistic metallic surface textures."
image = pipe(
prompt,
height=1024,
width=768,
guidance_scale=3.5,
num_inference_steps=50,
max_sequence_length=512,
generator=paddle.Generator().manual_seed(8565564154156)
).images[0]
image.save("text_to_image_generation-flux-dev-result.png")`

加载过程没有报错，一切正常，但是推理的时候就报错了，说维度不一致。
4 random_seed = random.randint(0,2**32-1)
----> 5 image = pipe(
6 prompt,
7 height=1024,
8 width=768,
9 guidance_scale=3.5,
10 num_inference_steps=50,
11 max_sequence_length=512,
12 generator=paddle.Generator().manual_seed(random_seed)
13 ).images[0]
14 image.save("text_to_image_generation-flux-dev-result.png")
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/decorator.py:235, in decorate..fun(*args, **kw)
233 if not kwsyntax:
234 args, kw = fix(args, kw, sig)
--> 235 return caller(func, *(extras + args), **kw)
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/base/dygraph/base.py:400, in _DecoratorContextManager.call.._decorate_function(func, *args, **kwargs)
397 @decorator.decorator
398 def _decorate_function(func, *args, **kwargs):
399 with self:
--> 400 return func(*args, **kwargs)
File ~/PaddleMIX/ppdiffusers/ppdiffusers/pipelines/flux/pipeline_flux.py:857, in FluxPipeline.call(self, prompt, prompt_2, negative_prompt, negative_prompt_2, true_cfg_scale, height, width, num_inference_steps, sigmas, guidance_scale, num_images_per_prompt, generator, latents, prompt_embeds, pooled_prompt_embeds, ip_adapter_image, ip_adapter_image_embeds, negative_ip_adapter_image, negative_ip_adapter_image_embeds, negative_prompt_embeds, negative_pooled_prompt_embeds, text_ids, output_type, return_dict, joint_attention_kwargs, callback_on_step_end, callback_on_step_end_tensor_inputs, max_sequence_length)
854 # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
855 timestep = t.expand(latents.shape[0]).astype(latents.dtype)
--> 857 noise_pred = self.transformer(
858 hidden_states=latents,
859 timestep=timestep / 1000,
860 guidance=guidance,
861 pooled_projections=pooled_prompt_embeds,
862 encoder_hidden_states=prompt_embeds,
863 txt_ids=text_ids,
864 img_ids=latent_image_ids,
865 joint_attention_kwargs=self.joint_attention_kwargs,
866 return_dict=False,
867 )[0]
869 if do_true_cfg:
870 if negative_image_embeds is not None:
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/nn/layer/layers.py:1567, in Layer.call(self, *inputs, **kwargs)
1558 def call(self, *inputs: Any, **kwargs: Any) -> Any:
1559 if (
1560 (not in_to_static_mode())
1561 and (not self._forward_pre_hooks)
(...)
1565 and (not in_profiler_mode() or in_sot_simulation_mode())
1566 ):
-> 1567 return self.forward(*inputs, **kwargs)
1568 else:
1569 return self._dygraph_call_func(*inputs, **kwargs)
File ~/PaddleMIX/ppdiffusers/ppdiffusers/models/transformer_flux.py:518, in FluxTransformer2DModel.forward(self, hidden_states, encoder_hidden_states, pooled_projections, timestep, img_ids, txt_ids, guidance, joint_attention_kwargs, controlnet_block_samples, controlnet_single_block_samples, return_dict, controlnet_blocks_repeat)
508 encoder_hidden_states, hidden_states = paddle.utils.checkpoint.checkpoint(
509 create_custom_forward(block),
510 hidden_states,
(...)
514 **ckpt_kwargs,
515 )
517 else:
--> 518 encoder_hidden_states, hidden_states = block(
519 hidden_states=hidden_states,
520 encoder_hidden_states=encoder_hidden_states,
521 temb=temb,
522 image_rotary_emb=image_rotary_emb,
523 joint_attention_kwargs=joint_attention_kwargs,
524 )
526 # controlnet residual
527 if controlnet_block_samples is not None:
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/nn/layer/layers.py:1567, in Layer.call(self, *inputs, **kwargs)
1558 def call(self, *inputs: Any, **kwargs: Any) -> Any:
1559 if (
1560 (not in_to_static_mode())
1561 and (not self._forward_pre_hooks)
(...)
1565 and (not in_profiler_mode() or in_sot_simulation_mode())
1566 ):
-> 1567 return self.forward(*inputs, **kwargs)
1568 else:
1569 return self._dygraph_call_func(*inputs, **kwargs)
File ~/PaddleMIX/ppdiffusers/ppdiffusers/models/transformer_flux.py:169, in FluxTransformerBlock.forward(self, hidden_states, encoder_hidden_states, temb, image_rotary_emb, joint_attention_kwargs)
161 def forward(
162 self,
163 hidden_states: paddle.Tensor,
(...)
167 joint_attention_kwargs=None,
168 ):
--> 169 norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb)
171 norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context(
172 encoder_hidden_states, emb=temb
173 )
174 joint_attention_kwargs = joint_attention_kwargs or {}
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/nn/layer/layers.py:1567, in Layer.call(self, *inputs, **kwargs)
1558 def call(self, *inputs: Any, **kwargs: Any) -> Any:
1559 if (
1560 (not in_to_static_mode())
1561 and (not self._forward_pre_hooks)
(...)
1565 and (not in_profiler_mode() or in_sot_simulation_mode())
1566 ):
-> 1567 return self.forward(*inputs, **kwargs)
1568 else:
1569 return self._dygraph_call_func(*inputs, **kwargs)
File ~/PaddleMIX/ppdiffusers/ppdiffusers/models/normalization.py:172, in AdaLayerNormZero.forward(self, x, timestep, class_labels, hidden_dtype, emb)
170 if self.emb is not None:
171 emb = self.emb(timestep, class_labels, hidden_dtype=hidden_dtype)
--> 172 emb = self.linear(self.silu(emb))
173 shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, axis=1)
174 x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/nn/layer/layers.py:1567, in Layer.call(self, *inputs, **kwargs)
1558 def call(self, *inputs: Any, **kwargs: Any) -> Any:
1559 if (
1560 (not in_to_static_mode())
1561 and (not self._forward_pre_hooks)
(...)
1565 and (not in_profiler_mode() or in_sot_simulation_mode())
1566 ):
-> 1567 return self.forward(*inputs, **kwargs)
1568 else:
1569 return self._dygraph_call_func(*inputs, **kwargs)
File ~/PaddleMIX/ppdiffusers/ppdiffusers/peft/tuners/lora/layer.py:340, in Linear.forward(self, x, *args, **kwargs)
338 scaling = self.scaling[active_adapter]
339 x = x.cast(dtype=lora_A.weight.dtype)
--> 340 result += lora_B(lora_A(dropout(x))) * scaling
342 result = result.cast(dtype=previous_dtype)
343 return result
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/nn/layer/layers.py:1567, in Layer.call(self, *inputs, **kwargs)
1558 def call(self, *inputs: Any, **kwargs: Any) -> Any:
1559 if (
1560 (not in_to_static_mode())
1561 and (not self._forward_pre_hooks)
(...)
1565 and (not in_profiler_mode() or in_sot_simulation_mode())
1566 ):
-> 1567 return self.forward(*inputs, **kwargs)
1568 else:
1569 return self._dygraph_call_func(*inputs, **kwargs)
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/nn/layer/common.py:223, in Linear.forward(self, input)
222 def forward(self, input: Tensor) -> Tensor:
--> 223 out = F.linear(
224 x=input, weight=self.weight, bias=self.bias, name=self.name
225 )
226 return out
File ~/external-libraries/lib/python3.10/site-packages/paddlenlp/transformers/deepseek_v2/fp8_linear.py:75, in fp8_linear(x, weight, bias, name)
73 if paddle.in_dynamic_mode():
74 if weight.element_size() > 1:
---> 75 return original_linear(x, weight, bias)
76 elif gemm_impl == "bf16":
77 weight = weight_dequant(weight, weight._scale)
File /opt/conda/envs/python35-paddle120-env/lib/python3.10/site-packages/paddle/nn/functional/common.py:2238, in linear(x, weight, bias, name)
2178 r"""
2179
2180 Fully-connected linear transformation operator. For each input :math:X ,
(...)
2234 [-0.67769694, -0.67769694, -0.67769694, -0.67769694]])
2235 """
2236 if in_dynamic_mode():
2237 # TODO(jiabin): using addmm for fast forward route
-> 2238 return _C_ops.linear(x, weight, bias)
2240 elif in_pir_mode():
2241 out = _C_ops.matmul(x, weight, False, False)
ValueError: (InvalidArgument) Input(Y) has error dim. Y'dims[0] must be equal to 3072, but received Y'dims[0] is 64.
[Hint: Expected y_dims[y_ndim - 2] == K, but received y_dims[y_ndim - 2]:64 != K:3072.] (at ../paddle/phi/kernels/impl/matmul_kernel_impl.h:331)
[operator < linear > error]