From bda2f184690fe76f1b6f776f4b7b1de5dbd5076d Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 16 Jul 2025 23:40:12 +0200 Subject: [PATCH 1/7] make modular pipeline work with model_index.json --- .../modular_pipelines/modular_pipeline.py | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index b99478cb58d1..5bf473078272 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -1876,18 +1876,38 @@ def __init__( # update component_specs and config_specs from modular_repo if pretrained_model_name_or_path is not None: - config_dict = self.load_config(pretrained_model_name_or_path, **kwargs) - - for name, value in config_dict.items(): - # all the components in modular_model_index.json are from_pretrained components - if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 3: - library, class_name, component_spec_dict = value - component_spec = self._dict_to_component_spec(name, component_spec_dict) - component_spec.default_creation_method = "from_pretrained" - self._component_specs[name] = component_spec - - elif name in self._config_specs: - self._config_specs[name].default = value + + try: + config_dict = self.load_config(pretrained_model_name_or_path, **kwargs) + for name, value in config_dict.items(): + # all the components in modular_model_index.json are from_pretrained components + if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 3: + library, class_name, component_spec_dict = value + component_spec = self._dict_to_component_spec(name, component_spec_dict) + component_spec.default_creation_method = "from_pretrained" + self._component_specs[name] = component_spec + + elif name in self._config_specs: + self._config_specs[name].default = value + + except EnvironmentError as e: + logger.debug(e) + logger.debug(f" modular_model_index.json not found in the repo, trying to load from model_index.json") + from diffusers import DiffusionPipeline + config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path) + for name, value in config_dict.items(): + if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 2: + library, class_name = value + component_spec_dict = { + "repo": pretrained_model_name_or_path, + "subfolder": name, + "type_hint": (library, class_name), + } + component_spec = self._dict_to_component_spec(name, component_spec_dict) + component_spec.default_creation_method = "from_pretrained" + self._component_specs[name] = component_spec + elif name in self._config_specs: + self._config_specs[name].default = value register_components_dict = {} for name, component_spec in self._component_specs.items(): From 2c45daff1c08a22f04eb3bb7a81aa1894a79fb60 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 17 Jul 2025 01:19:08 +0200 Subject: [PATCH 2/7] up --- .../modular_pipelines/modular_pipeline.py | 28 ++++++++++++----- src/diffusers/pipelines/pipeline_utils.py | 30 +++++++++++++++++++ 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 5bf473078272..601b03dc90db 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -1876,7 +1876,6 @@ def __init__( # update component_specs and config_specs from modular_repo if pretrained_model_name_or_path is not None: - try: config_dict = self.load_config(pretrained_model_name_or_path, **kwargs) for name, value in config_dict.items(): @@ -1892,8 +1891,9 @@ def __init__( except EnvironmentError as e: logger.debug(e) - logger.debug(f" modular_model_index.json not found in the repo, trying to load from model_index.json") + logger.debug(" modular_model_index.json not found in the repo, trying to load from model_index.json") from diffusers import DiffusionPipeline + config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path) for name, value in config_dict.items(): if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 2: @@ -2435,17 +2435,31 @@ def update_components(self, **kwargs): for name, component in passed_components.items(): current_component_spec = self._component_specs[name] - # warn if type changed + # log if type changed if current_component_spec.type_hint is not None and not isinstance( component, current_component_spec.type_hint ): - logger.warning( + logger.info( f"ModularPipeline.update_components: adding {name} with new type: {component.__class__.__name__}, previous type: {current_component_spec.type_hint.__name__}" ) # update _component_specs based on the new component - new_component_spec = ComponentSpec.from_component(name, component) - if new_component_spec.default_creation_method != current_component_spec.default_creation_method: + if component is None: + new_component_spec = current_component_spec + if hasattr(self, name) and getattr(self, name) is not None: + logger.warning(f"ModularPipeline.update_components: setting {name} to None (spec unchanged)") + elif current_component_spec.default_creation_method == "from_pretrained" and not ( + hasattr(component, "_diffusers_load_id") and component._diffusers_load_id is not None + ): logger.warning( + f"ModularPipeline.update_components: {name} has no valid _diffusers_load_id. " + f"Updating the component but skipping spec update, use ComponentSpec.load() for proper specs" + ) + new_component_spec = current_component_spec + else: + new_component_spec = ComponentSpec.from_component(name, component) + + if new_component_spec.default_creation_method != current_component_spec.default_creation_method: + logger.info( f"ModularPipeline.update_components: changing the default_creation_method of {name} from {current_component_spec.default_creation_method} to {new_component_spec.default_creation_method}." ) @@ -2466,7 +2480,7 @@ def update_components(self, **kwargs): if current_component_spec.type_hint is not None and not isinstance( created_components[name], current_component_spec.type_hint ): - logger.warning( + logger.info( f"ModularPipeline.update_components: adding {name} with new type: {created_components[name].__class__.__name__}, previous type: {current_component_spec.type_hint.__name__}" ) # update _component_specs based on the user passed component_spec diff --git a/src/diffusers/pipelines/pipeline_utils.py b/src/diffusers/pipelines/pipeline_utils.py index 6b8ba55941b7..a5e2f4965aee 100644 --- a/src/diffusers/pipelines/pipeline_utils.py +++ b/src/diffusers/pipelines/pipeline_utils.py @@ -1706,6 +1706,36 @@ def _get_signature_types(cls): logger.warning(f"cannot get type annotation for Parameter {k} of {cls}.") return signature_types + @property + def parameters(self) -> Dict[str, Any]: + r""" + The `self.parameters` property can be useful to run different pipelines with the same weights and + configurations without reallocating additional memory. + + Returns (`dict`): + A dictionary containing all the optional parameters needed to initialize the pipeline. + + Examples: + + ```py + >>> from diffusers import ( + ... StableDiffusionPipeline, + ... StableDiffusionImg2ImgPipeline, + ... StableDiffusionInpaintPipeline, + ... ) + + >>> text2img = StableDiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5") + >>> img2img = StableDiffusionImg2ImgPipeline(**text2img.components, **text2img.parameters) + >>> inpaint = StableDiffusionInpaintPipeline(**text2img.components, **text2img.parameters) + ``` + """ + expected_modules, optional_parameters = self._get_signature_keys(self) + pipeline_parameters = { + k: self.config[k] for k in self.config.keys() if not k.startswith("_") and k in optional_parameters + } + + return pipeline_parameters + @property def components(self) -> Dict[str, Any]: r""" From b79dcb81e1092bc993caaa2cb842b613900b3ec5 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 17 Jul 2025 02:42:11 +0200 Subject: [PATCH 3/7] style --- .../modular_pipelines/modular_pipeline.py | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 601b03dc90db..f0a2332aa4d2 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -1878,6 +1878,11 @@ def __init__( if pretrained_model_name_or_path is not None: try: config_dict = self.load_config(pretrained_model_name_or_path, **kwargs) + except EnvironmentError as e: + logger.debug(f"modular_model_index.json not found: {e}") + config_dict = None + + if config_dict is not None: for name, value in config_dict.items(): # all the components in modular_model_index.json are from_pretrained components if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 3: @@ -1889,12 +1894,11 @@ def __init__( elif name in self._config_specs: self._config_specs[name].default = value - except EnvironmentError as e: - logger.debug(e) - logger.debug(" modular_model_index.json not found in the repo, trying to load from model_index.json") + else: + logger.debug(" loading config from model_index.json") from diffusers import DiffusionPipeline - config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path) + config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path, **kwargs) for name, value in config_dict.items(): if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 2: library, class_name = value @@ -2094,10 +2098,23 @@ def from_pretrained( try: config_dict = cls.load_config(pretrained_model_name_or_path, **load_config_kwargs) + except EnvironmentError as e: + logger.debug(f" modular_model_index.json not found in the repo: {e}") + config_dict = None + + if config_dict is not None: pipeline_class = _get_pipeline_class(cls, config=config_dict) - except EnvironmentError: - pipeline_class = cls - pretrained_model_name_or_path = None + else: + logger.debug(" determining the modular pipeline class from model_index.json") + from diffusers import DiffusionPipeline + from diffusers.pipelines.auto_pipeline import _get_model + + config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path, **load_config_kwargs) + standard_pipeline_class = _get_pipeline_class(cls, config=config_dict) + model_name = _get_model(standard_pipeline_class.__name__) + pipeline_class_name = MODULAR_PIPELINE_MAPPING.get(model_name, ModularPipeline.__name__) + diffusers_module = importlib.import_module("diffusers") + pipeline_class = getattr(diffusers_module, pipeline_class_name) pipeline = pipeline_class( blocks=blocks, From a1c72f665fa7309036095e84ab47f07bcc7bed6c Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 17 Jul 2025 03:43:47 +0200 Subject: [PATCH 4/7] up --- .../modular_pipelines/modular_pipeline.py | 104 ++++++++++++------ 1 file changed, 73 insertions(+), 31 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index f0a2332aa4d2..5580f6b7d991 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -1825,9 +1825,10 @@ def __init__( Args: blocks: `ModularPipelineBlocks` instance. If None, will attempt to load default blocks based on the pipeline class name. - pretrained_model_name_or_path: Path to a pretrained pipeline configuration. If provided, - will load component specs (only for from_pretrained components) and config values from the saved - modular_model_index.json file. + pretrained_model_name_or_path: Path to a pretrained pipeline configuration. Can be None if the pipeline + does not require any additional loading config. If provided, will first try to load component specs + (only for from_pretrained components) and config values from `modular_model_index.json`, then + fallback to `model_index.json` for compatibility with standard non-modular repositories. components_manager: Optional ComponentsManager for managing multiple component cross different pipelines and apply offloading strategies. @@ -1876,12 +1877,29 @@ def __init__( # update component_specs and config_specs from modular_repo if pretrained_model_name_or_path is not None: + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + + load_config_kwargs = { + "cache_dir": cache_dir, + "force_download": force_download, + "proxies": proxies, + "token": token, + "local_files_only": local_files_only, + "revision": revision, + } + # try to load modular_model_index.json try: - config_dict = self.load_config(pretrained_model_name_or_path, **kwargs) + config_dict = self.load_config(pretrained_model_name_or_path, **load_config_kwargs) except EnvironmentError as e: logger.debug(f"modular_model_index.json not found: {e}") config_dict = None + # update component_specs and config_specs based on modular_model_index.json if config_dict is not None: for name, value in config_dict.items(): # all the components in modular_model_index.json are from_pretrained components @@ -1894,24 +1912,35 @@ def __init__( elif name in self._config_specs: self._config_specs[name].default = value + # if modular_model_index.json is not found, try to load model_index.json else: logger.debug(" loading config from model_index.json") - from diffusers import DiffusionPipeline + try: + from diffusers import DiffusionPipeline + + config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path, **load_config_kwargs) + except EnvironmentError as e: + logger.debug(f" model_index.json not found in the repo: {e}") + config_dict = None + + # update component_specs and config_specs based on model_index.json + if config_dict is not None: + for name, value in config_dict.items(): + if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 2: + library, class_name = value + component_spec_dict = { + "repo": pretrained_model_name_or_path, + "subfolder": name, + "type_hint": (library, class_name), + } + component_spec = self._dict_to_component_spec(name, component_spec_dict) + component_spec.default_creation_method = "from_pretrained" + self._component_specs[name] = component_spec + elif name in self._config_specs: + self._config_specs[name].default = value - config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path, **kwargs) - for name, value in config_dict.items(): - if name in self._component_specs and isinstance(value, (tuple, list)) and len(value) == 2: - library, class_name = value - component_spec_dict = { - "repo": pretrained_model_name_or_path, - "subfolder": name, - "type_hint": (library, class_name), - } - component_spec = self._dict_to_component_spec(name, component_spec_dict) - component_spec.default_creation_method = "from_pretrained" - self._component_specs[name] = component_spec - elif name in self._config_specs: - self._config_specs[name].default = value + if len(kwargs) > 0: + logger.warning(f"Unexpected input '{kwargs.keys()}' provided. This input will be ignored.") register_components_dict = {} for name, component_spec in self._component_specs.items(): @@ -2060,8 +2089,10 @@ def from_pretrained( Args: pretrained_model_name_or_path (`str` or `os.PathLike`, optional): - Path to a pretrained pipeline configuration. If provided, will load component specs (only for - from_pretrained components) and config values from the modular_model_index.json file. + Path to a pretrained pipeline configuration. It will first try to load config from + `modular_model_index.json`, then fallback to `model_index.json` for compatibility with standard + non-modular repositories. If the repo does not contain any pipeline config, it will be set to None + during initialization. trust_remote_code (`bool`, optional): Whether to trust remote code when loading the pipeline, need to be set to True if you want to create pipeline blocks based on the custom code in `pretrained_model_name_or_path` @@ -2097,6 +2128,7 @@ def from_pretrained( } try: + # try to load modular_model_index.json config_dict = cls.load_config(pretrained_model_name_or_path, **load_config_kwargs) except EnvironmentError as e: logger.debug(f" modular_model_index.json not found in the repo: {e}") @@ -2105,16 +2137,26 @@ def from_pretrained( if config_dict is not None: pipeline_class = _get_pipeline_class(cls, config=config_dict) else: - logger.debug(" determining the modular pipeline class from model_index.json") - from diffusers import DiffusionPipeline - from diffusers.pipelines.auto_pipeline import _get_model - - config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path, **load_config_kwargs) - standard_pipeline_class = _get_pipeline_class(cls, config=config_dict) - model_name = _get_model(standard_pipeline_class.__name__) - pipeline_class_name = MODULAR_PIPELINE_MAPPING.get(model_name, ModularPipeline.__name__) - diffusers_module = importlib.import_module("diffusers") - pipeline_class = getattr(diffusers_module, pipeline_class_name) + try: + logger.debug(" try to load model_index.json") + from diffusers import DiffusionPipeline + from diffusers.pipelines.auto_pipeline import _get_model + + config_dict = DiffusionPipeline.load_config(pretrained_model_name_or_path, **load_config_kwargs) + except EnvironmentError as e: + logger.debug(f" model_index.json not found in the repo: {e}") + + if config_dict is not None: + logger.debug(" try to determine the modular pipeline class from model_index.json") + standard_pipeline_class = _get_pipeline_class(cls, config=config_dict) + model_name = _get_model(standard_pipeline_class.__name__) + pipeline_class_name = MODULAR_PIPELINE_MAPPING.get(model_name, ModularPipeline.__name__) + diffusers_module = importlib.import_module("diffusers") + pipeline_class = getattr(diffusers_module, pipeline_class_name) + else: + # there is no config for modular pipeline, assuming that the pipeline block does not need any from_pretrained components + pipeline_class = cls + pretrained_model_name_or_path = None pipeline = pipeline_class( blocks=blocks, From 7add115b658e44b9dc2cea128415a2881a7703a0 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 22 Jul 2025 00:12:01 +0200 Subject: [PATCH 5/7] up --- src/diffusers/modular_pipelines/modular_pipeline.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 5580f6b7d991..d988a26cb3d6 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -210,6 +210,15 @@ def to_dict(self) -> Dict[str, Any]: """ return {**self.__dict__, "inputs": self.inputs, "intermediates": self.intermediates} + def __getattr__(self, name): + """ + Allow attribute access to intermediate values. + If an attribute is not found in the object, look for it in the intermediates dict. + """ + if name in self.intermediates: + return self.intermediates[name] + raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'") + def __repr__(self): def format_value(v): if hasattr(v, "shape") and hasattr(v, "dtype"): @@ -874,7 +883,7 @@ def __call__(self, pipeline, state: PipelineState) -> PipelineState: break if block is None: - logger.warning(f"skipping auto block: {self.__class__.__name__}") + logger.info(f"skipping auto block: {self.__class__.__name__}") return pipeline, state try: From 0be3dec1f7ecd04c4e49bfc16547b9984f5e8acf Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 22 Jul 2025 00:37:59 +0200 Subject: [PATCH 6/7] style --- src/diffusers/modular_pipelines/modular_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index d988a26cb3d6..ce0068e0369c 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -212,8 +212,8 @@ def to_dict(self) -> Dict[str, Any]: def __getattr__(self, name): """ - Allow attribute access to intermediate values. - If an attribute is not found in the object, look for it in the intermediates dict. + Allow attribute access to intermediate values. If an attribute is not found in the object, look for it in the + intermediates dict. """ if name in self.intermediates: return self.intermediates[name] From 796c99270c9ed572034445a4c7450c67e8962882 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 22 Jul 2025 02:19:53 +0200 Subject: [PATCH 7/7] up more --- src/diffusers/modular_pipelines/modular_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index ce0068e0369c..8039e4ecaf48 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -2520,9 +2520,9 @@ def update_components(self, **kwargs): ): logger.warning( f"ModularPipeline.update_components: {name} has no valid _diffusers_load_id. " - f"Updating the component but skipping spec update, use ComponentSpec.load() for proper specs" + f"This will result in empty loading spec, use ComponentSpec.load() for proper specs" ) - new_component_spec = current_component_spec + new_component_spec = ComponentSpec(name=name, type_hint=type(component)) else: new_component_spec = ComponentSpec.from_component(name, component)