Merge pull request #97 from ArcanaFramework/pydra-syntax-changes

Cleaned up converter

Author: tclose

fileformats/core/converter_helpers.py

@@ -28,7 +28,7 @@ def unzip(in_file: Zip[AnyFileFormat], out_file: AnyFileFormat):
         ...
     """
 
-    converters: ty.Dict[ty.Type["fileformats.core.FileSet"], "Converter[ty.Any]"] = {}
+    converters: ty.Dict[ty.Type["fileformats.core.FileSet"], "Converter"] = {}
 
     @classmethod
     def new(cls, name: str, klass: type) -> "SubtypeVar":
@@ -57,10 +57,10 @@ def __subclasshook__(cls, subclass: type) -> bool:
     @classmethod
     def get_converter_defs(
         cls, source_format: ty.Type["mixin.WithClassifiers"], target_format: type
-    ) -> ty.List["Converter[T]"]:
+    ) -> ty.List["Converter"]:
         # check to see whether there are converters from a base class of the source
         # format
-        available_converters: ty.List[Converter[T]] = []
+        available_converters: ty.List[Converter] = []
         # assert isinstance(source_format, WithClassifiers)
         if source_format.is_classified:
             for template_source_format, converter in cls.converters.items():
@@ -83,7 +83,7 @@ def get_converter_defs(
     def register_converter(
         cls,
         source_format: ty.Type["mixin.WithClassifiers"],
-        converter: "Converter[T]",
+        converter: "Converter",
     ) -> None:
         """Registers a converter task within a class attribute. Called by the
         @fileformats.core.converter decorator.
@@ -138,11 +138,11 @@ def register_converter(
         cls.converters[source_format] = converter  # type: ignore
 
 
-class Converter(ty.Generic[T]):
+class Converter:
     """Specification of a converter task, including the task callable, its arguments and
     the classifiers"""
 
-    task_def: "TaskDef[T]"
+    task_def: "TaskDef[ty.Any]"
     classifiers: ty.Tuple[ty.Type[Classifier], ...]
     in_file: str
     out_file: str

fileformats/core/datatype.py

@@ -23,6 +23,9 @@
 )
 from .classifier import Classifier
 
+if ty.TYPE_CHECKING:
+    from .converter_helpers import Converter
+
 
 class DataType(Classifier, metaclass=ABCMeta):
     """
@@ -39,7 +42,7 @@ def nested_types(cls) -> ty.Tuple[ty.Type[Classifier], ...]:
     # Store converters registered by @converter decorator that convert to FileSet
     # NB: each class will have its own version of this dictionary
     converters: ty.Dict[
-        ty.Type["DataType"], "fileformats.core.converter_helpers.Converter[ty.Any]"  # type: ignore[type-arg]
+        ty.Type["DataType"], "fileformats.core.converter_helpers.Converter"  # type: ignore[type-arg]
     ] = {}
 
     @classmethod
@@ -92,8 +95,7 @@ def subclasses(cls) -> ty.Generator[ty.Type[Self], None, None]:
     def get_converter(
         cls,
         source_format: ty.Type[DataType],
-        **kwargs: ty.Any,
-    ) -> ty.Union[None]:
+    ) -> "Converter | None":
         if issubclass(source_format, cls):
             return None
         else:

fileformats/core/extras.py

@@ -229,6 +229,10 @@ def converter(
             "installed, this should be installed with Pydra by default",
         )
         raise e
+    if "out_file_" in converter_kwargs:
+        converter_kwargs["out_file"] = converter_kwargs.pop("out_file_")
+    if "in_file_" in converter_kwargs:
+        converter_kwargs["in_file"] = converter_kwargs.pop("in_file_")
 
     def decorator(
         task_type: ty.Type["TaskDef[T]"],
@@ -268,6 +272,7 @@ def decorator(
                 f"Target file format '{target.__name__}' is not of subtype of "
                 "fileformats.core.FileSet"
             )
+
         target.register_converter(
             source_format=source,
             converter=Converter(

fileformats/core/fileset.py

@@ -43,7 +43,6 @@
 from .mock import MockMixin
 
 if ty.TYPE_CHECKING:
-    from pydra.engine.specs import TaskDef, OutputsType
     from .converter_helpers import Converter
 
 
@@ -501,7 +500,6 @@ def matching_exts(
     def convert(
         cls,
         fileset: "FileSet",
-        worker: ty.Optional[str] = None,
         **kwargs: ty.Any,
     ) -> Self:
         """Convert a given file-set into the format specified by the class
@@ -510,12 +508,8 @@ def convert(
         ----------
         fileset : FileSet
             the file-set object to convert
-        plugin : str
-            the "execution plugin" used to run the conversion task
-        task_name : str
-            the name given to the converter task
         **kwargs
-            args to pass to the conversion process
+            args to pass to customise the converter task definition
 
         Returns
         -------
@@ -525,12 +519,14 @@ def convert(
         import attrs
 
         # Make unique, yet somewhat recognisable task name
-        task = cls.get_converter(source_format=type(fileset), **kwargs)
-        if task is None:
-            return copy(fileset)  # type: ignore[return-value]
-        task = attrs.evolve(task, in_file=fileset)
-        outputs = task(worker=worker)
-        out_file = outputs.out_file
+        converter = cls.get_converter(source_format=type(fileset))
+        if converter is None:
+            assert isinstance(fileset, cls)
+            return copy(fileset)
+        kwargs[converter.in_file] = fileset
+        task_def = attrs.evolve(converter.task_def, **kwargs)
+        outputs = task_def()
+        out_file = getattr(outputs, converter.out_file)
         if not isinstance(out_file, cls):
             out_file = cls(out_file)
         return out_file  # type: ignore
@@ -539,8 +535,7 @@ def convert(
     def get_converter(
         cls,
         source_format: ty.Type[DataType],
-        **kwargs: ty.Any,
-    ) -> "TaskDef[OutputsType] | None":
+    ) -> "Converter | None":
         """Get a converter that converts from the source format type
         into the format specified by the class
 
@@ -577,7 +572,7 @@ def get_converter(
         else:
             import_extras_module(unclassified)
         try:
-            converter_def = converters[source_format]
+            converter = converters[source_format]
         except KeyError:
             # If no direct mapping check for mapping from source super types and wildcard
             # matches
@@ -608,36 +603,30 @@ def get_converter(
                         f"from PyPI (e.g. pip install {extras_mod.pypi}) or check it isn't broken"
                     )
                 raise FormatConversionError(msg) from None
-            converter_def = available_converters[0]
+            converter = available_converters[0]
             # Store mapping for future reference
-            converters[source_format] = converter_def
-        if kwargs:
-            task_def = copy(converter_def.task_def)
-            for key, val in kwargs.items():
-                setattr(task_def, key, val)
-        else:
-            task_def = converter_def.task_def
-        return task_def
+            converters[source_format] = converter
+        return converter
 
     @classmethod
     def get_converters_dict(
         cls, klass: ty.Optional[ty.Type[DataType]] = None
-    ) -> ty.Dict[ty.Type[DataType], "Converter[ty.Any]"]:
+    ) -> ty.Dict[ty.Type[DataType], "Converter"]:
         # Only access converters to the specific class, not superclasses (which may not
         # be able to convert to the specific type)
         if klass is None:
             klass = cls
         # import related extras module for the target class
         import_extras_module(klass)
-        converters_dict: ty.Dict[ty.Type[DataType], "Converter[ty.Any]"]
+        converters_dict: ty.Dict[ty.Type[DataType], "Converter"]
         try:
             converters_dict = klass.__dict__["converters"]
         except KeyError:
             converters_dict = klass.converters = {}
         return converters_dict
 
     @classmethod
-    def get_converter_defs(cls, source_format: type) -> ty.List["Converter[ty.Any]"]:
+    def get_converter_defs(cls, source_format: type) -> ty.List["Converter"]:
         """Search the registered converters to find any matches and return list of
         task and associated key-word args to perform the conversion between source and
         target formats
@@ -669,7 +658,7 @@ def get_converter_defs(cls, source_format: type) -> ty.List["Converter[ty.Any]"]
     def register_converter(
         cls,
         source_format: ty.Type["FileSet"],
-        converter: "Converter[T]",
+        converter: "Converter",
     ) -> None:
         """Registers a converter task within a class attribute. Called by the
         @fileformats.core.converter decorator.

fileformats/core/mixin.py

@@ -458,7 +458,7 @@ def __class_getitem__(
         return classified
 
     @classmethod
-    def get_converter_defs(cls, source_format: type) -> ty.List[Converter[ty.Any]]:
+    def get_converter_defs(cls, source_format: type) -> ty.List[Converter]:
         """Search the registered converters to find an appropriate task and associated
         key-word args to perform the conversion between source and target formats
 
@@ -470,7 +470,7 @@ def get_converter_defs(cls, source_format: type) -> ty.List[Converter[ty.Any]]:
         from fileformats.core import FileSet
 
         # Try to see if a converter has been defined to the exact type
-        available_converters: ty.List[Converter[ty.Any]] = super().get_converter_defs(  # type: ignore[misc, type-arg]
+        available_converters: ty.List[Converter] = super().get_converter_defs(  # type: ignore[misc, type-arg]
             source_format
         )
         # Failing that, see if there is a generic conversion between the container type
@@ -603,7 +603,7 @@ def __subclasshook__(cls, subclass: type) -> bool:
     def register_converter(
         cls,
         source_format: ty.Type["fileformats.core.FileSet"],
-        converter: Converter[T],
+        converter: Converter,
     ) -> None:
         """Registers a converter task within a class attribute. Called by the @fileformats.converter
         decorator.