[RFC] Multi-backend recipes for easy target focused model deployment

🚀 The feature, motivation and pitch

Goal

Reduce cognitive friction to learn about Executorch (and its backends) to lower and execute a model on a specific target. With multi-backend recipes, given a target, we provide an optimized set of configurations that work out of the box, enabling users to easily lower and execute their models with just a few lines of code.

Before

Without executorch.export() and ExportRecipe, one typically has to follow these steps for successful model export and execution on a target device:

Let's discuss with an example model that one want to execute with CoreML backend and fallback to XNNPACK for CPU execution:

# BEFORE: Manual lowering and API orchestration

import torch
from executorch.exir.program import to_edge
from executorch.exir.backend.backend_api import validation_disabled
from executorch.exir import EdgeCompileConfig
from torchao.quantization.pt2e.quantize_pt2e import prepare_pt2e, convert_pt2e
from torchao.quantization.pt2e.quantizer import ComposableQuantizer

# CoreML imports
import coremltools as ct
from executorch.backends.apple.coreml.compiler import CoreMLBackend
from executorch.backends.apple.coreml.partition.coreml_partitioner import CoreMLPartitioner
from executorch.backends.apple.coreml.quantizer import CoreMLQuantizer

# XNNPACK imports
from executorch.backends.xnnpack.partition.xnnpack_partitioner import XnnpackPartitioner
from executorch.backends.xnnpack.partition.config.xnnpack_config import ConfigPrecisionType
from executorch.backends.xnnpack.utils.configs import get_xnnpack_executorch_backend_config
from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import (
    XNNPACKQuantizer, get_symmetric_quantization_config
)

# Create CoreML quantizer (optional)
config = ct.optimize.torch.quantization.LinearQuantizerConfig(
    global_config=ct.optimize.torch.quantization.ModuleLinearQuantizerConfig(
        quantization_scheme="symmetric",
        activation_dtype=torch.quint8,
        weight_dtype=torch.qint8,
        weight_per_channel=True,
    )
)
coreml_quantizer = CoreMLQuantizer(config)

# create coreml partitioner
compile_specs = CoreMLBackend.generate_compile_specs(
    compute_precision=ct.precision.FLOAT16,
    compute_unit=ct.ComputeUnit.CPU_AND_NE,
    minimum_deployment_target=ct.target.iOS16,
)

coreml_partitioner = CoreMLPartitioner(
    compile_specs=compile_specs,
    take_over_mutable_buffer=True,
    skip_ops_for_coreml_delegation=None,
    lower_full_graph=False,
    take_over_constant_data=True,
)

xnnpack_partitioner = XnnpackPartitioner(
    precision_type=ConfigPrecisionType.FP32
)

# Create XNNPACK quantizer (optional)
xnnpack_quantizer = XNNPACKQuantizer()
operator_config = get_symmetric_quantization_config(
    is_per_channel=True,
    is_dynamic=False,
    is_qat=False,
)
xnnpack_quantizer.set_global(operator_config)

partitioners = [coreml_partitioner, xnnpack_partitioner]
quantizers = [coreml_quantizer, xnnpack_quantizer]

##  Sequence of APIs

# source quantization if needed
...

# PT2E quantization step
if quantizers:
     captured_graph = torch.export.export(model, example_inputs)
     quantizer = ComposableQuantizer(quantizers) if len(quantizers) > 1 else quantizers[0]
     prepared_model = prepare_pt2e(captured_graph, quantizer)
        
     for calibration_input in calibration_inputs:
        prepared_model(*calibration_input)
        
     quantized_model = convert_pt2e(prepared_model)

# export step
exported_program = torch.export.export(
  quantized_model, example_inputs[method_name][0], strict=True
)

# to_edge and lower conversion
edge_program_manager = to_edge_transform_and_lower(
     exported_programs,
     partitioner=self._partitioners,
     transform_passes=self._transform_passes,
     constant_methods=constant_methods,
     compile_config=self._compile_config,
     generate_etrecord=generate_etrecord,
 )

# to_executorch conversion
backend_config = exir.ExecutorchBackendConfig(extract_delegate_segments=True)
executorch_program_manager = edge_program_manager.to_executorch(backend_config)

# serialization
with open("model.pte", "wb") as f:
    executorch_program_manager.write_to_file(f)

After

executorch.export(), recipes and Multi-Backend Target Recipes working together

from executorch.export import export
from executorch.export.target_recipes import get_ios_recipe

# CoreML + XNNPACK (FP32)
recipe = get_ios_recipe() # default = "ios-arm64-coreml-fp16"
session = export(model, recipe, example_inputs)
session.save_pte_file("model.pte")

Alternatives

No response

Additional context

No response

RFC (Optional)

Core Components

1. Target Recipe Types (target_recipe_types.py)

IOS_CONFIGS: Dict[str, List[RecipeType]] = {
    # pyre-ignore
    "ios-arm64-coreml-fp32": [CoreMLRecipeType.FP32, XNNPackRecipeType.FP32],
    # pyre-ignore
    "ios-arm64-coreml-fp16": [CoreMLRecipeType.FP16],
    # pyre-ignore
    "ios-arm64-coreml-int8": [CoreMLRecipeType.PT2E_INT8_STATIC],
}

Example IOS recipe

# IOS Recipe
def get_ios_recipe(
    target_config: str = "ios-arm64-coreml-fp16", **kwargs
) -> ExportRecipe:
    """
    Get iOS-optimized recipe for specified hardware configuration.

    Supported configurations:
    - 'ios-arm64-coreml-fp32': CoreML + XNNPACK fallback (FP32)
    - 'ios-arm64-coreml-fp16': CoreML fp16 recipe
    - 'ios-arm64-coreml-int8': CoreML INT8 quantization recipe

    Args:
        target_config: iOS configuration string
        **kwargs: Additional parameters for backend recipes

    Returns:
        ExportRecipe configured for iOS deployment

    Raises:
        ValueError: If target configuration is not supported

    Example:
        recipe = get_ios_recipe('ios-arm64-coreml-int8')
        session = export(model, recipe, example_inputs)
    """
    if target_config not in IOS_CONFIGS:
        supported = list(IOS_CONFIGS.keys())
        raise ValueError(
            f"Unsupported iOS configuration: '{target_config}'. "
            f"Supported: {supported}"
        )

    kwargs = kwargs or {}

    if target_config == "ios-arm64-coreml-int8":
        if "minimum_deployment_target" not in kwargs:
            kwargs["minimum_deployment_target"] = ct.target.iOS17

    backend_recipes = IOS_CONFIGS[target_config]
    return _create_target_recipe(target_config, backend_recipes, **kwargs)

Multi-Backend Recipe Combination

The system combines individual backend recipes by merging their components:

@dataclass
class ExportRecipe:
    @classmethod
    def combine(
        cls, recipes: List["ExportRecipe"], recipe_name: Optional[str] = None
    ) -> "ExportRecipe":
        if not recipes:
            raise ValueError("Cannot combine empty list of recipes")

        if len(recipes) == 1:
            return recipes[0]

        return cls._combine_recipes(recipes, recipe_name)

    @classmethod
    def _combine_recipes(
        cls, backend_recipes: List["ExportRecipe"], recipe_name: Optional[str] = None
    ) -> "ExportRecipe": # noqa: C901
        # Extract components from individual recipes
        all_partitioners = []
        all_quantizers = []
        all_ao_quantization_configs = []
        all_pre_edge_passes = []
        all_transform_passes = []
        combined_backend_config = None

        for recipe in backend_recipes:
            # Collect pre-edge transform passes
            if recipe.pre_edge_transform_passes:
                all_pre_edge_passes.extend(recipe.pre_edge_transform_passes)

            # Collect partitioners from lowering recipes
            if recipe.lowering_recipe and recipe.lowering_recipe.partitioners:
                all_partitioners.extend(recipe.lowering_recipe.partitioners)

            # Collect transform passes from lowering recipes
            if recipe.lowering_recipe and recipe.lowering_recipe.edge_transform_passes:
                all_transform_passes.extend(
                    recipe.lowering_recipe.edge_transform_passes
                )

            # Collect for quantize stage
            if recipe.quantization_recipe:
                # Collect PT2E quantizers
                if recipe.quantization_recipe.quantizers:
                    all_quantizers.extend(recipe.quantization_recipe.quantizers)

                # Collect source transform configs
                ao_configs = getattr(
                    recipe.quantization_recipe, "ao_quantization_configs", None
                )
                if ao_configs:
                    all_ao_quantization_configs.extend(ao_configs)

            # Use the first backend config as base (can be enhanced later)
            if combined_backend_config is None and recipe.executorch_backend_config:
                combined_backend_config = copy.deepcopy(
                    recipe.executorch_backend_config
                )

        # Create combined quantization recipe
        combined_quantization_recipe = None
        if all_quantizers or all_ao_quantization_configs:
            combined_quantization_recipe = QuantizationRecipe(
                quantizers=all_quantizers if all_quantizers else None,
                ao_quantization_configs=(
                    all_ao_quantization_configs if all_ao_quantization_configs else None
                ),
            )

        # Create combined lowering recipe
        combined_lowering_recipe = None
        if all_partitioners or all_transform_passes:
            # Use the edge compile config from the first recipe that has one
            edge_compile_config = None
            for recipe in backend_recipes:
                if (
                    recipe.lowering_recipe
                    and recipe.lowering_recipe.edge_compile_config
                ):
                    edge_compile_config = recipe.lowering_recipe.edge_compile_config
                    break

            combined_lowering_recipe = LoweringRecipe(
                partitioners=all_partitioners if all_partitioners else None,
                edge_transform_passes=(
                    all_transform_passes if all_transform_passes else None
                ),
                edge_compile_config=edge_compile_config,
            )

        # Create the combined export recipe
        return cls(
            name=recipe_name,
            quantization_recipe=combined_quantization_recipe,
            pre_edge_transform_passes=all_pre_edge_passes,
            lowering_recipe=combined_lowering_recipe,
            executorch_backend_config=combined_backend_config,
        )

Open questions

1. Number of multi-backend combinations to be published per target?
2. How to combine precisions across backends, i.e., what combination of recipes would work the best per target?

CC: @mergennachin, @cbilgin, @GregoryComer, @JacobSzwejbka