Linear' object has no attribute 'base_layer

### Describe the bug

I was trying to inference flux lora , lora was finetuned on ai toolkit

I have installed diffusers from source

while inferencing with lora , I am getting this error ===> Linear' object has no attribute 'base_layer

Please let me know if you need any info

Thanks

### Reproduction

import io
import os
from pathlib import Path
from typing import Optional
from dataclasses import dataclass
# from dotenv import load_dotenv
from io import BytesIO

import modal
import torch
from diffusers import DiffusionPipeline
from fastapi import FastAPI, Response, HTTPException, Request
from pydantic import BaseModel, Field

# List of environment variables to include

# Configure logging
# logging.basicConfig(level=logging.INFO)
# logger = logging.getLogger(__name__)
lora_path = Path("./LoraTextToImage/fuelGaso/my_fuel_flux_lora_v4.safetensors")

# Configuration
VOLUME_NAME = "flux-model-weights"
WEIGHTS_DIR = "/weights"
LORA_DIR = "/lora" # New directory for LoRA files

# Create volume
model_volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True)
lora_mount = modal.Mount.from_local_file(
local_path=str(lora_path),
remote_path=f"{LORA_DIR}/{lora_path.name}"
)

@dataclass
class ModelConfig:
"""Configuration for model parameters and paths"""
model_name: str = "black-forest-labs/FLUX.1-dev"
weights_dir: str = WEIGHTS_DIR
use_safetensors: bool = True
torch_dtype: torch.dtype = torch.bfloat16

# Modal image configuration
diffusion_image = (
modal.Image.debian_slim(python_version="3.11")
.apt_install(
"libglib2.0-0",
"libsm6",
"libxrender1",
"libxext6",
"ffmpeg",
"libgl1",
"git"
)
.pip_install(
"git+https://github.com/huggingface/diffusers.git",
"transformers[torch]",
"accelerate",
"safetensors",
"torch",
"fastapi[standard]",
"pydantic",
"huggingface-hub",
"sentencepiece",
"peft"

)

# .run_function(get_diffusion_pipelines,secrets=[modal.Secret.from_name("HF_TOKEN")])

)

app = modal.App("flux-pipeline")

class GenerationRequest(BaseModel):
prompt: str = Field(..., min_length=1, max_length=1000)
seed: int = Field(..., ge=0)
num_inference_steps: int = Field(default=20, ge=1, le=100)

@app.cls(
gpu=modal.gpu.A10G(),
container_idle_timeout=240,
image=diffusion_image,
volumes={WEIGHTS_DIR: model_volume},
mounts=[lora_mount] , # Add the LoRA mount
secrets=[modal.Secret.from_name("HF_TOKEN")]
)
class Model:
def __init__(self):
self.pipe = None
self.device = None
self.config = ModelConfig()
self.lora_path = Path(f"{LORA_DIR}/{lora_path.name}") # Update LoRA path

def setup_model(self):
"""Setup model with proper authentication"""
try:
HF_TOKEN = os.environ["HF_TOKEN"]
# logger.info("Setting up model...")
from huggingface_hub import login

# Ensure we're logged in
login(token=HF_TOKEN)

# Initialize model
self.pipe = DiffusionPipeline.from_pretrained(
self.config.model_name,
torch_dtype=torch.bfloat16,
use_safetensors=True,
).to(self.device)

if self.pipe is None:
raise ValueError("Failed to initialize DiffusionPipeline.")

self.pipe.to(self.device)

if not self.lora_path.exists():
raise FileNotFoundError(f"Lora weights file not found at {self.lora_path}")

self.pipe.load_lora_weights("sandeep65432/gaso", weight_name="degen_gaso_v5.safetensors")
self.pipe.fuse_lora(lora_scale=1.7)
self.pipe.unload_lora_weights()

return self.pipe

except Exception as e:
raise

@modal.build()
def build(self):
"""Build phase to download model"""
try:
# logger.info("Starting build phase...")
weights_path = Path(self.config.weights_dir)
weights_path.mkdir(exist_ok=True, parents=True)

# Download model
pipe = self.setup_model()

# Save to volume
# logger.info(f"Saving model to volume at {weights_path}")
pipe.save_pretrained(str(weights_path))
model_volume.commit()


except Exception as e:
# logger.error(f"Build failed: {e}")
raise

@modal.enter()
def enter(self):
"""Initialize model"""
try:
# logger.info("Initializing model...")
self.device = "cuda" if torch.cuda.is_available() else "cpu"

# Load from volume
weights_path = Path(self.config.weights_dir)
self.pipe = DiffusionPipeline.from_pretrained(
str(weights_path),
torch_dtype=torch.bfloat16,
use_safetensors=True,
)
if self.pipe is None:
raise ValueError("Failed to initialize DiffusionPipeline.")

self.pipe.to(self.device)
if not self.lora_path.exists():
raise FileNotFoundError(f"Lora weights file not found at {self.lora_path}")

self.pipe.load_lora_weights("sandeep65432/gaso", weight_name="degen_gaso_v5.safetensors")
self.pipe.fuse_lora(lora_scale=1.7)
self.pipe.unload_lora_weights()

except Exception as e:
raise

def _inference(self, prompt: str, seed: int = 42, num_steps: int = 20) -> BytesIO:
"""Run inference"""
try:
generator = torch.Generator(device=self.device).manual_seed(seed)

image = self.pipe(
prompt=prompt,
num_inference_steps=num_steps,
generator=generator,
output_type="pil"
).images[0]

buffer = BytesIO()
image.save(buffer, format="JPEG", quality=95)
buffer.seek(0)

return buffer

except Exception as e:
raise

@modal.method()
def inference(self, prompt: str, seed: int = 42, num_steps: int = 20) -> bytes:
"""Remote inference method"""
return self._inference(prompt, seed, num_steps).getvalue()

@modal.web_endpoint(method="POST")
async def generate(self, request: Request):
"""Web endpoint for inference"""
try:
data = await request.json()
gen_request = GenerationRequest(**data)

image_bytes = self._inference(
gen_request.prompt,
gen_request.seed,
gen_request.num_inference_steps
).getvalue()

return Response(
content=image_bytes,
media_type="image/jpeg",
headers={
"Cache-Control": "no-cache",
"Content-Disposition": "attachment; filename=generated.jpg"
}
)

except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
print(e,"FFFFFFFFFF")
raise HTTPException(status_code=500, detail="Internal server error")

@app.local_entrypoint()
def main(prompt: str = "A majestic mountain landscape at sunset"):
"""Local entrypoint for testing"""
try:
image_bytes = Model().inference.remote(prompt)

output_path = Path("/tmp/generated.jpg")
with open(output_path, "wb") as f:
f.write(image_bytes)


except Exception as e:
raise

### Logs

```shell
Traceback (most recent call last):
File "/pkg/modal/_runtime/container_io_manager.py", line 727, in handle_user_exception
yield
File "/pkg/modal/_container_entrypoint.py", line 377, in call_lifecycle_functions
res = func(*args)
^^^^^^^^^^^
File "/root/fuelGaso.py", line 195, in enter
self.pipe = DiffusionPipeline.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_utils.py", line 896, in from_pretrained
loaded_sub_model = load_sub_model(
^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_loading_utils.py", line 725, in load_sub_model
loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 929, in from_pretrained
raise e
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 886, in from_pretrained
accelerate.load_checkpoint_and_dispatch(
File "/usr/local/lib/python3.11/site-packages/accelerate/big_modeling.py", line 613, in load_checkpoint_and_dispatch
load_checkpoint_in_model(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 1780, in load_checkpoint_in_model
set_module_tensor_to_device(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 247, in set_module_tensor_to_device
new_module = getattr(module, split)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1931, in __getattr__
raise AttributeError(
AttributeError: 'Linear' object has no attribute 'base_layer'
Loading pipeline components...: 29%|██▊ | 2/7 [00:07<00:18, 3.69s/it]
Traceback (most recent call last):
File "/pkg/modal/_runtime/container_io_manager.py", line 727, in handle_user_exception
yield
File "/pkg/modal/_container_entrypoint.py", line 377, in call_lifecycle_functions
res = func(*args)
^^^^^^^^^^^
File "/root/fuelGaso.py", line 195, in enter
self.pipe = DiffusionPipeline.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_utils.py", line 896, in from_pretrained
loaded_sub_model = load_sub_model(
^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_loading_utils.py", line 725, in load_sub_model
loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 929, in from_pretrained
raise e
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 886, in from_pretrained
accelerate.load_checkpoint_and_dispatch(
File "/usr/local/lib/python3.11/site-packages/accelerate/big_modeling.py", line 613, in load_checkpoint_and_dispatch
load_checkpoint_in_model(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 1780, in load_checkpoint_in_model
set_module_tensor_to_device(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 247, in set_module_tensor_to_device
new_module = getattr(module, split)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1931, in __getattr__
raise AttributeError(
AttributeError: 'Linear' object has no attribute 'base_layer'
Runner failed with exception: AttributeError("'Linear' object has no attribute 'base_layer'")
```

### System Info

MarkupSafe-3.0.2 Pillow-11.0.0 accelerate-1.1.1 annotated-types-0.7.0 anyio-4.6.2.post1 charset-normalizer-3.4.0 click-8.1.7 diffusers-0.32.0.dev0 dnspython-2.7.0 email-validator-2.2.0 fastapi-0.115.5 fastapi-cli-0.0.5 filelock-3.16.1 fsspec-2024.10.0 h11-0.14.0 httpcore-1.0.7 httptools-0.6.4 httpx-0.27.2 huggingface-hub-0.26.2 importlib_metadata-8.5.0 jinja2-3.1.4 markdown-it-py-3.0.0 mdurl-0.1.2 mpmath-1.3.0 networkx-3.4.2 numpy-2.1.3 nvidia-cublas-cu12-12.4.5.8 nvidia-cuda-cupti-cu12-12.4.127 nvidia-cuda-nvrtc-cu12-12.4.127 nvidia-cuda-runtime-cu12-12.4.127 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.2.1.3 nvidia-curand-cu12-10.3.5.147 nvidia-cusolver-cu12-11.6.1.9 nvidia-cusparse-cu12-12.3.1.170 nvidia-nccl-cu12-2.21.5 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.4.127 packaging-24.2 peft-0.13.2 psutil-6.1.0 pydantic-2.10.2 pydantic-core-2.27.1 pygments-2.18.0 python-dotenv-1.0.1 python-multipart-0.0.17 pyyaml-6.0.2 regex-2024.11.6 requests-2.32.3 rich-13.9.4 safetensors-0.4.5 sentencepiece-0.2.0 shellingham-1.5.4 sniffio-1.3.1 starlette-0.41.3 sympy-1.13.1 tokenizers-0.20.4 torch-2.5.1 tqdm-4.67.1 transformers-4.46.3 triton-3.1.0 typer-0.13.1 urllib3-2.2.3 uvicorn-0.32.1 uvloop-0.21.0 watchfiles-1.0.0 websockets-14.1 zipp-3.21.0

python version 3.11

### Who can help?

@sayakpaul

Reproduce the failure with the provided Modal inference script and inspect the loading path in diffusers/pipelines/pipeline_loading_utils.py and diffusers/models/modeling_utils.py, where the traceback enters Accelerate's checkpoint loader. Compare the model-loading state at the failing Linear attribute access with the LoRA setup shown in the script. Done means the pipeline loads successfully and the reported FLUX LoRA inference path no longer raises this AttributeError.

Vom Indexierungsmodell aus dem Issue-Text verfasst.