Linear' object has no attribute 'base_layer

### Describe the bug

I was trying to inference flux lora , lora was finetuned on ai toolkit

I have installed diffusers from source

while inferencing with lora , I am getting this error ===> Linear' object has no attribute 'base_layer

Please let me know if you need any info

Thanks

### Reproduction

import io
import os
from pathlib import Path
from typing import Optional
from dataclasses import dataclass
# from dotenv import load_dotenv
from io import BytesIO

import modal
import torch
from diffusers import DiffusionPipeline
from fastapi import FastAPI, Response, HTTPException, Request
from pydantic import BaseModel, Field

# List of environment variables to include

# Configure logging
# logging.basicConfig(level=logging.INFO)
# logger = logging.getLogger(__name__)
lora_path = Path("./LoraTextToImage/fuelGaso/my_fuel_flux_lora_v4.safetensors")

# Configuration
VOLUME_NAME = "flux-model-weights"
WEIGHTS_DIR = "/weights"
LORA_DIR = "/lora" # New directory for LoRA files

# Create volume
model_volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True)
lora_mount = modal.Mount.from_local_file(
local_path=str(lora_path),
remote_path=f"{LORA_DIR}/{lora_path.name}"
)

@dataclass
class ModelConfig:
"""Configuration for model parameters and paths"""
model_name: str = "black-forest-labs/FLUX.1-dev"
weights_dir: str = WEIGHTS_DIR
use_safetensors: bool = True
torch_dtype: torch.dtype = torch.bfloat16

# Modal image configuration
diffusion_image = (
modal.Image.debian_slim(python_version="3.11")
.apt_install(
"libglib2.0-0",
"libsm6",
"libxrender1",
"libxext6",
"ffmpeg",
"libgl1",
"git"
)
.pip_install(
"git+https://github.com/huggingface/diffusers.git",
"transformers[torch]",
"accelerate",
"safetensors",
"torch",
"fastapi[standard]",
"pydantic",
"huggingface-hub",
"sentencepiece",
"peft"

)

# .run_function(get_diffusion_pipelines,secrets=[modal.Secret.from_name("HF_TOKEN")])

)

app = modal.App("flux-pipeline")

class GenerationRequest(BaseModel):
prompt: str = Field(..., min_length=1, max_length=1000)
seed: int = Field(..., ge=0)
num_inference_steps: int = Field(default=20, ge=1, le=100)

@app.cls(
gpu=modal.gpu.A10G(),
container_idle_timeout=240,
image=diffusion_image,
volumes={WEIGHTS_DIR: model_volume},
mounts=[lora_mount] , # Add the LoRA mount
secrets=[modal.Secret.from_name("HF_TOKEN")]
)
class Model:
def __init__(self):
self.pipe = None
self.device = None
self.config = ModelConfig()
self.lora_path = Path(f"{LORA_DIR}/{lora_path.name}") # Update LoRA path

def setup_model(self):
"""Setup model with proper authentication"""
try:
HF_TOKEN = os.environ["HF_TOKEN"]
# logger.info("Setting up model...")
from huggingface_hub import login

# Ensure we're logged in
login(token=HF_TOKEN)

# Initialize model
self.pipe = DiffusionPipeline.from_pretrained(
self.config.model_name,
torch_dtype=torch.bfloat16,
use_safetensors=True,
).to(self.device)

if self.pipe is None:
raise ValueError("Failed to initialize DiffusionPipeline.")

self.pipe.to(self.device)

if not self.lora_path.exists():
raise FileNotFoundError(f"Lora weights file not found at {self.lora_path}")

self.pipe.load_lora_weights("sandeep65432/gaso", weight_name="degen_gaso_v5.safetensors")
self.pipe.fuse_lora(lora_scale=1.7)
self.pipe.unload_lora_weights()

return self.pipe

except Exception as e:
raise

@modal.build()
def build(self):
"""Build phase to download model"""
try:
# logger.info("Starting build phase...")
weights_path = Path(self.config.weights_dir)
weights_path.mkdir(exist_ok=True, parents=True)

# Download model
pipe = self.setup_model()

# Save to volume
# logger.info(f"Saving model to volume at {weights_path}")
pipe.save_pretrained(str(weights_path))
model_volume.commit()


except Exception as e:
# logger.error(f"Build failed: {e}")
raise

@modal.enter()
def enter(self):
"""Initialize model"""
try:
# logger.info("Initializing model...")
self.device = "cuda" if torch.cuda.is_available() else "cpu"

# Load from volume
weights_path = Path(self.config.weights_dir)
self.pipe = DiffusionPipeline.from_pretrained(
str(weights_path),
torch_dtype=torch.bfloat16,
use_safetensors=True,
)
if self.pipe is None:
raise ValueError("Failed to initialize DiffusionPipeline.")

self.pipe.to(self.device)
if not self.lora_path.exists():
raise FileNotFoundError(f"Lora weights file not found at {self.lora_path}")

self.pipe.load_lora_weights("sandeep65432/gaso", weight_name="degen_gaso_v5.safetensors")
self.pipe.fuse_lora(lora_scale=1.7)
self.pipe.unload_lora_weights()

except Exception as e:
raise

def _inference(self, prompt: str, seed: int = 42, num_steps: int = 20) -> BytesIO:
"""Run inference"""
try:
generator = torch.Generator(device=self.device).manual_seed(seed)

image = self.pipe(
prompt=prompt,
num_inference_steps=num_steps,
generator=generator,
output_type="pil"
).images[0]

buffer = BytesIO()
image.save(buffer, format="JPEG", quality=95)
buffer.seek(0)

return buffer

except Exception as e:
raise

@modal.method()
def inference(self, prompt: str, seed: int = 42, num_steps: int = 20) -> bytes:
"""Remote inference method"""
return self._inference(prompt, seed, num_steps).getvalue()

@modal.web_endpoint(method="POST")
async def generate(self, request: Request):
"""Web endpoint for inference"""
try:
data = await request.json()
gen_request = GenerationRequest(**data)

image_bytes = self._inference(
gen_request.prompt,
gen_request.seed,
gen_request.num_inference_steps
).getvalue()

return Response(
content=image_bytes,
media_type="image/jpeg",
headers={
"Cache-Control": "no-cache",
"Content-Disposition": "attachment; filename=generated.jpg"
}
)

except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
print(e,"FFFFFFFFFF")
raise HTTPException(status_code=500, detail="Internal server error")

@app.local_entrypoint()
def main(prompt: str = "A majestic mountain landscape at sunset"):
"""Local entrypoint for testing"""
try:
image_bytes = Model().inference.remote(prompt)

output_path = Path("/tmp/generated.jpg")
with open(output_path, "wb") as f:
f.write(image_bytes)


except Exception as e:
raise

### Logs

```shell
Traceback (most recent call last):
File "/pkg/modal/_runtime/container_io_manager.py", line 727, in handle_user_exception
yield
File "/pkg/modal/_container_entrypoint.py", line 377, in call_lifecycle_functions
res = func(*args)
^^^^^^^^^^^
File "/root/fuelGaso.py", line 195, in enter
self.pipe = DiffusionPipeline.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_utils.py", line 896, in from_pretrained
loaded_sub_model = load_sub_model(
^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_loading_utils.py", line 725, in load_sub_model
loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 929, in from_pretrained
raise e
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 886, in from_pretrained
accelerate.load_checkpoint_and_dispatch(
File "/usr/local/lib/python3.11/site-packages/accelerate/big_modeling.py", line 613, in load_checkpoint_and_dispatch
load_checkpoint_in_model(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 1780, in load_checkpoint_in_model
set_module_tensor_to_device(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 247, in set_module_tensor_to_device
new_module = getattr(module, split)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1931, in __getattr__
raise AttributeError(
AttributeError: 'Linear' object has no attribute 'base_layer'
Loading pipeline components...: 29%|██▊ | 2/7 [00:07<00:18, 3.69s/it]
Traceback (most recent call last):
File "/pkg/modal/_runtime/container_io_manager.py", line 727, in handle_user_exception
yield
File "/pkg/modal/_container_entrypoint.py", line 377, in call_lifecycle_functions
res = func(*args)
^^^^^^^^^^^
File "/root/fuelGaso.py", line 195, in enter
self.pipe = DiffusionPipeline.from_pretrained(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_utils.py", line 896, in from_pretrained
loaded_sub_model = load_sub_model(
^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/pipelines/pipeline_loading_utils.py", line 725, in load_sub_model
loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 929, in from_pretrained
raise e
File "/usr/local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 886, in from_pretrained
accelerate.load_checkpoint_and_dispatch(
File "/usr/local/lib/python3.11/site-packages/accelerate/big_modeling.py", line 613, in load_checkpoint_and_dispatch
load_checkpoint_in_model(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 1780, in load_checkpoint_in_model
set_module_tensor_to_device(
File "/usr/local/lib/python3.11/site-packages/accelerate/utils/modeling.py", line 247, in set_module_tensor_to_device
new_module = getattr(module, split)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1931, in __getattr__
raise AttributeError(
AttributeError: 'Linear' object has no attribute 'base_layer'
Runner failed with exception: AttributeError("'Linear' object has no attribute 'base_layer'")
```

### System Info

MarkupSafe-3.0.2 Pillow-11.0.0 accelerate-1.1.1 annotated-types-0.7.0 anyio-4.6.2.post1 charset-normalizer-3.4.0 click-8.1.7 diffusers-0.32.0.dev0 dnspython-2.7.0 email-validator-2.2.0 fastapi-0.115.5 fastapi-cli-0.0.5 filelock-3.16.1 fsspec-2024.10.0 h11-0.14.0 httpcore-1.0.7 httptools-0.6.4 httpx-0.27.2 huggingface-hub-0.26.2 importlib_metadata-8.5.0 jinja2-3.1.4 markdown-it-py-3.0.0 mdurl-0.1.2 mpmath-1.3.0 networkx-3.4.2 numpy-2.1.3 nvidia-cublas-cu12-12.4.5.8 nvidia-cuda-cupti-cu12-12.4.127 nvidia-cuda-nvrtc-cu12-12.4.127 nvidia-cuda-runtime-cu12-12.4.127 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.2.1.3 nvidia-curand-cu12-10.3.5.147 nvidia-cusolver-cu12-11.6.1.9 nvidia-cusparse-cu12-12.3.1.170 nvidia-nccl-cu12-2.21.5 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.4.127 packaging-24.2 peft-0.13.2 psutil-6.1.0 pydantic-2.10.2 pydantic-core-2.27.1 pygments-2.18.0 python-dotenv-1.0.1 python-multipart-0.0.17 pyyaml-6.0.2 regex-2024.11.6 requests-2.32.3 rich-13.9.4 safetensors-0.4.5 sentencepiece-0.2.0 shellingham-1.5.4 sniffio-1.3.1 starlette-0.41.3 sympy-1.13.1 tokenizers-0.20.4 torch-2.5.1 tqdm-4.67.1 transformers-4.46.3 triton-3.1.0 typer-0.13.1 urllib3-2.2.3 uvicorn-0.32.1 uvloop-0.21.0 watchfiles-1.0.0 websockets-14.1 zipp-3.21.0

python version 3.11

### Who can help?

@sayakpaul