Inference on 4090: CUDA error: no kernel image is available for execution on the device

I've running the script projects/inference_seedvr2_3b.py pretty fine on H200, but when I move to 4090 the program gave me this error. I've install torch 2.3+cu12.1 and flash_attn 2.5.9post1 as the requirements ordered. Is there something wrong with my deployment? The following is the complete traceback:

[rank0]: Traceback (most recent call last): | 0/1 [00:00
[rank0]: generation_loop(runner, **vars(args))
[rank0]: File "/data/zpool1/seedvr/projects/inference_seedvr2_3b.py", line 286, in generation_loop
[rank0]: samples = generation_step(runner, text_embeds, cond_latents=cond_latents)
[rank0]: File "/data/zpool1/seedvr/projects/inference_seedvr2_3b.py", line 123, in generation_step
[rank0]: video_tensors = runner.inference(
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
[rank0]: return func(*args, **kwargs)
[rank0]: File "/data/zpool1/seedvr/projects/video_diffusion_sr/infer.py", line 306, in inference
[rank0]: latents = self.sampler.sample(
[rank0]: File "/data/zpool1/seedvr/common/diffusion/samplers/euler.py", line 54, in sample
[rank0]: pred = f(SamplerModelArgs(x, t, i))
[rank0]: File "/data/zpool1/seedvr/projects/video_diffusion_sr/infer.py", line 308, in
[rank0]: f=lambda args: classifier_free_guidance_dispatcher(
[rank0]: File "/data/zpool1/seedvr/common/diffusion/utils.py", line 76, in classifier_free_guidance_dispatcher
[rank0]: return pos()
[rank0]: File "/data/zpool1/seedvr/projects/video_diffusion_sr/infer.py", line 309, in
[rank0]: pos=lambda: self.dit(
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/data/zpool1/seedvr/models/dit_v2/nadit.py", line 220, in forward
[rank0]: vid, txt, vid_shape, txt_shape = gradient_checkpointing(
[rank0]: File "/data/zpool1/seedvr/models/dit_v2/nadit.py", line 32, in gradient_checkpointing
[rank0]: return module(*args, **kwargs)
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/data/zpool1/seedvr/models/dit_v2/nablocks/mmsr_block.py", line 108, in forward
[rank0]: vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs)
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/data/zpool1/seedvr/models/dit_v2/mm.py", line 70, in forward
[rank0]: vid = vid_module(vid, *get_args("vid", args), **get_kwargs("vid", kwargs))
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: File "/data/zpool1/ai/anaconda3/envs/seedvr/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: File "/data/zpool1/seedvr/models/dit_v2/modulation.py", line 80, in forward
[rank0]: emb = cache(
[rank0]: File "/data/zpool1/seedvr/common/cache.py", line 34, in __call__
[rank0]: result = fn()
[rank0]: File "/data/zpool1/seedvr/models/dit_v2/modulation.py", line 83, in
[rank0]: torch.cat([e.repeat(l, *([1] * e.ndim)) for e, l in zip(emb, hid_len)]),
[rank0]: File "/data/zpool1/seedvr/models/dit_v2/modulation.py", line 83, in
[rank0]: torch.cat([e.repeat(l, *([1] * e.ndim)) for e, l in zip(emb, hid_len)]),
[rank0]: RuntimeError: CUDA error: no kernel image is available for execution on the device
[rank0]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.