Allocation error Every other Prompt

### Expected Behavior

Run as normal

### Actual Behavior

Instead of running the next prompt it gives an error that I am out of memory and then starts the next one successfully, basically it can run, but it first needs to give me the error before running. missing a queue item each time. probably because it only clears after the error ram data left in memory.

### Steps to Reproduce

/

### Debug Logs

```powershell
!!! Exception during processing !!! Allocation on device
Traceback (most recent call last):
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/execution.py", line 328, in execute
output_data, output_ui, has_subgraph = get_output_data(obj, input_data_all, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/execution.py", line 203, in get_output_data
return_values = _map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True, execution_block_cb=execution_block_cb, pre_execute_cb=pre_execute_cb)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/execution.py", line 174, in _map_node_over_list
process_inputs(input_dict, i)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/execution.py", line 163, in process_inputs
results.append(getattr(obj, func)(**inputs))
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/nodes.py", line 67, in encode
return (clip.encode_from_tokens_scheduled(tokens), )
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/sd.py", line 146, in encode_from_tokens_scheduled
pooled_dict = self.encode_from_tokens(tokens, return_pooled=return_pooled, return_dict=True)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/sd.py", line 208, in encode_from_tokens
o = self.cond_stage_model.encode_token_weights(tokens)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/text_encoders/flux.py", line 55, in encode_token_weights
t5_out, t5_pooled = self.t5xxl.encode_token_weights(token_weight_pairs_t5)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/sd1_clip.py", line 45, in encode_token_weights
o = self.encode(to_encode)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/sd1_clip.py", line 252, in encode
return self(tokens)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/sd1_clip.py", line 224, in forward
outputs = self.transformer(tokens, attention_mask_model, intermediate_output=self.layer_idx, final_layer_norm_intermediate=self.layer_norm_hidden_state, dtype=torch.float32)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/text_encoders/t5.py", line 245, in forward
return self.encoder(x, *args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/text_encoders/t5.py", line 217, in forward
x, past_bias = l(x, mask, past_bias, optimized_attention)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/text_encoders/t5.py", line 188, in forward
x, past_bias = self.layer[0](x, mask, past_bias, optimized_attention)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/text_encoders/t5.py", line 175, in forward
output, past_bias = self.SelfAttention(self.layer_norm(x), mask=mask, past_bias=past_bias, optimized_attention=optimized_attention)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/venv/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
return forward_call(*args, **kwargs)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/text_encoders/t5.py", line 164, in forward
out = optimized_attention(q, k * ((k.shape[-1] / self.num_heads) ** 0.5), v, self.num_heads, mask)
File "/home/brechtcorbeel/Downloads/Data/Packages/ComfyUINVIDIA/comfy/ldm/modules/attention.py", line 431, in attention_pytorch
out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=False)
torch.OutOfMemoryError: Allocation on device
```
```

### Other

_No response_