Out of memory when using ControlNet on memory-constrained hardware in GPU mode

I'm using pretty much lowend laptop to generate images using ComfyUI, with MX150 and 2Gb Vram (split out of 16Gb RAM).

When using straightforward workflow with text2img + upscaler it runs fine in GPU mode. However, when I add a Load ControlNet + Apply ControlNet nodes, the KSampler node yields following error:

```python
Error occurred when executing KSampler:

Allocation on device 0 would exceed allowed memory. (out of memory)
Currently allocated : 1.70 GiB
Requested : 10.00 MiB
Device limit : 2.00 GiB
Free (according to CUDA): 0 bytes
PyTorch limit (set by user-supplied memory fraction)
: 17179869184.00 GiB

File "E:\ComfyUI_windows_portable\ComfyUI\execution.py", line 145, in recursive_execute
output_data, output_ui = get_output_data(obj, input_data_all)
File "E:\ComfyUI_windows_portable\ComfyUI\execution.py", line 75, in get_output_data
return_values = map_node_over_list(obj, input_data_all, obj.FUNCTION, allow_interrupt=True)
File "E:\ComfyUI_windows_portable\ComfyUI\execution.py", line 68, in map_node_over_list
results.append(getattr(obj, func)(**slice_dict(input_data_all, i)))
File "E:\ComfyUI_windows_portable\ComfyUI\nodes.py", line 1082, in sample
return common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=denoise)
File "E:\ComfyUI_windows_portable\ComfyUI\nodes.py", line 1052, in common_ksampler
samples = comfy.sample.sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image,
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\sample.py", line 88, in sample
samples = sampler.sample(noise, positive_copy, negative_copy, cfg=cfg, latent_image=latent_image, start_step=start_step, last_step=last_step, force_full_denoise=force_full_denoise, denoise_mask=noise_mask, sigmas=sigmas, callback=callback, disable_pbar=disable_pbar, seed=seed)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\samplers.py", line 671, in sample
samples = getattr(k_diffusion_sampling, "sample_{}".format(self.sampler))(self.model_k, noise, sigmas, extra_args=extra_args, callback=k_callback, disable=disable_pbar)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\utils\_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\k_diffusion\sampling.py", line 138, in sample_euler
denoised = model(x, sigma_hat * s_in, **extra_args)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\samplers.py", line 308, in forward
out = self.inner_model(x, sigma, cond=cond, uncond=uncond, cond_scale=cond_scale, cond_concat=cond_concat, model_options=model_options, seed=seed)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\k_diffusion\external.py", line 125, in forward
eps = self.get_eps(input * c_in, self.sigma_to_t(sigma), **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\k_diffusion\external.py", line 151, in get_eps
return self.inner_model.apply_model(*args, **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\samplers.py", line 296, in apply_model
out = sampling_function(self.inner_model.apply_model, x, timestep, uncond, cond, cond_scale, cond_concat, model_options=model_options, seed=seed)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\samplers.py", line 274, in sampling_function
cond, uncond = calc_cond_uncond_batch(model_function, cond, uncond, x, timestep, max_total_area, cond_concat, model_options)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\samplers.py", line 232, in calc_cond_uncond_batch
c['control'] = control.get_control(input_x, timestep_, c, len(cond_or_uncond))
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\sd.py", line 706, in get_control
control = self.control_model(x=x_noisy, hint=self.cond_hint, timesteps=t, context=context, y=y)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\cldm\cldm.py", line 311, in forward
h = module(h, emb, context)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\ldm\modules\diffusionmodules\openaimodel.py", line 82, in forward
x = layer(x, emb)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\ldm\modules\diffusionmodules\openaimodel.py", line 271, in forward
return checkpoint(
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\ldm\modules\diffusionmodules\util.py", line 123, in checkpoint
return func(*inputs)
File "E:\ComfyUI_windows_portable\ComfyUI\comfy\ldm\modules\diffusionmodules\openaimodel.py", line 295, in _forward
h = self.out_layers(h)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\container.py", line 217, in forward
input = module(input)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\modules\activation.py", line 396, in forward
return F.silu(input, inplace=self.inplace)
File "E:\ComfyUI_windows_portable\python_embeded\lib\site-packages\torch\nn\functional.py", line 2059, in silu
return torch._C._nn.silu(input)
```

Running in CPU mode works, albeit obviously very slow.

Both working and erroring workflows are attached:
[workflows.zip](https://github.com/comfyanonymous/ComfyUI/files/12130873/workflows.zip)