PermissionError with ModelCheckpoints

Hi I'm trying to train a model and am getting this error:

Traceback (most recent call last):
  File "/home/jovyan/talmolab-smb/aadi/biogtr_expts/run/animal/SLAP_M74/single_run.py", line 91, in <module>
    main(cfg.cfg)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/hydra/main.py", line 83, in decorated_main
    return task_function(cfg_passthrough)
  File "/home/jovyan/talmolab-smb/aadi/biogtr_expts/src/biogtr/biogtr/training/train.py", line 101, in main
    trainer.fit(model, dataset)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 544, in fit
    call._call_and_handle_interrupt(
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 44, in _call_and_handle_interrupt
    return trainer_fn(*args, **kwargs)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 580, in _fit_impl
    self._run(model, ckpt_path=ckpt_path)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 989, in _run
    results = self._run_stage()
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py", line 1035, in _run_stage
    self.fit_loop.run()
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 203, in run
    self.on_advance_end()
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py", line 374, in on_advance_end
    call._call_callback_hooks(trainer, "on_train_epoch_end", monitoring_callbacks=True)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/trainer/call.py", line 208, in _call_callback_hooks
    fn(trainer, trainer.lightning_module, *args, **kwargs)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 314, in on_train_epoch_end
    self._save_last_checkpoint(trainer, monitor_candidates)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 679, in _save_last_checkpoint
    self._link_checkpoint(trainer, self._last_checkpoint_saved, filepath)
  File "/opt/conda/envs/biogtr/lib/python3.9/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 397, in _link_checkpoint
    shutil.copy(filepath, linkpath)
  File "/opt/conda/envs/biogtr/lib/python3.9/shutil.py", line 428, in copy
    copymode(src, dst, follow_symlinks=follow_symlinks)
  File "/opt/conda/envs/biogtr/lib/python3.9/shutil.py", line 317, in copymode
    chmod_func(dst, stat.S_IMODE(st.st_mode))
PermissionError: [Errno 1] Operation not permitted: '/home/jovyan/talmolab-smb/aadi/biogtr_expts/run/animal/SLAP_M74/models/tests/test_chkpt/epoch=1-best-val_num_switches=36.0.ckpt'

This is how i set up my checkpoints:

def get_checkpointing(self) -> pl.callbacks.ModelCheckpoint:
        """Getter for lightning checkpointing callback.

        Returns:
            A lightning checkpointing callback with specified params
        """
        # convert to dict to enable extracting/removing params
        checkpoint_params = OmegaConf.to_container(self.cfg.checkpointing, resolve=True)
        logging_params = self.cfg.logging
        if "dirpath" not in checkpoint_params or checkpoint_params["dirpath"] is None:
            if "group" in logging_params:
                dirpath = f"./models/{logging_params.group}/{logging_params.name}"
            else:
                dirpath = f"./models/{logging_params.name}"

        else:
            dirpath = checkpoint_params["dirpath"]
        
        dirpath = Path(dirpath).resolve()
        if not Path(dirpath).exists():
            try:
                Path(dirpath).mkdir(parents=True, exist_ok=True)
            except OSError as e:
                print(
                    f"Cannot create a new folder. Check the permissions to the given Checkpoint directory. \n {e}"
                )
        
        _ = checkpoint_params.pop("dirpath")
        checkpointers = []
        monitor = checkpoint_params.pop("monitor")
        for metric in monitor:
            checkpointer = pl.callbacks.ModelCheckpoint(
                monitor=metric, dirpath=dirpath, filename=f"{{epoch}}-{{{metric}}}", **checkpoint_params
            )
            checkpointer.CHECKPOINT_NAME_LAST = f"{{epoch}}-best-{{{metric}}}"
            checkpointers.append(checkpointer)
        return checkpointers

Its quite strange because this error never used to happen before

Originally posted by @aaprasad in https://github.com/Lightning-AI/pytorch-lightning/discussions/19396

cc @carmocca @awaelchli

Start in pytorch_lightning/callbacks/model_checkpoint.py at _save_last_checkpoint and _link_checkpoint, where the traceback reaches shutil.copy and fails during chmod. Reproduce the reported training setup with the shown ModelCheckpoint configuration and determine whether checkpoint saving can complete without the PermissionError. No repository test is named, so completion would require a focused regression test or documented reproduction if the failure is supported.