Training runs failing because of "Unable to locate credentials" errors

We've noticed that our long-running training runs often fail because of some deep "Unable to locate credentials" errors. We're using it on ec2 machines whose permissions are dictated by IAM.

```
Error logging answer rate metrics to MLflow: API request to https://us-east-2.experiments.sagemaker.aws/api/2.0/mlflow/runs/log-metric failed with exception Unable to locate credentials
[rank0]: Traceback (most recent call last):
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/utils/rest_utils.py", line 230, in http_request
[rank0]: return _get_http_response_with_retries(
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/utils/request_utils.py", line 237, in _get_http_response_with_retries
[rank0]: return session.request(method, url, allow_redirects=allow_redirects, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/requests/sessions.py", line 575, in request
[rank0]: prep = self.prepare_request(req)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/requests/sessions.py", line 484, in prepare_request
[rank0]: p.prepare(
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/requests/models.py", line 371, in prepare
[rank0]: self.prepare_auth(auth, url)
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/requests/models.py", line 602, in prepare_auth
[rank0]: r = auth(self)
[rank0]: ^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/sagemaker_mlflow/auth.py", line 66, in __call__
[rank0]: self.sigv4.add_auth(aws_request)
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/botocore/auth.py", line 421, in add_auth
[rank0]: raise NoCredentialsError()
[rank0]: botocore.exceptions.NoCredentialsError: Unable to locate credentials

[rank0]: During handling of the above exception, another exception occurred:

[rank0]: Traceback (most recent call last):
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/multi_gpu_training.py", line 475, in
[rank0]: main()
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/multi_gpu_training.py", line 450, in main
[rank0]: trainer.train()
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/transformers/trainer.py", line 2238, in train
[rank0]: return inner_training_loop(
[rank0]: ^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/transformers/trainer.py", line 2582, in _inner_training_loop
[rank0]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/transformers/trainer.py", line 3790, in training_step
[rank0]: inputs = self._prepare_inputs(inputs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/trl/extras/profiling.py", line 98, in wrapper
[rank0]: return func(self, *args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/trl/trainer/grpo_trainer.py", line 1279, in _prepare_inputs
[rank0]: generation_batch = self._generate_and_score_completions(generation_batch)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/trl/trainer/grpo_trainer.py", line 1677, in _generate_and_score_completions
[rank0]: rewards_per_func = self._calculate_rewards(inputs, original_prompts, completions, completion_ids_list)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/trl/extras/profiling.py", line 98, in wrapper
[rank0]: return func(self, *args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/trl/trainer/grpo_trainer.py", line 1307, in _calculate_rewards
[rank0]: with profiling_context(self, reward_func_name):
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/.local/share/uv/python/cpython-3.12.10-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
[rank0]: next(self.gen)
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/trl/extras/profiling.py", line 68, in profiling_context
[rank0]: mlflow.log_metrics(profiling_metrics, step=trainer.state.global_step)
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/tracking/fluent.py", line 1120, in log_metrics
[rank0]: else (_get_model_ids_for_new_metric_if_exist(run_id, step) or [None])
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/tracking/fluent.py", line 1052, in _get_model_ids_for_new_metric_if_exist
[rank0]: run = client.get_run(run_id)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/tracking/client.py", line 268, in get_run
[rank0]: return self._tracking_client.get_run(run_id)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py", line 100, in get_run
[rank0]: return self.store.get_run(run_id)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/store/tracking/rest_store.py", line 227, in get_run
[rank0]: response_proto = self._call_endpoint(GetRun, req_body)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/store/tracking/rest_store.py", line 133, in _call_endpoint
[rank0]: return call_endpoint(
[rank0]: ^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/utils/rest_utils.py", line 548, in call_endpoint
[rank0]: response = http_request(**call_kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/home/user/src/answerbot/src/fine_tuning/exp/finetuning_da/.venv/lib/python3.12/site-packages/mlflow/utils/rest_utils.py", line 253, in http_request
[rank0]: raise MlflowException(f"API request to {url} failed with exception {e}")
[rank0]: mlflow.exceptions.MlflowException: API request to https://us-east-2.experiments.sagemaker.aws/api/2.0/mlflow/runs/get failed with exception Unable to locate credentials

```

Start at multi_gpu_training.py around main() and trainer.train() in the traceback, then reproduce the metric-logging failure on an EC2 machine using IAM permissions. Trace the MLflow request that raises NoCredentialsError and verify a long-running training run completes with its answer-rate and profiling metrics logged.

Written by the indexing model from the issue text.