Unhelpful logs for auth failures caused by SMTP server timeouts

Start by reproducing the SMTP timeout during the /otp user confirmation request and inspect the API authentication logging around that path. Done means the logs identify the SMTP timeout as the cause instead of reporting only a generic 504.

I experienced SMTP server timeouts (not errors) and the result was 504 errors from the api, and no indication that SMTP is the cause in the logs.

When the SMTP server errors, we get 500 errors with appropriate error messages, as expected.

A sample auth log event for the SMTP timeout:

[
  {
    "host": "db-pwgydselyeejhebljvsb",
    "component": "api",
    "_SYSTEMD_CGROUP": null,
    "request_id": "8f49f8f1a0914d17-BOS",
    "mail_from": null,
    "message": null,
    "_SOURCE_REALTIME_TIMESTAMP": null,
    "PRIORITY": null,
    "_AUDIT_LOGINUID": null,
    "panic": null,
    "metering": null,
    "UNIT": null,
    "event": null,
    "SYSLOG_FACILITY": null,
    "msg": "504: Processing this request timed out, please retry after a moment.",
    "mail_type": null,
    "EXECUTABLE": null,
    "user_id": null,
    "_CMDLINE": null,
    "action": null,
    "auth_event": [
      {
        "action": "user_confirmation_requested",
        "actor_id": "cdac3d23-a57e-4c22-a430-7526a39fa4f6",
        "actor_name": null,
        "actor_username": "redacted",
        "actor_via_sso": false,
        "log_type": "user",
        "traits": [
          {
            "channel": null,
            "identity_id": null,
            "provider": "email",
            "provider_id": null,
            "provider_type": null,
            "user_email": null,
            "user_id": null,
            "user_phone": null
          }
        ]
      }
    ],
    "level": "error",
    "_PID": null,
    "path": "/otp",
    "duration": null,
    "_COMM": null,
    "header": null,
    "_MACHINE_ID": null,
    "login_method": null,
    "_STREAM_ID": null,
    "source_type": null,
    "_LINE_BREAK": null,
    "_EXE": null,
    "_AUDIT_SESSION": null,
    "_TRANSPORT": null,
    "x_forwarded_proto": null,
    "time": null,
    "mail_to": null,
    "_GID": null,
    "stack": null,
    "x_forwarded_host": null,
    "status": null,
    "_UID": null,
    "method": "POST",
    "CODE_FILE": null,
    "remote_addr": "redacted",
    "provider": null,
    "_SYSTEMD_UNIT": null,
    "issuer": null,
    "error": "context deadline exceeded",
    "client_id": null,
    "MESSAGE_ID": null,
    "referer": "redacted",
    "_SYSTEMD_INVOCATION_ID": null,
    "CODE_FUNC": null,
    "_BOOT_ID": null,
    "INVOCATION_ID": null,
    "__MONOTONIC_TIMESTAMP": null,
    "timestamp": null,
    "__REALTIME_TIMESTAMP": null,
    "CODE_LINE": null,
    "_SYSTEMD_SLICE": null,
    "instance_id": null,
    "args": [],
    "SYSLOG_IDENTIFIER": null,
    "metadata": [],
    "_CAP_EFFECTIVE": null,
    "factor_id": null,
    "_SELINUX_CONTEXT": null,
    "project": null
  }
]