Downloading large files

I wrote this function for one of my projects that, I think, can be used here for downloading large files concurrently and efficiently using requests and joblib. It only downloads files if they have been modified on the server since they have been download or if their sizes have changed.

import functools
from pathlib import Path
from collections.abc import Callable

import joblib
import pandas as pd
from requests import Response, Session

MAX_CONN = 10
CHUNK_SIZE = int(100 * 1024 * 1024)  # 100 MB

def _download(
    session_func: Callable[[str], Response],
    url: str,
    fname: Path,
    chunk_size: int,
) -> None:
    """Download a single file."""
    resp = session_func(url)
    fsize = int(resp.headers.get("Content-Length", -1))
    last_modifed = resp.headers.get("Last-Modified", None)
    if fname.exists():
        if last_modifed:
            if pd.to_datetime(last_modifed) >= pd.to_datetime(fname.stat().st_mtime, unit="s"):
                return
        elif fname.stat().st_size == fsize:
            return

    fname.parent.mkdir(exist_ok=True, parents=True)
    with fname.open("wb") as f:
        f.writelines(resp.iter_content(chunk_size))


def streaming_download(
    urls: str | list[str],
    file_paths: str | Path | list[str] | list[Path],
    chunk_size: int = CHUNK_SIZE,
    n_jobs: int = MAX_CONN,
) -> None:
    """Download and store files in parallel from a list of URLs/Keywords.

    Notes
    -----
    This function runs asynchronously in parallel using ``n_jobs`` threads.

    Parameters
    ----------
    urls : str or list of str
        A list of URLs to download.
    file_paths : str, pathlib.Path or list of them
        A list of filenames associated with each URL, e.g.,
        ("file1.zip", ...).
    chunk_size : int, optional
        Chunk size to use when downloading, defaults to 100 * 1024 * 1024
        i.e., 100 MB.
    n_jobs: int, optional
        The maximum number of concurrent downloads, defaults to 10.
    """
    url_list = [urls] if isinstance(urls, str) else urls
    file_list = [file_paths] if isinstance(file_paths, (str, Path)) else file_paths
    file_list = [Path(f) for f in file_list]
    if len(url_list) != len(file_list):
        raise TypeError("urls/file_paths should be equal length lists")

    session = Session()
    func = functools.partial(session.get, stream=True)
    n_jobs = min(n_jobs, len(url_list))
    joblib.Parallel(n_jobs=n_jobs, prefer="threads")(
        joblib.delayed(_download)(func, u, f, chunk_size)
        for u, f in zip(url_list, file_list, strict=False)
    )
    session.close()

The issue names no repository files, tests, or entry points; start by locating the existing download or data-ingestion code and reviewing how it handles remote files. Clarify where this functionality belongs and define completion around concurrent large-file downloads with conditional refresh based on server metadata or size.