Recommended way to shuffle intra and inter archives?

Say I have a bunch of archives containing samples. In my case each archive is a pickle file containing a list of samples, but it could be a tar or something else.

I want to shuffle between archives (inter) and within archives (intra). My current way of doing it is below. Is there a more canonical solution?

from torchdata.dataloader2 import DataLoader2, adapter
from torchdata.datapipes.iter import IterDataPipe, FileLister, IterableWrapper
from pathlib import Path

import pickle

# Create archives
root = Path("/tmp/dataset/")
with open(root / "1.pkl", "wb") as f:
    pickle.dump(list(range(10)), f)
with open(root / "2.pkl", "wb") as f:
    pickle.dump(list(range(10, 20)), f)

class PickleLoaderDataPipe(IterDataPipe):
    def __init__(self, source_datapipe):
        self.source_datapipe = source_datapipe

    def __iter__(self):
        for path in self.source_datapipe:
            with open(path, "rb") as f:
                yield pickle.load(f)  # <- this is a list

class ConcaterIterable(IterDataPipe):
    # Same as unbatch(), kinda
    def __init__(self, source_datapipe):
        self.source_datapipe = source_datapipe

    def __iter__(self):
        for iterable in self.source_datapipe:
            yield from iterable

def intra_archive_shuffle(archive_content):
    return IterableWrapper(archive_content).shuffle()
    
    
dp = FileLister(str(root), masks=["*.pkl"])
dp = dp.shuffle()  # inter-archive shuffling
dp = PickleLoaderDataPipe(dp)
dp = dp.map(intra_archive_shuffle)
dp = ConcaterIterable(dp)  # Note: unbatch doesn't work because it's a datapipe of datapipes

print(list(dp))

The issue names DataLoader2, iterable DataPipes, FileLister, shuffle, map, and unbatch, but no repository files or tests. Start by locating the existing shuffle and nested DataPipe implementations; done would require a decided canonical approach for inter- and intra-archive shuffling, with tests covering the resulting sample ordering.