c4 dataset download fails

Tests currently skip over it.

Log:

dataset_name = 'c4'

@pytest.mark.parametrize(
    "dataset_name",
    ["wikitext2", "ptb", "c4", "alpaca"],
)
def test_get_dataset(dataset_name) -> None:

  ds = data_utils.get_dataset(name=dataset_name)

tests/test_dataloading.py:14:

src/slicegpt/data_utils.py:43: in get_dataset
ds = datasets.load_dataset(
/usr/local/lib/python3.10/dist-packages/datasets/load.py:2129: in load_dataset
builder_instance = load_dataset_builder(
/usr/local/lib/python3.10/dist-packages/datasets/load.py:1852: in load_dataset_builder
builder_instance: DatasetBuilder = builder_cls(
/usr/local/lib/python3.10/dist-packages/datasets/builder.py:373: in init
self.config, self.config_id = self._create_builder_config(

self = <datasets.packaged_modules.json.json.JsonC4 object at 0x7ffa704b1d20>, config_name = 'allenai--c4', custom_features = None
config_kwargs = {'data_files': {'train': ['hf://datasets/allenai/c4@1588ec454efa1a09f29cd18ddd04fe05fc8653a2/en/c4-train.00000-of-0102...ation': ['hf://datasets/allenai/c4@1588ec454efa1a09f29cd18ddd04fe05fc8653a2/en/c4-validation.00000-of-00008.json.gz']}}
builder_config = None

def _create_builder_config(
    self, config_name=None, custom_features=None, **config_kwargs
) -> Tuple[BuilderConfig, str]:
    """Create and validate BuilderConfig object as well as a unique config id for this config.
    Raises ValueError if there are multiple builder configs and config_name and DEFAULT_CONFIG_NAME are None.
    config_kwargs override the defaults kwargs in config
    """
    builder_config = None

    # try default config
    if config_name is None and self.BUILDER_CONFIGS and not config_kwargs:
        if self.DEFAULT_CONFIG_NAME is not None:
            builder_config = self.builder_configs.get(self.DEFAULT_CONFIG_NAME)
            logger.info(f"No config specified, defaulting to: {self.dataset_name}/{builder_config.name}")
        else:
            if len(self.BUILDER_CONFIGS) > 1:
                example_of_usage = f"load_dataset('{self.dataset_name}', '{self.BUILDER_CONFIGS[0].name}')"
                raise ValueError(
                    "Config name is missing."
                    f"\nPlease pick one among the available configs: {list(self.builder_configs.keys())}"
                    + f"\nExample of usage:\n\t`{example_of_usage}`"
                )
            builder_config = self.BUILDER_CONFIGS[0]
            logger.info(
                f"No config specified, defaulting to the single config: {self.dataset_name}/{builder_config.name}"
            )

    # try to get config by name
    if isinstance(config_name, str):
        builder_config = self.builder_configs.get(config_name)
        if builder_config is None and self.BUILDER_CONFIGS:

          raise ValueError(

                f"BuilderConfig '{config_name}' not found. Available: {list(self.builder_configs.keys())}"
            )

E ValueError: BuilderConfig 'allenai--c4' not found. Available: ['en', 'en.noblocklist', 'en.noclean', 'realnewslike', 'multilingual', 'af', 'am', 'ar', 'az', 'be', 'bg', 'bg-Latn', 'bn', 'ca', 'ceb', 'co', 'cs', 'cy', 'da', 'de', 'el', 'el-Latn', 'en-multi', 'eo', 'es', 'et', 'eu', 'fa', 'fi', 'fil', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'haw', 'hi', 'hi-Latn', 'hmn', 'ht', 'hu', 'hy', 'id', 'ig', 'is', 'it', 'iw', 'ja', 'ja-Latn', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'ku', 'ky', 'la', 'lb', 'lo', 'lt', 'lv', 'mg', 'mi', 'mk', 'ml', 'mn', 'mr', 'ms', 'mt', 'my', 'ne', 'nl', 'no', 'ny', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'ru-Latn', 'sd', 'si', 'sk', 'sl', 'sm', 'sn', 'so', 'sq', 'sr', 'st', 'su', 'sv', 'sw', 'ta', 'te', 'tg', 'th', 'tr', 'uk', 'und', 'ur', 'uz', 'vi', 'xh', 'yi', 'yo', 'zh', 'zh-Latn', 'zu']

/usr/local/lib/python3.10/dist-packages/datasets/builder.py:539: ValueError

Start with tests/test_dataloading.py and src/slicegpt/data_utils.py, then run the parametrized dataset test to reproduce the C4 failure. Check how get_dataset calls the Hugging Face datasets loader and compare the requested C4 configuration with the available configurations. Done means the C4 case no longer fails or needs to be skipped.