Ensure that Croissant versions that only specify major and minor versions are properly converted to Semantic versions.

PiperOrigin-RevId: 760995569

Author: The TensorFlow Datasets Authors

tensorflow_datasets/core/dataset_builders/croissant_builder.py

@@ -279,11 +279,17 @@ def __init__(
     self.name = croissant_utils.get_tfds_dataset_name(dataset)
     self.metadata = dataset.metadata
 
-    # In TFDS, version is a mandatory attribute, while in Croissant it is only a
-    # recommended attribute. If the version is unspecified in Croissant, we set
-    # it to `1.0.0` in TFDS.
+    # The dataset version is determined using the following precedence:
+    # * overwrite_version (if provided).
+    # * The version from Croissant metadata (self.metadata.version),
+    # automatically converting major.minor formats to major.minor.0 (e.g., "1.2"
+    # becomes "1.2.0"). See croissant_utils.get_croissant_version for details.
+    # * Defaults to '1.0.0' if no version is specified (version is optional in
+    # Croissant, but mandatory in TFDS).
     self.VERSION = version_lib.Version(  # pylint: disable=invalid-name
-        overwrite_version or self.metadata.version or '1.0.0'
+        overwrite_version
+        or croissant_utils.get_croissant_version(self.metadata.version)
+        or '1.0.0'
     )
     self.RELEASE_NOTES = {}  # pylint: disable=invalid-name
 

tensorflow_datasets/core/dataset_builders/croissant_builder_test.py

@@ -249,6 +249,17 @@ def test_sequence_feature_datatype_converter():
   assert isinstance(actual_feature.feature, text_feature.Text)
 
 
+def test_version_converter(tmp_path):
+  with testing.dummy_croissant_file(version="1.0") as croissant_file:
+    builder = croissant_builder.CroissantBuilder(
+        jsonld=croissant_file,
+        file_format=FileFormat.ARRAY_RECORD,
+        disable_shuffling=True,
+        data_dir=tmp_path,
+    )
+    assert builder.version == "1.0.0"
+
+
 @pytest.fixture(name="crs_builder")
 def mock_croissant_dataset_builder(tmp_path, request):
   dataset_name = request.param["dataset_name"]

tensorflow_datasets/core/utils/croissant_utils.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import dataclasses
+import re
 import typing
 
 from tensorflow_datasets.core.utils import conversion_utils
@@ -29,6 +30,12 @@
 
 _HUGGINGFACE_URL_PREFIX = "https://huggingface.co/datasets/"
 
+_VERSION_TMPL_WITHOUT_PATCH = r"^(?P<major>{v})" r"\.(?P<minor>{v})$"
+_NO_LEADING_ZEROS = r"\d|[1-9]\d*"
+_VERSION_REGEX_WITHOUT_PATCH = re.compile(
+    _VERSION_TMPL_WITHOUT_PATCH.format(v=_NO_LEADING_ZEROS)
+)
+
 
 @dataclasses.dataclass(frozen=True)
 class SplitReference:
@@ -40,6 +47,28 @@ class SplitReference:
   reference_field: mlc.Field
 
 
+def get_croissant_version(version: str | None) -> str | None:
+  """Returns the possibly corrected Croissant version in TFDS format.
+
+  TFDS expects versions to follow the Semantic versioning 2.0.0 syntax, but
+  Croissant is more lax and accepts also {major.minor} (without leading zeros).
+  To avoid raising errors in these cases, we add a 0 as a patch version to the
+  Croissant-provided version.
+
+  Args:
+    version: The Croissant version.
+
+  Returns:
+    The Croissant version in TFDS format.
+  """
+  if not version:
+    return None
+  res = _VERSION_REGEX_WITHOUT_PATCH.match(version)
+  if res:
+    return f"{version}.0"
+  return version
+
+
 def get_dataset_name(dataset: mlc.Dataset) -> str:
   """Returns dataset name of the given MLcroissant dataset."""
   if (url := dataset.metadata.url) and url.startswith(_HUGGINGFACE_URL_PREFIX):

tensorflow_datasets/core/utils/croissant_utils_test.py

@@ -36,6 +36,23 @@ def test_get_tfds_dataset_name(croissant_name, croissant_url, tfds_name):
   assert croissant_utils.get_tfds_dataset_name(dataset) == tfds_name
 
 
+@pytest.mark.parametrize(
+    'croissant_version,tfds_version',
+    [
+        ('1.0', '1.0.0'),
+        ('1.2', '1.2.0'),
+        ('1.2.3', '1.2.3'),
+        ('1.2.3.4', '1.2.3.4'),
+        ('0.1', '0.1'),
+        (None, None),
+    ],
+)
+def test_get_croissant_version(croissant_version, tfds_version):
+  assert (
+      croissant_utils.get_croissant_version(croissant_version) == tfds_version
+  )
+
+
 def test_get_record_set_ids():
   metadata = mlc.Metadata(
       name='dummy_dataset',

tensorflow_datasets/testing/test_utils.py

@@ -724,6 +724,7 @@ def dummy_croissant_file(
     raw_data_filename: epath.PathLike = 'raw_data.jsonl',
     croissant_filename: epath.PathLike = 'croissant.json',
     split_names: Sequence[str] | None = None,
+    version: str = '1.2.0',
 ) -> Iterator[epath.Path]:
   """Yields temporary path to a dummy Croissant file.
 
@@ -746,6 +747,7 @@ def dummy_croissant_file(
       If None, the function will create a split record set with the default
       split names `train` and `test`. If `split_names` is defined, the `split`
       key in the entries must match one of the split names.
+    version: The version of the dataset. Defaults to `1.2.0`.
   """
   if entries is None:
     entries = [
@@ -874,7 +876,7 @@ def dummy_croissant_file(
         url='https://dummy_url',
         distribution=distribution,
         record_sets=record_sets,
-        version='1.2.0',
+        version=version,
         license='Public',
     )
     # Write Croissant JSON-LD to tempdir.