Fix minio/parquet url to reflect the latest changes

Author: PGijsbers

src/core/formatting.py

@@ -27,7 +27,7 @@ def _format_parquet_url(dataset: Row) -> str | None:
 
     minio_base_url = load_configuration()["minio_base_url"]
     prefix = dataset.did // 10_000
-    return f"{minio_base_url}/{prefix:04d}/{dataset.did:04d}/dataset_{dataset.did}.pq"
+    return f"{minio_base_url}/datasets/{prefix:04d}/{dataset.did:04d}/dataset_{dataset.did}.pq"
 
 
 def _format_dataset_url(dataset: Row) -> str:

src/routers/openml/datasets.py

@@ -443,7 +443,6 @@ def get_dataset(
         row_id_attribute=row_id_attribute,
         url=dataset_url,
         parquet_url=parquet_url,
-        minio_url=parquet_url,
         file_id=dataset.file_id,
         format=dataset.format.lower(),
         paper_url=dataset.paper_url or None,

src/schemas/datasets/openml.py

@@ -124,12 +124,12 @@ class DatasetMetadata(BaseModel):
             "description": "URL of the parquet dataset data file.",
         },
     )
-    minio_url: HttpUrl | None = Field(
-        json_schema_extra={
-            "example": "http://openml1.win.tue.nl/dataset2/dataset_2.pq",
-            "description": "Deprecated, I think.",
-        },
-    )
+    # minio_url: HttpUrl | None = Field(
+    #     json_schema_extra={
+    #         "example": "http://openml1.win.tue.nl/dataset2/dataset_2.pq",
+    #         "description": "Deprecated, I think.",
+    #     },
+    # )
     file_id: int = Field(json_schema_extra={"example": 1})
     format_: DatasetFileFormat = Field(
         json_schema_extra={"example": DatasetFileFormat.ARFF},

tests/routers/openml/datasets_test.py

@@ -42,13 +42,12 @@ def test_get_dataset(py_api: TestClient) -> None:
         "upload_date": "2014-04-06T23:19:24",
         "licence": "Public",
         "url": "https://test.openml.org/data/v1/download/1/anneal.arff",
-        "parquet_url": "https://openml1.win.tue.nl/0000/0001/dataset_1.pq",
+        "parquet_url": "https://openml1.win.tue.nl/datasets/0000/0001/dataset_1.pq",
         "file_id": 1,
         "default_target_attribute": ["class"],
         "version_label": "1",
         "tag": ["study_14"],
         "visibility": "public",
-        "minio_url": "https://openml1.win.tue.nl/0000/0001/dataset_1.pq",
         "status": "in_preparation",
         "processing_date": "2024-01-04T10:13:59",
         "md5_checksum": "4eaed8b6ec9d8211024b6c089b064761",

tests/routers/openml/migration/datasets_migration_test.py

@@ -58,14 +58,6 @@ def test_dataset_response_is_identical(
     new_body = new.json()
     if processing_data := new_body.get("processing_date"):
         new_body["processing_date"] = str(processing_data).replace("T", " ")
-    if parquet_url := new_body.get("parquet_url"):
-        bucket, prefix, did, filename = parquet_url.rsplit("/", 3)
-        new_body["parquet_url"] = f"{bucket}/dataset{did.lstrip('0')}/{filename}"
-        new_body["parquet_url"] = new_body["parquet_url"].replace("https", "http")
-    if minio_url := new_body.get("minio_url"):
-        bucket, prefix, did, filename = minio_url.rsplit("/", 3)
-        new_body["minio_url"] = f"{bucket}/dataset{did.lstrip('0')}/{filename}"
-        new_body["minio_url"] = new_body["minio_url"].replace("https", "http")
 
     manual = []
     # ref test.openml.org/d/33 (contributor) and d/34 (creator)