Add Hugging Face filesystem support to fsspec (#1997)

# Rationale for this change

Add support for the Hugging Face filesystem in `fsspec`, which uses
`hf://` paths.
This allows to import [HF datasets](https://huggingface.co/datasets).

Authentication is done using the `"hf.token"` property.

# Are these changes tested?

I tried locally but haven't added tests in test_fsspec.py (lmk if it's a
requirement)

# Are there any user-facing changes?

No changes, it simply adds support for `hf://` URLs

Author: lhoestq

mkdocs/docs/configuration.md

@@ -95,6 +95,7 @@ Iceberg works with the concept of a FileIO which is a pluggable module for readi
 - **hdfs**: `PyArrowFileIO`
 - **abfs**, **abfss**: `FsspecFileIO`
 - **oss**: `PyArrowFileIO`
+- **hf**: `FsspecFileIO`
 
 You can also set the FileIO explicitly:
 
@@ -193,6 +194,17 @@ PyIceberg uses [S3FileSystem](https://arrow.apache.org/docs/python/generated/pya
 
 <!-- markdown-link-check-enable-->
 
+### Hugging Face
+
+<!-- markdown-link-check-disable -->
+
+| Key         | Example                  | Description                                               |
+| ----------- | ------------------------ | --------------------------------------------------------- |
+| hf.endpoint | <https://huggingface.co> | Configure the endpoint for Hugging Face                   |
+| hf.token    | hf_xxx                   | The Hugging Face token to access HF Datasets repositories |
+
+<!-- markdown-link-check-enable-->
+
 ### PyArrow
 
 <!-- markdown-link-check-disable -->

poetry.lock

@@ -92,6 +92,8 @@
 GCS_SERVICE_HOST = "gcs.service.host"
 GCS_DEFAULT_LOCATION = "gcs.default-bucket-location"
 GCS_VERSION_AWARE = "gcs.version-aware"
+HF_ENDPOINT = "hf.endpoint"
+HF_TOKEN = "hf.token"
 PYARROW_USE_LARGE_TYPES_ON_READ = "pyarrow.use-large-types-on-read"
 
 
@@ -306,6 +308,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None:
     "viewfs": [ARROW_FILE_IO],
     "abfs": [FSSPEC_FILE_IO],
     "abfss": [FSSPEC_FILE_IO],
+    "hf": [FSSPEC_FILE_IO],
 }
 
 

pyiceberg/io/__init__.py

@@ -59,6 +59,8 @@
     GCS_SESSION_KWARGS,
     GCS_TOKEN,
     GCS_VERSION_AWARE,
+    HF_ENDPOINT,
+    HF_TOKEN,
     S3_ACCESS_KEY_ID,
     S3_CONNECT_TIMEOUT,
     S3_ENDPOINT,
@@ -209,6 +211,15 @@ def _adls(properties: Properties) -> AbstractFileSystem:
     )
 
 
+def _hf(properties: Properties) -> AbstractFileSystem:
+    from huggingface_hub import HfFileSystem
+
+    return HfFileSystem(
+        endpoint=properties.get(HF_ENDPOINT),
+        token=properties.get(HF_TOKEN),
+    )
+
+
 SCHEME_TO_FS = {
     "": _file,
     "file": _file,
@@ -219,6 +230,7 @@ def _adls(properties: Properties) -> AbstractFileSystem:
     "abfss": _adls,
     "gs": _gs,
     "gcs": _gs,
+    "hf": _hf,
 }
 
 

pyiceberg/io/fsspec.py

@@ -76,6 +76,7 @@ boto3 = { version = ">=1.24.59", optional = true }
 s3fs = { version = ">=2023.1.0", optional = true }
 adlfs = { version = ">=2023.1.0", optional = true }
 gcsfs = { version = ">=2023.1.0", optional = true }
+huggingface-hub = { version = ">=0.24.0", optional = true }
 psycopg2-binary = { version = ">=2.9.6", optional = true }
 sqlalchemy = { version = "^2.0.18", optional = true }
 getdaft = { version = ">=0.2.12", optional = true }
@@ -306,6 +307,7 @@ sql-postgres = ["sqlalchemy", "psycopg2-binary"]
 sql-sqlite = ["sqlalchemy"]
 gcsfs = ["gcsfs"]
 rest-sigv4 = ["boto3"]
+hf = ["huggingface-hub"]
 pyiceberg-core = ["pyiceberg-core"]
 
 [tool.pytest.ini_options]
@@ -427,6 +429,10 @@ ignore_missing_imports = true
 module = "gcsfs.*"
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = "huggingface_hub.*"
+ignore_missing_imports = true
+
 [[tool.mypy.overrides]]
 module = "packaging.*"
 ignore_missing_imports = true