From 6a3956e615b3a23aeee92a6036ed9ab819ab9f3a Mon Sep 17 00:00:00 2001 From: laurianvm Date: Thu, 5 Nov 2026 15:34:05 +0100 Subject: [PATCH 1/2] implement yaml support --- sema/subyt/sources.py | 68 ++++++++++++++++++- tests/subyt/test_yaml_source.py | 114 ++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 tests/subyt/test_yaml_source.py diff --git a/sema/subyt/sources.py b/sema/subyt/sources.py index 2f5ce1f1..25ec36af 100644 --- a/sema/subyt/sources.py +++ b/sema/subyt/sources.py @@ -38,7 +38,11 @@ class SourceFactory: def __init__(self): self._register = dict() - self._ext_mime_map = {"csv": "text/csv"} + self._ext_mime_map = { + "csv": "text/csv", + "yaml": "application/yaml", + "yml": "application/yaml", + } @property def ext_2_mime(self): @@ -568,3 +572,65 @@ def __repr__(self) -> str: SourceFactory.register("application/xml", XMLFileSource) except ImportError: log.warning("Python XML module not available -- disabling XML support!") + + +try: + import yaml + + class YamlFileSource(Source): + """ + Source producing iterator over data-set coming from YAML on file + """ + + def __init__(self, yaml_file_path: Path, config: dict = {}) -> None: + super().__init__() + assert_readable(yaml_file_path) + self._yaml: Path = yaml_file_path.absolute() + self._init_source(self._yaml) + + def __enter__(self) -> object: + with open(self._yaml, mode="r", encoding="utf-8-sig") as yamlfile: + docs = [ + doc + for doc in yaml.safe_load_all(yamlfile) + if doc is not None + ] + + if not docs: + data = [] + elif len(docs) > 1: + data = docs + else: + data = docs[0] + # unwrap nested structures + while isinstance(data, dict) and len(data.keys()) == 1: + child = list(data.values())[0] + if isinstance(child, (dict, list)): + if ( + isinstance(child, dict) + and len(child.keys()) == 0 + ): + data = [data] + break + data = child + else: + break + if not isinstance(data, list): + data = [data] + + return iter(data) + + def __exit__(self, *exc) -> None: + pass + + def __repr__(self) -> str: + return f"YamlFileSource('{self._yaml!s}')" + + SourceFactory.map("yaml", "application/yaml") + SourceFactory.map("yml", "application/yaml") + SourceFactory.register("application/yaml", YamlFileSource) + SourceFactory.register("text/yaml", YamlFileSource) + SourceFactory.register("text/x-yaml", YamlFileSource) + SourceFactory.register("application/x-yaml", YamlFileSource) +except ImportError: + log.warning("Python YAML module not available -- disabling YAML support!") diff --git a/tests/subyt/test_yaml_source.py b/tests/subyt/test_yaml_source.py new file mode 100644 index 00000000..5a6e570d --- /dev/null +++ b/tests/subyt/test_yaml_source.py @@ -0,0 +1,114 @@ +import tempfile +from pathlib import Path + +import pytest + +from sema.subyt.api import Source +from sema.subyt.sources import SourceFactory, YamlFileSource + +YAML_LIST_CONTENT = """ +- name: Alice + age: 30 +- name: Bob + age: 25 +""" + +YAML_MULTI_DOC_CONTENT = """ +--- +name: Alice +age: 30 +--- +name: Bob +age: 25 +""" + +YAML_WRAPPED_CONTENT = """ +people: + - name: Alice + age: 30 + - name: Bob + age: 25 +""" + +YAML_SINGLE_RECORD = """ +name: Alice +age: 30 +""" + + +@pytest.mark.parametrize( + "content,expected_len", + [ + (YAML_LIST_CONTENT, 2), + (YAML_MULTI_DOC_CONTENT, 2), + (YAML_WRAPPED_CONTENT, 2), + (YAML_SINGLE_RECORD, 1), + ], +) +def test_yaml_file_source_contents(content: str, expected_len: int) -> None: + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + f.write(content) + temp_path = Path(f.name) + + try: + source: Source = SourceFactory.make_source(str(temp_path)) + assert isinstance(source, YamlFileSource) + with source as items: + data = list(items) + assert len(data) == expected_len + assert data[0]["name"] == "Alice" + assert data[0]["age"] == 30 + finally: + temp_path.unlink() + + +def test_yaml_yml_extension_and_mime() -> None: + with tempfile.NamedTemporaryFile("w", suffix=".yml", delete=False) as f: + f.write(YAML_LIST_CONTENT) + temp_path = Path(f.name) + + try: + # Check .yml detection + source_yml = SourceFactory.make_source(str(temp_path)) + assert isinstance(source_yml, YamlFileSource) + + # Check explicit config with mime + source_mime = SourceFactory.make_source( + {"path": str(temp_path), "mime": "text/yaml"} + ) + assert isinstance(source_mime, YamlFileSource) + + # Check explicit config with ext + source_ext = SourceFactory.make_source( + {"path": str(temp_path), "ext": "yaml"} + ) + assert isinstance(source_ext, YamlFileSource) + finally: + temp_path.unlink() + + +def test_yaml_unique_filter() -> None: + content = """ +- id: 1 + name: Alice +- id: 1 + name: Alice Duplicate +- id: 2 + name: Bob +""" + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + f.write(content) + temp_path = Path(f.name) + + try: + source = SourceFactory.make_source( + str(temp_path), unique_pattern="{id}" + ) + with source as items: + records = list(items) + assert len(records) == 2 + assert records[0]["name"] == "Alice" + assert records[1]["name"] == "Bob" + finally: + temp_path.unlink() + From 6f3164df64f24d7a300a226d6e8a5bc2dee6696c Mon Sep 17 00:00:00 2001 From: laurianvm Date: Mon, 9 Nov 2026 08:13:10 +0100 Subject: [PATCH 2/2] clean up MIME type handling (#215) --- sema/subyt/sources.py | 14 ++++---------- tests/subyt/test_yaml_source.py | 27 +++++++++++++++------------ 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/sema/subyt/sources.py b/sema/subyt/sources.py index 25ec36af..5ef4bd06 100644 --- a/sema/subyt/sources.py +++ b/sema/subyt/sources.py @@ -38,11 +38,7 @@ class SourceFactory: def __init__(self): self._register = dict() - self._ext_mime_map = { - "csv": "text/csv", - "yaml": "application/yaml", - "yml": "application/yaml", - } + self._ext_mime_map = {"csv": "text/csv"} @property def ext_2_mime(self): @@ -626,11 +622,9 @@ def __exit__(self, *exc) -> None: def __repr__(self) -> str: return f"YamlFileSource('{self._yaml!s}')" - SourceFactory.map("yaml", "application/yaml") - SourceFactory.map("yml", "application/yaml") + # Official IANA media type for YAML is application/yaml (RFC 9512) + mimetypes.add_type("application/yaml", ".yaml") + mimetypes.add_type("application/yaml", ".yml") SourceFactory.register("application/yaml", YamlFileSource) - SourceFactory.register("text/yaml", YamlFileSource) - SourceFactory.register("text/x-yaml", YamlFileSource) - SourceFactory.register("application/x-yaml", YamlFileSource) except ImportError: log.warning("Python YAML module not available -- disabling YAML support!") diff --git a/tests/subyt/test_yaml_source.py b/tests/subyt/test_yaml_source.py index 5a6e570d..3f4977a4 100644 --- a/tests/subyt/test_yaml_source.py +++ b/tests/subyt/test_yaml_source.py @@ -65,26 +65,29 @@ def test_yaml_file_source_contents(content: str, expected_len: int) -> None: def test_yaml_yml_extension_and_mime() -> None: with tempfile.NamedTemporaryFile("w", suffix=".yml", delete=False) as f: f.write(YAML_LIST_CONTENT) - temp_path = Path(f.name) + temp_yml = Path(f.name) + + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + f.write(YAML_LIST_CONTENT) + temp_yaml = Path(f.name) try: - # Check .yml detection - source_yml = SourceFactory.make_source(str(temp_path)) + # Check .yml detection via mimetypes + source_yml = SourceFactory.make_source(str(temp_yml)) assert isinstance(source_yml, YamlFileSource) - # Check explicit config with mime + # Check .yaml detection via mimetypes + source_yaml = SourceFactory.make_source(str(temp_yaml)) + assert isinstance(source_yaml, YamlFileSource) + + # Check explicit config with official IANA mime type (application/yaml) source_mime = SourceFactory.make_source( - {"path": str(temp_path), "mime": "text/yaml"} + {"path": str(temp_yaml), "mime": "application/yaml"} ) assert isinstance(source_mime, YamlFileSource) - - # Check explicit config with ext - source_ext = SourceFactory.make_source( - {"path": str(temp_path), "ext": "yaml"} - ) - assert isinstance(source_ext, YamlFileSource) finally: - temp_path.unlink() + temp_yml.unlink() + temp_yaml.unlink() def test_yaml_unique_filter() -> None: