Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions sema/subyt/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -568,3 +568,63 @@ def __repr__(self) -> str:
SourceFactory.register("application/xml", XMLFileSource)
except ImportError:
log.warning("Python XML module not available -- disabling XML support!")


try:
import yaml

class YamlFileSource(Source):
"""
Source producing iterator over data-set coming from YAML on file
"""

def __init__(self, yaml_file_path: Path, config: dict = {}) -> None:
super().__init__()
assert_readable(yaml_file_path)
self._yaml: Path = yaml_file_path.absolute()
self._init_source(self._yaml)

def __enter__(self) -> object:
with open(self._yaml, mode="r", encoding="utf-8-sig") as yamlfile:
docs = [
doc
for doc in yaml.safe_load_all(yamlfile)
if doc is not None
]

if not docs:
data = []
elif len(docs) > 1:
data = docs
else:
data = docs[0]
# unwrap nested structures
while isinstance(data, dict) and len(data.keys()) == 1:
child = list(data.values())[0]
if isinstance(child, (dict, list)):
if (
isinstance(child, dict)
and len(child.keys()) == 0
):
data = [data]
break
data = child
else:
break
if not isinstance(data, list):
data = [data]

return iter(data)

def __exit__(self, *exc) -> None:
pass

def __repr__(self) -> str:
return f"YamlFileSource('{self._yaml!s}')"

# Official IANA media type for YAML is application/yaml (RFC 9512)
mimetypes.add_type("application/yaml", ".yaml")
mimetypes.add_type("application/yaml", ".yml")
SourceFactory.register("application/yaml", YamlFileSource)
except ImportError:
log.warning("Python YAML module not available -- disabling YAML support!")
117 changes: 117 additions & 0 deletions tests/subyt/test_yaml_source.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import tempfile
from pathlib import Path

import pytest

from sema.subyt.api import Source
from sema.subyt.sources import SourceFactory, YamlFileSource

YAML_LIST_CONTENT = """
- name: Alice
age: 30
- name: Bob
age: 25
"""

YAML_MULTI_DOC_CONTENT = """
---
name: Alice
age: 30
---
name: Bob
age: 25
"""

YAML_WRAPPED_CONTENT = """
people:
- name: Alice
age: 30
- name: Bob
age: 25
"""

YAML_SINGLE_RECORD = """
name: Alice
age: 30
"""


@pytest.mark.parametrize(
"content,expected_len",
[
(YAML_LIST_CONTENT, 2),
(YAML_MULTI_DOC_CONTENT, 2),
(YAML_WRAPPED_CONTENT, 2),
(YAML_SINGLE_RECORD, 1),
],
)
def test_yaml_file_source_contents(content: str, expected_len: int) -> None:
with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f:
f.write(content)
temp_path = Path(f.name)

try:
source: Source = SourceFactory.make_source(str(temp_path))
assert isinstance(source, YamlFileSource)
with source as items:
data = list(items)
assert len(data) == expected_len
assert data[0]["name"] == "Alice"
assert data[0]["age"] == 30
finally:
temp_path.unlink()


def test_yaml_yml_extension_and_mime() -> None:
with tempfile.NamedTemporaryFile("w", suffix=".yml", delete=False) as f:
f.write(YAML_LIST_CONTENT)
temp_yml = Path(f.name)

with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f:
f.write(YAML_LIST_CONTENT)
temp_yaml = Path(f.name)

try:
# Check .yml detection via mimetypes
source_yml = SourceFactory.make_source(str(temp_yml))
assert isinstance(source_yml, YamlFileSource)

# Check .yaml detection via mimetypes
source_yaml = SourceFactory.make_source(str(temp_yaml))
assert isinstance(source_yaml, YamlFileSource)

# Check explicit config with official IANA mime type (application/yaml)
source_mime = SourceFactory.make_source(
{"path": str(temp_yaml), "mime": "application/yaml"}
)
assert isinstance(source_mime, YamlFileSource)
finally:
temp_yml.unlink()
temp_yaml.unlink()


def test_yaml_unique_filter() -> None:
content = """
- id: 1
name: Alice
- id: 1
name: Alice Duplicate
- id: 2
name: Bob
"""
with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f:
f.write(content)
temp_path = Path(f.name)

try:
source = SourceFactory.make_source(
str(temp_path), unique_pattern="{id}"
)
with source as items:
records = list(items)
assert len(records) == 2
assert records[0]["name"] == "Alice"
assert records[1]["name"] == "Bob"
finally:
temp_path.unlink()

Loading