From fc3888f2082211a2d9b44efa957d444c4ef5153d Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Fri, 12 Sep 2025 14:44:08 -0400 Subject: [PATCH 1/3] add bin/fix_acceptance_tests_yml.py --- bin/fix_acceptance_tests_yml.py | 78 +++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 bin/fix_acceptance_tests_yml.py diff --git a/bin/fix_acceptance_tests_yml.py b/bin/fix_acceptance_tests_yml.py new file mode 100755 index 000000000..8825ef7a4 --- /dev/null +++ b/bin/fix_acceptance_tests_yml.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +""" +Script to transform YAML acceptance tests structure. + +This script transforms YAML files from the old format: + tests: + spec: [...] + connection: [...] + +To the new format: + acceptance_tests: + spec: + tests: [...] + connection: + tests: [...] +""" + +import yaml +import sys +from pathlib import Path +from typing import Dict, Any + + +class AlreadyUpdatedError(Exception): + """Exception raised when the YAML file has already been updated.""" + pass + + +def transform(file_path: Path) -> None: + with open(file_path, 'r') as f: + data = yaml.safe_load(f) + + if 'acceptance_tests' in data: + raise AlreadyUpdatedError() + + if 'tests' not in data: + raise ValueError(f"No 'tests' key found in {file_path}, skipping transformation") + + # Extract the tests data + tests_data = data.pop('tests') + + if not isinstance(tests_data, dict): + raise ValueError(f"Error: 'tests' key in {file_path} is not a dictionary") + + # Create the new acceptance_tests structure + data['acceptance_tests'] = {} + + # Transform each test type + for test_type, test_content in tests_data.items(): + data['acceptance_tests'][test_type] = {'tests': test_content} + + # Write back to file with preserved formatting + with open(file_path, 'w') as f: + yaml.dump(data, f, default_flow_style=False, sort_keys=False, indent=2) + + print(f"Successfully transformed {file_path}") + + +def main(): + if len(sys.argv) != 2: + print("Usage: python fix_acceptance_tests_yml.py ") + sys.exit(1) + + repo_path = Path(sys.argv[1]) + + for file_path in repo_path.glob('airbyte-integrations/connectors/source-*/acceptance-test-config.yml'): + try: + transform(file_path) + except AlreadyUpdatedError: + print(f"File {file_path} has already been updated, skipping transformation") + except yaml.YAMLError as e: + print(f"Error parsing YAML file {file_path}: {e}") + except Exception as e: + print(f"Error transforming {file_path}: {e}") + + +if __name__ == "__main__": + main() From e79545c0e9e808ce012e8900874edc8046f9cb4a Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Fri, 12 Sep 2025 14:53:27 -0400 Subject: [PATCH 2/3] fix indentation --- bin/fix_acceptance_tests_yml.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/bin/fix_acceptance_tests_yml.py b/bin/fix_acceptance_tests_yml.py index 8825ef7a4..41797c216 100755 --- a/bin/fix_acceptance_tests_yml.py +++ b/bin/fix_acceptance_tests_yml.py @@ -21,6 +21,27 @@ from typing import Dict, Any +class FixingListIndentationDumper(yaml.Dumper): + """ + The original indentation for list generates formatting issues on our side. So this dumper goes from: + ``` + tests: + - config_path: secrets/config.json + status: succeed + ``` + + ...to: + ``` + tests: + - config_path: secrets/config.json + status: succeed + ``` + + """ + def increase_indent(self, flow=False, indentless=False): + return super(FixingListIndentationDumper, self).increase_indent(flow, False) + + class AlreadyUpdatedError(Exception): """Exception raised when the YAML file has already been updated.""" pass @@ -51,7 +72,7 @@ def transform(file_path: Path) -> None: # Write back to file with preserved formatting with open(file_path, 'w') as f: - yaml.dump(data, f, default_flow_style=False, sort_keys=False, indent=2) + yaml.dump(data, f, Dumper=FixingListIndentationDumper, default_flow_style=False, sort_keys=False) print(f"Successfully transformed {file_path}") From c40a57e2799a901842ef54d271eab238fb06529a Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Fri, 12 Sep 2025 19:00:15 +0000 Subject: [PATCH 3/3] Auto-fix lint and format issues --- bin/fix_acceptance_tests_yml.py | 47 +++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/bin/fix_acceptance_tests_yml.py b/bin/fix_acceptance_tests_yml.py index 41797c216..280a57493 100755 --- a/bin/fix_acceptance_tests_yml.py +++ b/bin/fix_acceptance_tests_yml.py @@ -6,7 +6,7 @@ tests: spec: [...] connection: [...] - + To the new format: acceptance_tests: spec: @@ -15,10 +15,11 @@ tests: [...] """ -import yaml import sys from pathlib import Path -from typing import Dict, Any +from typing import Any, Dict + +import yaml class FixingListIndentationDumper(yaml.Dumper): @@ -38,42 +39,46 @@ class FixingListIndentationDumper(yaml.Dumper): ``` """ + def increase_indent(self, flow=False, indentless=False): return super(FixingListIndentationDumper, self).increase_indent(flow, False) class AlreadyUpdatedError(Exception): """Exception raised when the YAML file has already been updated.""" + pass def transform(file_path: Path) -> None: - with open(file_path, 'r') as f: + with open(file_path, "r") as f: data = yaml.safe_load(f) - - if 'acceptance_tests' in data: + + if "acceptance_tests" in data: raise AlreadyUpdatedError() - if 'tests' not in data: + if "tests" not in data: raise ValueError(f"No 'tests' key found in {file_path}, skipping transformation") - + # Extract the tests data - tests_data = data.pop('tests') - + tests_data = data.pop("tests") + if not isinstance(tests_data, dict): raise ValueError(f"Error: 'tests' key in {file_path} is not a dictionary") - + # Create the new acceptance_tests structure - data['acceptance_tests'] = {} - + data["acceptance_tests"] = {} + # Transform each test type for test_type, test_content in tests_data.items(): - data['acceptance_tests'][test_type] = {'tests': test_content} - + data["acceptance_tests"][test_type] = {"tests": test_content} + # Write back to file with preserved formatting - with open(file_path, 'w') as f: - yaml.dump(data, f, Dumper=FixingListIndentationDumper, default_flow_style=False, sort_keys=False) - + with open(file_path, "w") as f: + yaml.dump( + data, f, Dumper=FixingListIndentationDumper, default_flow_style=False, sort_keys=False + ) + print(f"Successfully transformed {file_path}") @@ -81,10 +86,12 @@ def main(): if len(sys.argv) != 2: print("Usage: python fix_acceptance_tests_yml.py ") sys.exit(1) - + repo_path = Path(sys.argv[1]) - for file_path in repo_path.glob('airbyte-integrations/connectors/source-*/acceptance-test-config.yml'): + for file_path in repo_path.glob( + "airbyte-integrations/connectors/source-*/acceptance-test-config.yml" + ): try: transform(file_path) except AlreadyUpdatedError: