Merge pull request #12 from hathitrust/ETT-241_json2yaml_format

Ett 241 json2yaml format

Author: liseli

README.md

@@ -53,7 +53,7 @@ poetry run ruff format src/ tests/
 # Type checking
 poetry run mypy src/
 
-Note: The file `py.typed`  has added to the package and specified in pyproject.toml to ensure mypy treats 
+Note: The file `py.typed`  has been added to the package and specified in pyproject.toml to ensure mypy treats 
 tkseal as a typed package and avoids "Skipping analyzing 'tkseal': found module but no type hints or library stubs" warnings.
 ```
 
@@ -63,8 +63,8 @@ tkseal as a typed package and avoids "Skipping analyzing 'tkseal': found module
 - ✅ `tkseal ready` - Check dependencies (WIP)
 - ✅ `tkseal diff PATH` - Show differences between plain_secrets.json and cluster
 - ✅ `tkseal pull PATH` - Extracting secrets from cluster to plain_secrets.json
-- 💻 `tkseal seal PATH` - Convert plain_secrets.json to sealed_secrets.json
-- 
+- ✅ `tkseal seal PATH` - Convert plain_secrets.json to sealed_secrets.json
+
 
 ## Logic documentation
 
@@ -103,13 +103,18 @@ tkseal as a typed package and avoids "Skipping analyzing 'tkseal': found module
 
   `tkseal pull environments/testing/`
 
-  This shows how "plain_secrets.json" would change based on what's in the Kubernetes cluster
+  This shows how "plain_secrets.json" would change based on what's in the Kubernetes cluster, and it will 
+  warn about forbidden secrets:
 
-  `⚠ Warning: Found forbidden secrets in namespace that cannot be pulled:
-    - default-token-abc (type: kubernetes.io/service-account-token)
-    - helm-release-v1 (type: helm.sh/release.v1)`
+  ```
+  These secrets are system-managed and will not be included in plain_secrets.json:                                                                                                                                                         
+  - oidc-saml-proxy-tls (type: kubernetes.io/tls)
+  This shows how "plain_secrets.json" would change based on what's in the Kubernetes cluster
+--- plain_secrets.json
++++ cluster
+  ```
 
-### ready Command
+## ready Command
 
   **Core Functionality**
 
@@ -156,7 +161,7 @@ which can then be safely stored in version control systems like Git.
   - Example usage: `printf "secret" | kubeseal --raw --namespace ns --name secret-name --context ctx`
 
 
-### diff command
+## diff command
 
 **Core Functionality**
 The diff command compares the local `plain_secrets.json` file in a specified Tanka environment directory with 
@@ -175,6 +180,7 @@ Usage Example
 **Show what would change in cluster**
 
 ```tkseal diff /path/to/tanka/environments/production```
+Use `tkseal diff /path/to/env --format yaml` to output plain secrets in YAML format.
 
 **If there are differences, shows a unified diff**
 
@@ -202,7 +208,7 @@ No differences
   Error: Path '/nonexistent/path' does not exist.
 ```
 
-### pull command
+## pull command
 
 **Core Functionality**
 The pull command extracts existing Kubernetes secrets from the cluster and writes them to a local plain_secrets.json 
@@ -215,8 +221,11 @@ The flow is:
 3. Prompt user for confirmation
 4. Write kube secrets to plain_secrets.json  
 
+### pull secrets (with confirmation)
+`tkseal pull /path/to/tanka/environment`
+Use `tkseal pull /path/to/env --format yaml` to output plain secrets in YAML format.
 
-### seal command
+## seal command
 
 **Core Functionality**
 The seal command reads the local plain_secrets.json file in a specified Tanka environment directory,
@@ -232,11 +241,21 @@ The flow is:
 3. Seal each secret using kubeseal
 4. Write sealed secrets to sealed_secrets.json
 
-# Seal secrets (with confirmation)
+### Seal secrets (with confirmation)
 `tkseal seal /path/to/tanka/environment`
+Use `tkseal seal /path/to/env --format yaml` to output sealed secrets in YAML format.
 
 The command will:
 1. Show yellow warning about cluster changes
 2. Display diff of what would change
 3. Ask for confirmation
 4. Seal secrets to sealed_secrets.json
+
+# Example of errors running tkseal commands
+
+This error means that you probably are not in a Tanka environment directory or the directory structure is incorrect.
+Remember that Tanka expects a specific directory structure with `main.jsonnet` file in the environment's base directory.
+
+```Error: Failed to initialize Tanka environment: Command failed with exit code 1: Error: Unable to identify the environments base directory.
+Tried to find 'main.jsonnet' in the parent directories.
+Please refer to https://tanka.dev/directory-structure for more information```

src/tkseal/cli.py

@@ -49,22 +49,28 @@ def ready() -> None:
 
 @cli.command()
 @click.argument("path", type=click.Path(exists=True))
-def diff(path: str) -> None:
-    """Show differences between plain_secrets.json and cluster secrets.
+@click.option(
+    "--format",
+    type=click.Choice(["json", "yaml"], case_sensitive=False),
+    default="json",
+    help="Output format for secret files (default: json)",
+)
+def diff(path: str, format: str) -> None:
+    """Show differences between plain_secrets file and cluster secrets.
 
     PATH: Path to Tanka environment directory or .jsonnet file
 
-    This shows what would change in the cluster based on plain_secrets.json
+    This shows what would change in the cluster based on plain_secrets file
     """
     try:
-        # Create SecretState from path
-        secret_state = SecretState.from_path(path)
+        # Create SecretState from path with specified format
+        secret_state = SecretState.from_path(path, format=format)
 
         # Create a Diff instance and run comparison
         diff_obj = Diff(secret_state)
         result = diff_obj.plain()
 
-        # Display results
+        # Display results - Always in JSON format, it is independent of the sealed_secrets format (YAML/JSON)
         if result.has_differences:
             click.echo(result.diff_output)
         else:
@@ -77,17 +83,23 @@ def diff(path: str) -> None:
 
 @cli.command()
 @click.argument("path", type=click.Path(exists=True))
-def pull(path: str) -> None:
-    """Pull secrets from the cluster to plain_secrets.json.
+@click.option(
+    "--format",
+    type=click.Choice(["json", "yaml"], case_sensitive=False),
+    default="json",
+    help="Output format for secret files (default: json)",
+)
+def pull(path: str, format: str) -> None:
+    """Pull secrets from the cluster to plain_secrets file.
 
     PATH: Path to Tanka environment directory or .jsonnet file
 
     This extracts unencrypted secrets from the Kubernetes cluster
-    and saves them to plain_secrets.json in the environment directory.
+    and saves them to plain_secrets.json or plain_secrets.yaml in the environment directory.
     """
     try:
-        # Create SecretState from path
-        secret_state = SecretState.from_path(path)
+        # Create SecretState from path with specified format
+        secret_state = SecretState.from_path(path, format=format)
 
         # Create Pull instance and show differences
         pull_obj = Pull(secret_state)
@@ -104,18 +116,24 @@ def pull(path: str) -> None:
                 click.secho(f"  - {secret.name} (type: {secret.type})", fg="yellow")
 
         # Show informational message
+        plain_secrets_file = f"plain_secrets.{format}"
         click.secho(
-            'This shows how "plain_secrets.json" would change based on what\'s in the Kubernetes cluster',
+            f'This shows how "{plain_secrets_file}" would change based on what\'s in the Kubernetes cluster',
             fg="yellow",
         )
+
+        # Create Pull instance and show differences
+        pull_obj = Pull(secret_state)
+        result = pull_obj.run()
+
         # Display diff results
         if result.has_differences:
             click.echo(result.diff_output)
 
             # Confirm before writing
             if click.confirm("Are you sure?"):
                 pull_obj.write()
-                click.echo("Successfully pulled secrets to plain_secrets.json")
+                click.echo(f"Successfully pulled secrets to {plain_secrets_file}")
         else:
             click.echo("No differences")
 
@@ -126,21 +144,29 @@ def pull(path: str) -> None:
 
 @cli.command()
 @click.argument("path", type=click.Path(exists=True))
-def seal(path: str) -> None:
-    """Seal plain_secrets.json to sealed_secrets.json.
+@click.option(
+    "--format",
+    type=click.Choice(["json", "yaml"], case_sensitive=False),
+    default="json",
+    help="Output format for secret files (default: json)",
+)
+def seal(path: str, format: str) -> None:
+    """Seal plain_secrets file to sealed_secrets file.
 
     PATH: Path to Tanka environment directory or .jsonnet file
 
-    Takes secrets from plain_secrets.json, encrypts them using kubeseal,
-    and saves the resulting SealedSecret resources to sealed_secrets.json.
+    Takes secrets from plain_secrets file, encrypts them using kubeseal,
+    and saves the resulting SealedSecret resources to sealed_secrets file.
     """
     try:
-        # Create SecretState from path
-        secret_state = SecretState.from_path(path)
+        # Create SecretState from path with specified format
+        secret_state = SecretState.from_path(path, format=format)
+
+        sealed_secrets_file = f"sealed_secrets.{format}"
 
         # Show informational message
         # click.secho(
-        #    'This shows what would change in the cluster based on "plain_secrets.json"',
+        #    f'This shows what would change in the cluster based on "plain_secrets.{format}"',
         #    fg="yellow",
         # )
 
@@ -156,7 +182,7 @@ def seal(path: str) -> None:
         if click.confirm("Are you sure?"):
             seal_obj = Seal(secret_state)
             seal_obj.run()
-            click.echo("Successfully sealed secrets to sealed_secrets.json")
+            click.echo(f"Successfully sealed secrets to {sealed_secrets_file}")
         # else:
         #    click.echo("No differences")
 

src/tkseal/configuration.py

@@ -8,11 +8,11 @@
 and defined in the configuration module.
 """
 
-# File name for plain (unencrypted) secrets JSON file
-PLAIN_SECRETS_FILE = "plain_secrets.json"
+# File name for plain (unencrypted) secrets JSON/YAML file
+PLAIN_SECRETS_FILE = "plain_secrets"
 
 # File name for sealed (encrypted) secrets JSON file
-SEALED_SECRETS_FILE = "sealed_secrets.json"
+SEALED_SECRETS_FILE = "sealed_secrets"
 
 # Allowed secret types that tkseal can manage
 MANAGED_SECRET_TYPES = {

src/tkseal/diff.py

@@ -5,6 +5,7 @@
 
 from tkseal.configuration import PLAIN_SECRETS_FILE
 from tkseal.secret_state import SecretState
+from tkseal.tkseal_utils import normalize_to_json
 
 
 @dataclass
@@ -27,11 +28,18 @@ def plain(self) -> DiffResult:
         kube_secrets = self.secret_state.kube_secrets()
         plain_secrets = self.secret_state.plain_secrets()
 
+        # Normalize both to JSON for comparison
+        kube_secrets_normalized = normalize_to_json(kube_secrets, "json")
+        plain_secrets_normalized = normalize_to_json(
+            plain_secrets, self.secret_state.format
+        )
+
+        #
         return self._generate_diff(
-            from_text=kube_secrets,
-            to_text=plain_secrets,
+            from_text=kube_secrets_normalized,
+            to_text=plain_secrets_normalized,
             from_label="cluster",
-            to_label=PLAIN_SECRETS_FILE,
+            to_label=f"{PLAIN_SECRETS_FILE}.{self.secret_state.format}",
         )
 
     def pull(self) -> DiffResult:
@@ -41,10 +49,16 @@ def pull(self) -> DiffResult:
         plain_secrets = self.secret_state.plain_secrets()
         kube_secrets = self.secret_state.kube_secrets()
 
+        # Normalize both to JSON for comparison
+        plain_secrets_normalized = normalize_to_json(
+            plain_secrets, self.secret_state.format
+        )
+        kube_secrets_normalized = normalize_to_json(kube_secrets, "json")
+
         return self._generate_diff(
-            from_text=plain_secrets,
-            to_text=kube_secrets,
-            from_label=PLAIN_SECRETS_FILE,
+            from_text=plain_secrets_normalized,
+            to_text=kube_secrets_normalized,
+            from_label=f"{PLAIN_SECRETS_FILE}.{self.secret_state.format}",
             to_label="cluster",
         )
 

src/tkseal/pull.py

@@ -1,12 +1,13 @@
 from tkseal.diff import Diff, DiffResult
 from tkseal.secret_state import SecretState
+from tkseal.serializers import get_serializer
 
 
 class Pull:
     """Handles pulling secrets from Kubernetes cluster to local files.
 
     This class coordinates the process of retrieving secrets from a Kubernetes
-    cluster and saving them to the local plain_secrets.json file.
+    cluster and saving them to the local plain_secrets.json or plain_secrets.yaml file.
     """
 
     def __init__(self, secret_state: SecretState):
@@ -20,8 +21,8 @@ def __init__(self, secret_state: SecretState):
     def run(self) -> DiffResult:
         """Show differences between local and cluster secrets.
 
-        This method displays what would change in the local plain_secrets.json
-        file if secrets were pulled from the cluster.
+        This method displays what would change in the local plain_secrets file
+        if secrets were pulled from the cluster.
 
         Returns:
             DiffResult: Result containing information about the difference
@@ -33,15 +34,27 @@ def run(self) -> DiffResult:
         return diff.pull()
 
     def write(self) -> None:
-        """Write cluster secrets to the plain_secrets.json file.
+        """Write cluster secrets to the plain_secrets file in the specified format.
 
-        This method retrieves secrets from the Kubernetes cluster and writes
-        them to the local plain_secrets.json file, overwriting any existing content.
+        This method retrieves secrets from the Kubernetes cluster (as JSON) and writes
+        them to the local plain_secrets file, converting to YAML if needed.
 
         Raises:
             TKSealError: If there's an error retrieving secrets from cluster
             PermissionError: If there's an error writing to the file
             OSError: If there's an I/O error writing to the file
         """
-        kube_secrets = self.secret_state.kube_secrets()
-        self.secret_state.plain_secrets_file_path.write_text(kube_secrets)
+        # Get secrets from cluster as JSON string
+        kube_secrets_json = self.secret_state.kube_secrets()
+
+        # Convert to the desired format if needed
+        if self.secret_state.format == "yaml":
+            # Deserialize JSON and re-serialize to YAML
+            secret_serializer = get_serializer(self.secret_state.format)
+            secrets_data = secret_serializer.deserialize_secrets(kube_secrets_json)
+            output = secret_serializer.serialize_secrets(secrets_data)
+        else:
+            # Keep as JSON (no conversion needed)
+            output = kube_secrets_json
+
+        self.secret_state.plain_secrets_file_path.write_text(output)