DRAFT: feat: #82: creating a workflow with signature load testing

.env-example

@@ -1,9 +1,9 @@
-C2PIE_PRIVATE_KEY_FILE=tests/credentials/private-key.pem
-C2PIE_CERTIFICATE_CHAIN_FILE=tests/credentials/certificate-chain.pub
+C2PIE_PRIVATE_KEY_FILE=tests/fixtures/credentials/private-key.pem
+C2PIE_CERTIFICATE_CHAIN_FILE=tests/fixtures/credentials/certificate-chain.pub
 C2PIE_TSA_URL=http://timestamp.digicert.com
 C2PIE_TSA_REQUIRED=true
 C2PIE_TSA_LOG_DIR=/var/log/c2pie/tsa
 
 # Paths to test files
-TEST_PDF_PATH=tests/test_files/test_doc.pdf
-TEST_IMAGE_PATH=tests/test_files/test_image.jpg
+TEST_PDF_PATH=tests/fixtures/test_doc.pdf
+TEST_IMAGE_PATH=tests/fixtures/test_image.jpg

.gitignore

@@ -5,7 +5,7 @@ build
 dist
 **/signed_*
 # This file is needed for unit tests
-!tests/test_files/signed_signed_test_doc.pdf
+!tests/fixtures/signed_signed_test_doc.pdf
 .vscode/launch.json
 .env
 **/.ipynb_checkpoints

CONTRIBUTING.md

@@ -37,7 +37,7 @@ To contribute to the c2pie package development, you can use one of the following
 
 3. To run any Python command related to the project's dependencies, remember to add `poetry run` in front of the command. For example:
     ```bash
-    poetry run c2pie sign --input_file tests/test_files/test_doc.pdf
+    poetry run c2pie sign --input_file tests/fixtures/test_doc.pdf
 
     poetry run ruff check
     ```
@@ -49,7 +49,7 @@ To contribute to the c2pie package development, you can use one of the following
 
 To run test applications, you need to fill out `TEST_PDF_PATH` and/or `TEST_IMAGE_PATH` in values in *.env*. Test scripts use these filepaths as input files for signing.
 
-Also make sure that you have test certificate chain and public key in `tests/credentials`. They should be there by default if you've cloned the repository. If needed, you can change their filepaths in *.env* as well.
+Also make sure that you have test certificate chain and public key in `tests/fixtures/credentials`. They should be there by default if you've cloned the repository. If needed, you can change their filepaths in *.env* as well.
 
 
 You can test the signing workflow with the following VS Code tasks:

README.md

@@ -9,7 +9,7 @@
 [![Linting](https://github.com/TourmalineCore/c2pie/actions/workflows/lint-on-pull-request.yml/badge.svg?branch=develop)](https://github.com/TourmalineCore/c2pie/actions/workflows/lint-on-pull-request.yml)
 [![c2pa](https://img.shields.io/badge/c2pa-v1.4-seagreen.svg)](https://c2pa.org/)
 [![coverage](https://img.shields.io/badge/e2e_coverage-71.13%25-yellow)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
-[![coverage](https://img.shields.io/badge/units_coverage-79.65%25-yellow)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
+[![coverage](https://img.shields.io/badge/units_coverage-91.80%25-forestgreen)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
 [![coverage](https://img.shields.io/badge/full_coverage-91.80%25-forestgreen)](https://github.com/TourmalineCore/c2pie/actions/workflows/calculate-tests-coverage-on-pull-request.yml)
 [![latest](https://img.shields.io/pypi/v/c2pie?label=latest&colorB=fc8021)](https://pypi.org/project/c2pie/)
 
@@ -99,7 +99,7 @@ export C2PIE_CERTIFICATE_CHAIN_FILE=./certificate_chain.pem
 pip install c2pie 
 
 # Download test image from this repo
-wget https://raw.githubusercontent.com/TourmalineCore/c2pie/refs/heads/master/example_app/test_files/test_image.jpg
+wget https://raw.githubusercontent.com/TourmalineCore/c2pie/refs/heads/master/example_app/fixtures/test_image.jpg
 
 # Sign downloaded image
 c2pie sign --input_file ./test_image.jpg
@@ -126,7 +126,7 @@ After being copied to host machine, signed files can then be validated using eit
 
 1) Python environment. Currently supported Python versions: 3.10.0 - 3.14.5. Make sure to [create and activate virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/) to avoid installing packages globally and any errors caused by that.
 
-2) Private key and certificate chain pair. The repo contains pre-generated mock credentials in `tests/credentials`. You can either download and use them for a quick start or go to [Certificates](#-certificates) for instructions on how to generate a similar key-certificate pair.
+2) Private key and certificate chain pair. The repo contains pre-generated mock credentials in `tests/fixtures/credentials`. You can either download and use them for a quick start or go to [Certificates](#-certificates) for instructions on how to generate a similar key-certificate pair.
 
 
 3) Key and certificate chain filepaths exported into the current environment with (pay attention to filenames):
@@ -158,8 +158,8 @@ c2pie sign --input_file path/to/input_file --output_file path/to/output_file
 
 If the file has been successfully signed, you'll see a message like this: 
 ```bash
-Successfully signed the file tests/test_files/test_doc.pdf!
-The result was saved to tests/test_files/signed_test_doc.pdf.
+Successfully signed the file tests/fixtures/test_doc.pdf!
+The result was saved to tests/fixtures/signed_test_doc.pdf.
 ```
 
 #### Code
@@ -184,8 +184,8 @@ sign_file(input_path=input_file_path, output_path=output_file_path)
 
 If the file has been successfully signed, you'll see a message like this: 
 ```bash
-Successfully signed the file tests/test_files/test_doc.pdf!
-The result was saved to tests/test_files/signed_test_doc.pdf.
+Successfully signed the file tests/fixtures/test_doc.pdf!
+The result was saved to tests/fixtures/signed_test_doc.pdf.
 ```
 
 <br>
@@ -219,11 +219,11 @@ Follow the steps:
     docker compose up c2pie-test-signing-pdf
     ```
 
-    After running either of these commands, you'll see a resulting signed file appear in `example_app/test_files` directory with a `signed-` prefix and a corresponding message with c2patool validation results in your terminal like this:
+    After running either of these commands, you'll see a resulting signed file appear in `example_app/fixtures` directory with a `signed-` prefix and a corresponding message with c2patool validation results in your terminal like this:
 
     ```bash
-    Successfully signed the file test_files/test_image.jpg!
-    The result was saved to test_files/signed_test_image.jpg. 
+    Successfully signed the file fixtures/test_image.jpg!
+    The result was saved to fixtures/signed_test_image.jpg. 
     c2patool_validation_results:
     {
         "active_manifest": "urn:uuid:f0ce8560b76342d1bb3085cfbe6cc5e9",
@@ -371,7 +371,7 @@ cargo install c2patool
 
 # 🥧 Certificates
 
-Example certificate chain and key file are located in `tests/credentials`. 
+Example certificate chain and key file are located in `tests/fixtures/credentials`. 
 
 >[!WARNING]
 >This repository's credentials are suitable for development only! 
@@ -442,7 +442,7 @@ For detailed information on signing and certificates please explore the [corresp
 
 ## Workflow of test applications
 
-1) Load a sample asset (`tests/test_files/..`);
+1) Load a sample asset (`tests/fixtures/..`);
 
 2) Build a manifest with `c2pie_GenerateAssertion`, `c2pie_GenerateHashDataAssertion`, `c2pie_GenerateManifestStore`;
 

tests/c2pa/claim_signature_test.py

@@ -28,8 +28,8 @@ def test_create_claim_signature_with_empty_claim():
 
 
 def test_create_claim_signature_with_non_empty_claim():
-    key_filepath = "tests/credentials/private-key.pem"
-    cert_filepath = "tests/credentials/certificate-chain.pub"
+    key_filepath = "tests/fixtures/credentials/private-key.pem"
+    cert_filepath = "tests/fixtures/credentials/certificate-chain.pub"
 
     with open(key_filepath, "rb") as f:
         key = f.read()

tests/c2pa/claim_signature_with_tsa_test.py

@@ -10,8 +10,8 @@
 from c2pie.tsa.exceptions import TSARequiredError
 from c2pie.utils.assertion_schemas import C2PA_AssertionTypes
 
-_KEY_FILE = "tests/credentials/private-key.pem"
-_CERT_FILE = "tests/credentials/certificate-chain.pub"
+_KEY_FILE = "tests/fixtures/credentials/private-key.pem"
+_CERT_FILE = "tests/fixtures/credentials/certificate-chain.pub"
 _FAKE_TST_DER = b"\x30\x82\x01\x00"
 
 

tests/c2pa/e2e_test.py

@@ -1,98 +1,129 @@
 import json
-import os
 import shutil
 import subprocess
+import uuid
 from pathlib import Path
+from unittest.mock import patch
 
 import pytest
 
 from c2pie.signing import sign_file
-from c2pie.utils.content_types import C2PA_ContentTypes
-
-FIXTURES_DIR = Path(__file__).parent.parent / "test_files"
-
-test_files_by_extension = {
-    "pdf": [
-        "test_doc.pdf",
-        "test_doc2.pdf",
-    ],
-    "jpg": [
-        "test_image.jpg",
-    ],
-    "jpeg": [
-        "test_image.jpeg",
-    ],
-}
-
-
-def get_test_file_full_path(filename: str) -> Path:
-    path = FIXTURES_DIR / filename
-    if not path.exists():
-        raise FileNotFoundError(f"Fixture not found: {path}")
-    return path
 
+FILES_DIR = Path(__file__).parent.parent / "fixtures"
+
+test_cases = [
+    (
+        Path(FILES_DIR / "test_image.jpg"),
+        Path(FILES_DIR / "schemas/test_image_jpg.schema.json"),
+    ),
+    (
+        Path(FILES_DIR / "test_image.jpeg"),
+        Path(FILES_DIR / "schemas/test_image_jpeg.schema.json"),
+    ),
+    (
+        Path(FILES_DIR / "test_doc.pdf"),
+        Path(FILES_DIR / "schemas/test_doc_pdf.schema.json"),
+    ),
+    (
+        Path(FILES_DIR / "test_doc2.pdf"),
+        Path(FILES_DIR / "schemas/test_doc2_pdf.schema.json"),
+    ),
+]
+
+
+def _has_c2patool() -> bool:
+    return shutil.which("c2patool") is not None
 
-def copy_test_file(source_path: str, destination_path: Path) -> None:
-    source_full_path = get_test_file_full_path(source_path)
-    destination_path.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copyfile(source_full_path, destination_path)
 
+def _sign_file_with_mock_uuids(
+    data_file: Path,
+    output_file: Path,
+):
+    fixed_uuid = uuid.UUID("47affab1a5d24c75b991fbc030e02448")
 
-def has_c2patool() -> bool:
-    return shutil.which("c2patool") is not None
+    # If the call is not intercepted and a fixed value
+    # is not specified, a random value will be generated
+    with (
+        patch("c2pie.interface.uuid.uuid4", return_value=fixed_uuid),
+        patch("c2pie.c2pa.claim.uuid.uuid4", return_value=fixed_uuid),
+    ):
+        sign_file(
+            input_path=data_file,
+            output_path=output_file,
+        )
 
 
-def _c2pa_json_report(asset_path: str) -> dict:
+def _validate_using_c2patool_and_return_json_report(asset_path: Path) -> dict:
     """
     Return c2patool's JSON report. If parsing fails, raise with stdout/stderr for debugging.
     """
     c2patool_launch_command = ["c2patool", asset_path, "-d"]
 
-    cp2atool_result = subprocess.run(c2patool_launch_command, capture_output=True, text=True)
-    evaluation_result = cp2atool_result
-    if cp2atool_result.returncode == 0:
-        return json.loads(cp2atool_result.stdout or "{}")
+    c2patool_result = subprocess.run(
+        c2patool_launch_command,
+        # If set to False (by default), 'stdout' and 'stderr' outputs
+        # will not be available via '.stderr' and '.stdout', correspondingly.
+        capture_output=True,
+        # If set to False (by default), a byte stream will be
+        # returned instead of a string.
+        text=True,
+    )
+
+    if c2patool_result.returncode == 0:
+        return json.loads(c2patool_result.stdout)
+
     pytest.fail(
         "c2patool failed or did not output JSON.\n"
-        f"args={evaluation_result.args if evaluation_result else None}\n"
-        f"stdout={evaluation_result.stdout if evaluation_result else None}\n"
-        f"stderr={evaluation_result.stderr if evaluation_result else None}"
+        f"args={c2patool_result.args if c2patool_result else None}\n"
+        f"stdout={c2patool_result.stdout if c2patool_result else None}\n"
+        f"stderr={c2patool_result.stderr if c2patool_result else None}"
     )
 
 
-@pytest.mark.e2e
-def test_e2e_signing_with_c2patool_validation(tmp_path):
-    if not has_c2patool():
+@pytest.mark.parametrize(
+    "data_file,schema_file",
+    test_cases,
+    ids=lambda p: p.name,
+)
+def test_e2e_signing_with_c2patool_validation(
+    data_file: Path,
+    schema_file: Path,
+    tmp_path,
+):
+    if not _has_c2patool():
         pytest.skip("c2patool not available")
-    if not sign_file:
-        pytest.skip("sign_file function not available yet")
 
-    os.environ["C2PA_BACKEND"] = "tool"
+    output_file = tmp_path / f"out{data_file.suffix}"
+
+    _sign_file_with_mock_uuids(
+        data_file,
+        output_file,
+    )
+
+    report = _validate_using_c2patool_and_return_json_report(output_file)
+    manifests = report.get("manifests")
+    expected_schema = json.loads(schema_file.read_text())
 
-    for content_type in C2PA_ContentTypes:
-        input_file = tmp_path / f"in.{content_type.name}"
-        output_file = tmp_path / f"out.{content_type.name}"
+    assert manifests == expected_schema
 
-        for test_file in test_files_by_extension[content_type.name]:
-            copy_test_file(f"./{test_file}", input_file)
 
-            try:
-                sign_file(
-                    input_path=input_file,
-                    output_path=output_file,
-                )
-            except NotImplementedError:
-                pytest.xfail("sign_file function not implemented yet")
+# @pytest.mark.parametrize(
+#     "iteration",
+#     range(100),
+# )
+# def test_e2e_signature_stability(iteration, tmp_path):
+#     if not _has_c2patool():
+#         pytest.skip("c2patool not available")
 
-            data = _c2pa_json_report(str(output_file))
-            assert "manifests" in data or "manifest" in data
+#     data_file = Path(FILES_DIR / "test_image.jpg")
+#     output_file = tmp_path / f"out{data_file.suffix}"
 
-            manifests = data.get("manifests")
-            assert manifests, "no manifests in output"
+#     _sign_file_with_mock_uuids(
+#         data_file,
+#         output_file,
+#     )
 
-            if isinstance(manifests, dict):
-                manifests_list = list(manifests.values())
-            else:
-                manifests_list = manifests
+#     report = _validate_using_c2patool_and_return_json_report(output_file)
+#     validation_state = report.get("validation_state")
 
-            assert manifests_list, "empty manifests list after normalization"
+#     assert validation_state == "Valid"

tests/c2pa/interface_test.py

@@ -15,8 +15,8 @@
 JPEG_HEADER = b"\xff\xd8\xff"
 MEDIA_TYPE = "image/jpeg"
 
-KEY_FILEPATH = "tests/credentials/private-key.pem"
-CERT_FILEPATH = "tests/credentials/certificate-chain.pub"
+KEY_FILEPATH = "tests/fixtures/credentials/private-key.pem"
+CERT_FILEPATH = "tests/fixtures/credentials/certificate-chain.pub"
 
 
 def test_generate_assertion_has_correct_type():
@@ -129,7 +129,7 @@ def test_emplace_manifest_returns_bytes_with_jpeg_signature():
         key = f.read()
     with open(CERT_FILEPATH, "rb") as f:
         cert = f.read()
-    with open("tests/test_files/test_image.jpg", "rb") as f:
+    with open("tests/fixtures/test_image.jpg", "rb") as f:
         jpeg_bytes = f.read()
 
     assertions = [

tests/c2pa_parsing/jumbf_parsing_test.py

@@ -19,7 +19,7 @@ def _mock_get_label(box: Box) -> str | None:
 
 @pytest.fixture
 def signed_pdf_bytes() -> bytes:
-    path = Path(__file__).parent.parent / "test_files" / "signed_signed_test_doc.pdf"
+    path = Path(__file__).parent.parent / "fixtures" / "signed_signed_test_doc.pdf"
     return path.read_bytes()
 
 

tests/c2pa_parsing/manifest_store_extractor_test.py

@@ -38,13 +38,13 @@ def make_jpeg(*app11_segments: bytes) -> bytes:
 
 @pytest.fixture
 def signed_jpg_bytes() -> bytes:
-    path = Path(__file__).parent.parent / "test_files" / "big_c2pa_test_image.jpg"
+    path = Path(__file__).parent.parent / "fixtures" / "big_c2pa_test_image.jpg"
     return path.read_bytes()
 
 
 @pytest.fixture
 def signed_pdf_bytes() -> bytes:
-    path = Path(__file__).parent.parent / "test_files" / "signed_signed_test_doc.pdf"
+    path = Path(__file__).parent.parent / "fixtures" / "signed_signed_test_doc.pdf"
     return path.read_bytes()