feat(bom downloadAttachments): read attachment id from control file

Author: gernot-h

capycli/bom/download_attachments.py

@@ -19,6 +19,7 @@
 from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport, SbomWriter
 from capycli.common.print import print_red, print_text, print_yellow
 from capycli.common.script_support import ScriptSupport
+from capycli.common.json_support import load_json_file
 from capycli.main.result_codes import ResultCode
 
 LOG = capycli.get_logger(__name__)
@@ -29,7 +30,7 @@ class BomDownloadAttachments(capycli.common.script_base.ScriptBase):
     Download SW360 attachments as specified in the SBOM.
     """
 
-    def download_attachments(self, sbom: Bom, source_folder: str, bompath: str = None,
+    def download_attachments(self, sbom: Bom, control_components: list, source_folder: str, bompath: str = None,
                              attachment_types: Tuple[str] = ("COMPONENT_LICENSE_INFO_XML", "CLEARING_REPORT")) -> Bom:
 
         for component in sbom.components:
@@ -46,27 +47,24 @@ def download_attachments(self, sbom: Bom, source_folder: str, bompath: str = Non
                 if not found:
                     continue
 
-                attachment_id = ext_ref.comment.split(", sw360Id: ")
-                if len(attachment_id) != 2:
-                    print_red("    No sw360Id for attachment!")
-                    continue
-                attachment_id = attachment_id[1]
-
                 release_id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID)
                 if not release_id:
                     print_red("    No sw360Id for release!")
                     continue
-                print("   ", ext_ref.url, release_id, attachment_id)
                 filename = os.path.join(source_folder, ext_ref.url)
 
+                details = [e for e in control_components
+                           if e["Sw360Id"] == release_id and (
+                               e.get("CliFile", "") == ext_ref.url
+                               or e.get("ReportFile", "") == ext_ref.url)]
+                if len(details) != 1:
+                    print_red("    ERROR: Found", len(details), "entries for attachment",
+                              ext_ref.url, "of", item_name, "in control file!")
+                    continue
+                attachment_id = details[0]["Sw360AttachmentId"]
+
                 print_text("    Downloading file " + filename)
                 try:
-                    at_info = self.client.get_attachment(attachment_id)
-                    at_info = {k: v for k, v in at_info.items()
-                               if k.startswith("check")
-                               or k.startswith("created")}
-                    print(at_info)
-
                     self.client.download_release_attachment(filename, release_id, attachment_id)
                     ext_ref.url = filename
                     try:
@@ -104,6 +102,7 @@ def run(self, args):
             print("optional arguments:")
             print("    -h, --help            show this help message and exit")
             print("    -i INPUTFILE,         input SBOM to read from, e.g. created by \"project CreateBom\"")
+            print("    -ct CONTROLFILE,      control file to read from as created by \"project CreateBom\"")
             print("    -source SOURCE        source folder or additional source file")
             print("    -o OUTPUTFILE         output file to write to")
             print("    -v                    be verbose")
@@ -113,6 +112,10 @@ def run(self, args):
             print_red("No input file specified!")
             sys.exit(ResultCode.RESULT_COMMAND_ERROR)
 
+        if not args.controlfile:
+            print_red("No control file specified!")
+            sys.exit(ResultCode.RESULT_COMMAND_ERROR)
+
         if not os.path.isfile(args.inputfile):
             print_red("Input file not found!")
             sys.exit(ResultCode.RESULT_FILE_NOT_FOUND)
@@ -127,6 +130,16 @@ def run(self, args):
         if args.verbose:
             print_text(" " + str(len(bom.components)) + "components read from SBOM file")
 
+        print_text("Loading control file " + args.controlfile)
+        try:
+            control = load_json_file(args.controlfile)
+        except Exception as ex:
+            print_red("JSON error reading control file: " + repr(ex))
+            sys.exit(ResultCode.RESULT_ERROR_READING_BOM)
+        if "Components" not in control:
+            print_red("missing Components in control file")
+            sys.exit(ResultCode.RESULT_ERROR_READING_BOM)
+
         source_folder = "./"
         if args.source:
             source_folder = args.source
@@ -144,7 +157,7 @@ def run(self, args):
 
         print_text("Downloading source files to folder " + source_folder + " ...")
 
-        self.download_attachments(bom, source_folder, os.path.dirname(args.outputfile))
+        self.download_attachments(bom, control["Components"], source_folder, os.path.dirname(args.outputfile))
 
         if args.outputfile:
             print_text("Updating path information")

tests/fixtures/sbom_for_download-control.json

@@ -0,0 +1,24 @@
+{
+  "ProjectName": "CaPyCLI, 2.0.0-dev1",
+  "Components": [
+    {
+      "ComponentName": "certifi 2022.12.7",
+      "Sw360Id": "ae8c7ed",
+      "Sw360AttachmentId": "794446",
+      "CreatedBy": "[email protected]",
+      "CreatedTeam": "AA",
+      "CreatedOn": "2020-10-23",
+      "CheckStatus": "ACCEPTED",
+      "CheckedBy": "[email protected]",
+      "CheckedTeam": "BB",
+      "CheckedOn": "2020-10-30",
+      "CliFile": "CLIXML_certifi-2022.12.7.xml"
+    },
+    {
+      "ComponentName": "certifi 2022.12.7",
+      "Sw360Id": "ae8c7ed",
+      "Sw360AttachmentId": "63b368",
+      "ReportFile": "certifi-2022.12.7_clearing_report.docx"
+    }
+  ]
+}

tests/test_bom_downloadattachments.py

@@ -11,15 +11,16 @@
 
 import responses
 
-from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport
+from capycli.common.capycli_bom_support import CaPyCliBom
+from capycli.common.json_support import load_json_file
 from capycli.bom.download_attachments import BomDownloadAttachments
 from capycli.main.result_codes import ResultCode
-from cyclonedx.model import ExternalReferenceType, HashAlgorithm
 from tests.test_base import AppArguments, TestBase
 
 
 class TestBomDownloadAttachments(TestBase):
     INPUTFILE = "sbom_for_download.json"
+    CONTROLFILE = "sbom_for_download-control.json"
     INPUTERROR = "plaintext.txt"
     OUTPUTFILE = "output.json"
 
@@ -69,6 +70,8 @@ def test_file_not_found(self) -> None:
             args.command.append("bom")
             args.command.append("downloadattachments")
             args.inputfile = "DOESNOTEXIST"
+            args.controlfile = os.path.join(os.path.dirname(__file__),
+                                            "fixtures", TestBomDownloadAttachments.CONTROLFILE)
 
             sut.run(args)
             self.assertTrue(False, "Failed to report missing file")
@@ -85,6 +88,8 @@ def test_error_loading_file(self) -> None:
             args.command.append("bom")
             args.command.append("downloadattachments")
             args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTERROR)
+            args.controlfile = os.path.join(os.path.dirname(__file__),
+                                            "fixtures", TestBomDownloadAttachments.CONTROLFILE)
 
             sut.run(args)
             self.assertTrue(False, "Failed to report invalid file")
@@ -103,6 +108,8 @@ def test_source_folder_does_not_exist(self) -> None:
             args.command.append("downloadattachments")
 
             args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
+            args.controlfile = os.path.join(os.path.dirname(__file__),
+                                            "fixtures", TestBomDownloadAttachments.CONTROLFILE)
             args.source = "XXX"
 
             sut.run(args)
@@ -113,32 +120,10 @@ def test_source_folder_does_not_exist(self) -> None:
     @responses.activate
     def test_simple_bom(self) -> None:
         bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
-        bom = CaPyCliBom.read_sbom(bom)
+        controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE)
 
-        # attachment info - CLI
-        responses.add(
-            method=responses.GET,
-            url=self.MYURL + "resource/api/attachments/794446",
-            body="""
-                {
-                    "filename": "CLIXML_certifi-2022.12.7.xml",
-                    "sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
-                    "attachmentType": "COMPONENT_LICENSE_INFO_XML",
-                    "createdBy": "[email protected]",
-                    "createdTeam": "AA",
-                    "createdComment": "comment1",
-                    "createdOn": "2020-10-08",
-                    "checkStatus": "NOTCHECKED",
-                    "_links": {
-                        "self": {
-                            "href": "https://my.server.com/resource/api/attachments/794446"
-                        }
-                    }
-                }""",
-            status=200,
-            content_type="application/json",
-            adding_headers={"Authorization": "Token " + self.MYTOKEN},
-        )
+        bom = CaPyCliBom.read_sbom(bom)
+        controlfile = load_json_file(controlfile)
 
         # get attachment - CLI
         cli_file = self.get_cli_file_mit()
@@ -150,35 +135,6 @@ def test_simple_bom(self) -> None:
             content_type="application/text",
             adding_headers={"Authorization": "Token " + self.MYTOKEN},
         )
-
-        # attachment info - report
-        responses.add(
-            method=responses.GET,
-            url=self.MYURL + "resource/api/attachments/63b368",
-            body="""
-                {
-                    "filename": "certifi-2022.12.7_clearing_report.docx",
-                    "sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
-                    "attachmentType": "CLEARING_REPORT",
-                    "createdBy": "[email protected]",
-                    "createdTeam": "BB",
-                    "createdComment": "comment3",
-                    "createdOn": "2020-10-08",
-                    "checkedBy": "[email protected]",
-                    "checkedOn" : "2021-01-18",
-                    "checkedComment": "comment4",
-                    "checkStatus": "ACCEPTED",
-                    "_links": {
-                        "self": {
-                            "href": "https://my.server.com/resource/api/attachments/63b368"
-                        }
-                    }
-                }""",
-            status=200,
-            content_type="application/json",
-            adding_headers={"Authorization": "Token " + self.MYTOKEN},
-        )
-
         # get attachment - report
         responses.add(
             method=responses.GET,
@@ -191,7 +147,7 @@ def test_simple_bom(self) -> None:
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             try:
-                bom = self.app.download_attachments(bom, tmpdirname)
+                bom = self.app.download_attachments(bom, controlfile["Components"], tmpdirname)
                 resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml")
                 self.assertEqual(bom.components[0].external_references[5].url, resultfile)
                 self.assertTrue(os.path.isfile(resultfile), "CLI file missing")
@@ -211,25 +167,8 @@ def test_simple_bom_relpath(self) -> None:
         bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
         bom = CaPyCliBom.read_sbom(bom)
 
-        # attachment info - CLI
-        responses.add(
-            method=responses.GET,
-            url=self.MYURL + "resource/api/attachments/794446",
-            body="""
-                {
-                    "filename": "CLIXML_certifi-2022.12.7.xml",
-                    "sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
-                    "attachmentType": "COMPONENT_LICENSE_INFO_XML",
-                    "_links": {
-                        "self": {
-                            "href": "https://my.server.com/resource/api/attachments/794446"
-                        }
-                    }
-                }""",
-            status=200,
-            content_type="application/json",
-            adding_headers={"Authorization": "Token " + self.MYTOKEN},
-        )
+        controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE)
+        controlfile = load_json_file(controlfile)
 
         # get attachment - CLI
         cli_file = self.get_cli_file_mit()
@@ -244,7 +183,8 @@ def test_simple_bom_relpath(self) -> None:
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             try:
-                bom = self.app.download_attachments(bom, tmpdirname, tmpdirname, ("COMPONENT_LICENSE_INFO_XML",))
+                bom = self.app.download_attachments(bom, controlfile["Components"],
+                                                    tmpdirname, tmpdirname, ("COMPONENT_LICENSE_INFO_XML",))
                 resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml")
                 self.assertEqual(bom.components[0].external_references[5].url,
                                  "file://CLIXML_certifi-2022.12.7.xml")
@@ -263,59 +203,29 @@ def test_simple_bom_download_errors(self) -> None:
         bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
         bom = CaPyCliBom.read_sbom(bom)
 
-        # attachment info - CLI, ok
-        responses.add(
-            method=responses.GET,
-            url=self.MYURL + "resource/api/attachments/794446",
-            body="""
-                {
-                    "filename": "CLIXML_certifi-2022.12.7.xml",
-                    "sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
-                    "attachmentType": "COMPONENT_LICENSE_INFO_XML",
-                    "_links": {
-                        "self": {
-                            "href": "https://my.server.com/resource/api/attachments/794446"
-                        }
-                    }
-                }""",
-            status=200,
-            content_type="application/json",
-            adding_headers={"Authorization": "Token " + self.MYTOKEN},
-        )
+        controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE)
+        controlfile = load_json_file(controlfile)
 
         # get attachment - CLI, error
         responses.add(
             method=responses.GET,
             url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/794446",
-            body="cli_file",
             status=500,
             content_type="application/text",
             adding_headers={"Authorization": "Token " + self.MYTOKEN},
         )
-
-        # attachment info - report, error
+        # get attachment - CLI, error
         responses.add(
             method=responses.GET,
-            url=self.MYURL + "resource/api/attachments/63b368",
-            body="""
-                {
-                    "filename": "certifi-2022.12.7_clearing_report.docx",
-                    "sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
-                    "attachmentType": "CLEARING_REPORT",
-                    "_links": {
-                        "self": {
-                            "href": "https://my.server.com/resource/api/attachments/63b368"
-                        }
-                    }
-                }""",
-            status=404,
-            content_type="application/json",
+            url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/63b368",
+            status=403,
+            content_type="application/text",
             adding_headers={"Authorization": "Token " + self.MYTOKEN},
         )
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             try:
-                bom = self.app.download_attachments(bom, tmpdirname)
+                bom = self.app.download_attachments(bom, controlfile["Components"], tmpdirname)
                 resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml")
                 self.assertFalse(os.path.isfile(resultfile), "CLI created despite HTTP 500")
 
@@ -335,8 +245,8 @@ def test_simple_bom_no_release_id(self) -> None:
         bom.components[0].properties = []
         with tempfile.TemporaryDirectory() as tmpdirname:
             try:
-                err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname)
-                assert "No sw360Id for release" in err
+                err = self.capture_stdout(self.app.download_attachments, bom, [], tmpdirname)
+                self.assertIn("No sw360Id for release", err)
 
                 return
             except Exception as e:  # noqa
@@ -346,18 +256,14 @@ def test_simple_bom_no_release_id(self) -> None:
         self.assertTrue(False, "Error: we must never arrive here")
 
     @responses.activate
-    def test_simple_bom_no_attachment_id(self) -> None:
+    def test_simple_bom_no_ctrl_file_entry(self) -> None:
         bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
         bom = CaPyCliBom.read_sbom(bom)
-        bom.components[0].external_references = []
-        CycloneDxSupport.set_ext_ref(bom.components[0], ExternalReferenceType.OTHER,
-                                     CaPyCliBom.CLI_FILE_COMMENT, "CLIXML_foo.xml",
-                                     HashAlgorithm.SHA_1, "123")
 
         with tempfile.TemporaryDirectory() as tmpdirname:
             try:
-                err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname)
-                assert "No sw360Id for attachment" in err
+                err = self.capture_stdout(self.app.download_attachments, bom, [], tmpdirname)
+                assert "Found 0 entries for attachment CLIXML_certifi-2022.12.7.xml" in err
 
                 return
             except Exception as e:  # noqa