Merge branch 'main' into minecode-pipeline-npm

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

minecode/tests/collectors/test_github.py

@@ -41,5 +41,5 @@ def test_github_get_all_versions(self):
             "minecode-pipelines/v0.0.1b7",
             "minecode-pipelines/v0.0.1b8",
         ]
-        for item in versions:
-            self.assertIn(item, expected)
+        for item in expected:
+            self.assertIn(item, versions)

minecode_pipelines/__init__.py

@@ -7,4 +7,5 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-VERSION = "0.0.1b9"
+
+VERSION = "0.0.1b15"

minecode_pipelines/miners/conan.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from pathlib import Path
+import saneyaml
+
+from scanpipe.pipes.federatedcode import commit_changes
+from scanpipe.pipes.federatedcode import push_changes
+from minecode_pipelines import VERSION
+from minecode_pipelines.pipes.conan import store_conan_packages
+
+PACKAGE_BATCH_SIZE = 1000
+
+
+def mine_and_publish_conan_packageurls(conan_index_repo, cloned_data_repo, logger):
+    base_path = Path(conan_index_repo.working_dir)
+
+    yml_files = []
+    for file_path in base_path.glob("recipes/**/*"):
+        if not file_path.name == "config.yml":
+            continue
+        yml_files.append(file_path)
+
+    file_counter = 0
+    purl_files = []
+    purls = []
+
+    total_files = len(yml_files)
+    logger(f"Processing total files: {total_files}")
+    for idx, file_path in enumerate(yml_files, start=1):
+        # Example: file_path = Path("repo_path/recipes/7zip/config.yml")
+        # - file_path.parts = ("repo_path", "recipes", "7zip", "config.yml")
+        # - file_path.parts[-2] = "7zip"  (the package name)
+        package = file_path.parts[-2]
+        with open(file_path, encoding="utf-8") as f:
+            versions = saneyaml.load(f)
+
+        if not versions:
+            continue
+
+        file_counter += 1
+        push_commit = file_counter >= PACKAGE_BATCH_SIZE or idx == total_files
+
+        result_store = store_conan_packages(package, versions, cloned_data_repo)
+        if result_store:
+            purl_file, base_purl = result_store
+            logger(f"writing packageURLs for package: {base_purl} at: {purl_file}")
+
+            purl_files.append(purl_file)
+            purls.append(str(base_purl))
+
+        if push_commit:
+            commit_changes(
+                repo=cloned_data_repo,
+                files_to_commit=purl_files,
+                purls=purls,
+                mine_type="packageURL",
+                tool_name="pkg:pypi/minecode-pipelines",
+                tool_version=VERSION,
+            )
+            push_changes(repo=cloned_data_repo)
+            file_counter = 0

minecode_pipelines/pipelines/mine_conan.py

@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import os
+from scanpipe.pipelines import Pipeline
+from minecode_pipelines import pipes
+from minecode_pipelines.miners import conan
+from scanpipe.pipes import federatedcode
+
+MINECODE_CONAN_INDEX_REPO = "https://github.com/conan-io/conan-center-index"
+
+MINECODE_DATA_CONAN_REPO = os.environ.get(
+    "MINECODE_DATA_CONAN_REPO", "https://github.com/aboutcode-data/minecode-data-conan-test"
+)
+
+
+class MineConan(Pipeline):
+    """Pipeline to mine Conan packages and publish them to FederatedCode repo."""
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.clone_conan_repos,
+            cls.mine_and_publish_conan_package_urls,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
+
+    def clone_conan_repos(self):
+        """
+        Clone the Conan-related repositories (index, data, and pipelines config)
+        and store their Repo objects in the corresponding instance variables.
+        """
+        self.conan_index_repo = federatedcode.clone_repository(MINECODE_CONAN_INDEX_REPO)
+        self.cloned_data_repo = federatedcode.clone_repository(MINECODE_DATA_CONAN_REPO)
+
+        if self.log:
+            self.log(
+                f"{MINECODE_CONAN_INDEX_REPO} repo cloned at: {self.conan_index_repo.working_dir}"
+            )
+            self.log(
+                f"{MINECODE_DATA_CONAN_REPO} repo cloned at: {self.cloned_data_repo.working_dir}"
+            )
+
+    def mine_and_publish_conan_package_urls(self):
+        conan.mine_and_publish_conan_packageurls(
+            self.conan_index_repo, self.cloned_data_repo, self.log
+        )
+
+    def delete_cloned_repos(self):
+        pipes.delete_cloned_repos(
+            repos=[self.conan_index_repo, self.cloned_data_repo],
+            logger=self.log,
+        )

minecode_pipelines/pipelines/mine_nuget.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+
+from pathlib import Path
+
+from minecode_pipelines.pipes import nuget
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import federatedcode
+
+
+class MineNuGet(Pipeline):
+    """
+    Mine and Publish NuGet PackageURLs.
+
+    Mine PackageURLs from AboutCode NuGet catalog mirror and publish
+    them to FederatedCode Git repository.
+    """
+
+    download_inputs = False
+    CATALOG_REPO_URL = "https://github.com/aboutcode-org/aboutcode-mirror-nuget-catalog.git"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.fetch_nuget_catalog,
+            cls.mine_nuget_package_versions,
+            cls.mine_and_publish_nuget_packageurls,
+            cls.delete_cloned_repos,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_configured_and_available()
+
+    def fetch_nuget_catalog(self):
+        """Fetch NuGet package catalog from AboutCode mirror."""
+        self.catalog_repo = federatedcode.clone_repository(
+            repo_url=self.CATALOG_REPO_URL,
+            logger=self.log,
+        )
+
+    def mine_nuget_package_versions(self):
+        """Mine NuGet package and versions from NuGet catalog."""
+        self.package_versions, self.skipped_packages = nuget.mine_nuget_package_versions(
+            catalog_path=Path(self.catalog_repo.working_dir),
+            logger=self.log,
+        )
+
+    def mine_and_publish_nuget_packageurls(self):
+        """Mine and publish PackageURLs from NuGet package versions."""
+        nuget.mine_and_publish_nuget_packageurls(
+            package_versions=self.package_versions,
+            logger=self.log,
+        )
+
+    def delete_cloned_repos(self):
+        """Remove cloned catalog repository."""
+        if self.catalog_repo:
+            self.log("Removing cloned repository")
+            federatedcode.delete_local_clone(repo=self.catalog_repo)

minecode_pipelines/pipes/__init__.py

@@ -12,14 +12,12 @@
 import os
 import shutil
 from pathlib import Path
-
+from git import Repo
 import requests
 import saneyaml
 
 from aboutcode.hashid import PURLS_FILENAME
-from git import Repo
 
-from scanpipe.pipes.federatedcode import delete_local_clone
 from scanpipe.pipes.federatedcode import commit_and_push_changes
 
 from minecode_pipelines.utils import get_temp_file
@@ -188,6 +186,8 @@ def write_data_to_json_file(path, data):
 
 
 def delete_cloned_repos(repos, logger=None):
+    from scanpipe.pipes.federatedcode import delete_local_clone
+
     if not repos:
         return
 

minecode_pipelines/pipes/alpine.py

@@ -118,11 +118,11 @@ def build_package(extracted_pkginfo, distro, repo):
     )
 
     parties = []
-    maintainers = extracted_pkginfo.get("maintainer")
-    if maintainers:
-        name, email = parse_email(maintainers)
-        if name:
-            party = Party(name=name, role="maintainer", email=email)
+    maintainer = extracted_pkginfo.get("maintainer")
+    if maintainer:
+        maintainer_name, maintainer_email = parse_email(maintainer)
+        if maintainer_name:
+            party = Party(name=maintainer_name, role="maintainer", email=maintainer_email)
             parties.append(party)
 
     purl = PackageURL(

minecode_pipelines/pipes/conan.py

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from packageurl import PackageURL
+from pathlib import Path
+from aboutcode import hashid
+from minecode_pipelines.pipes import write_data_to_yaml_file
+
+
+def store_conan_packages(pacakge_name, versions_data, fed_repo):
+    """Collect Conan package versions into purls and write them to the repo."""
+
+    base_purl = PackageURL(type="conan", name=pacakge_name)
+
+    updated_purls = []
+    versions = list(versions_data["versions"].keys())
+    for version in versions:
+        purl = PackageURL(type="conan", name=pacakge_name, version=version).to_string()
+        updated_purls.append(purl)
+
+    ppath = hashid.get_package_purls_yml_file_path(base_purl)
+    purl_file_full_path = Path(fed_repo.working_dir) / ppath
+    write_data_to_yaml_file(path=purl_file_full_path, data=updated_purls)
+    return purl_file_full_path, base_purl

minecode_pipelines/pipes/debian.py

@@ -119,9 +119,12 @@ def get_packages(self, previous_index_last_modified_date=None, logger=None):
                 previous_index_last_modified_date, "%Y-%m-%d %H:%M:%S"
             )
         for entry in ls.parse_directory_listing(content):
-            entry_date = datetime.strptime(entry.date, "%Y-%m-%d")
+            entry_date = None
+            if entry.date:
+                entry_date = datetime.strptime(entry.date, "%Y-%m-%d")
             if (entry.type != ls.FILE) or (
                 previous_index_last_modified_date
+                and entry_date
                 and (entry_date <= previous_index_last_modified_date)
             ):
                 continue
@@ -158,8 +161,6 @@ def get_packages(self, previous_index_last_modified_date=None, logger=None):
                 name=package_url.name,
                 version=package_url.version,
                 qualifiers=package_url.qualifiers,
-                file_name=file_name,
-                date=entry.date,
                 size=entry.size,
                 download_url=url_template.format(path=path),
             )
@@ -223,7 +224,7 @@ def collect_packages_from_debian(commits_per_push=PACKAGE_BATCH_SIZE, logger=Non
 
             current_purls = []
             prev_purl = current_purl
-        current_purls.append(package.to_string())
+        current_purls.append(package.purl)
 
     if current_purls:
         # write packageURLs to file