Implement support for Pulp manual createrepo

backend/copr_backend/pulp.py

@@ -187,6 +187,19 @@ def delete_content(self, repository, artifacts):
         data = {"remove_content_units": artifacts}
         return requests.post(url, json=data, **self.request_params)
 
+    def copy_content(self, repository, units):
+        """
+        Copy a list of content units (RPM packages) into this repository
+        The "copy" isn't the right word because the actual data isn't touched
+        at all. Only a new database entry pointing to the data is created.
+        So regardless of the number of units, this operation should be fast.
+        https://pulpproject.org/pulp_rpm/restapi/#tag/Repositories:-Rpm/operation/repositories_rpm_rpm_modify
+        """
+        path = os.path.join(repository, "modify/")
+        url = self.config["base_url"] + path
+        data = {"add_content_units": units}
+        return requests.post(url, json=data, **self.request_params)
+
     def delete_repository(self, repository):
         """
         Delete an RPM repository
@@ -229,3 +242,12 @@ def list_distributions(self, prefix):
         url = self.url("api/v3/distributions/rpm/rpm/?")
         url += urlencode({"name__startswith": prefix})
         return requests.get(url, **self.request_params)
+
+    def list_packages(self, repository_version):
+        """
+        Get a list of RPM packages provided by a given repository
+        https://pulpproject.org/pulp_rpm/restapi/#tag/Content:-Advisories/operation/content_rpm_advisories_list
+        """
+        url = self.url("api/v3/content/rpm/packages/?")
+        url += urlencode({"repository_version": repository_version})
+        return requests.get(url, **self.request_params)

backend/copr_backend/storage.py

@@ -213,8 +213,56 @@ def init_project(self, dirname, chroot):
                            distribution, response.text)
             return False
 
-        response = self.client.create_publication(repository)
-        return response.ok
+        # Copy packages from the devel repo to the main repo.
+        # It is weird to do this in the `init_project` method, pointing to the
+        # fact that our abstractions are probably wrong and should be fixed.
+        # We have the `publish_repository` method which is called after a build
+        # is finished. We don't want to copy the packages there. This
+        # `init_project` method is called when a project is created (for which
+        # the copying does nothing) but also when a "Regenerate" button is
+        # clicked (for which we copy the data). We should probably separate
+        # these two concepts.
+        if not self.devel:
+            response = self._copy_packages_from_devel(repository, dirname, chroot)
+            if response and not response.ok:
+                self.log.error("Failed to copy devel packages for %s/%s",
+                               dirname, chroot)
+                return False
+
+        return self.publish_repository(chroot)
+
+    def _copy_packages_from_devel(self, repository, dirname, chroot):
+        """
+        # Copy packages from the devel repo to the main repo.
+        """
+        devel_repository_name = "/".join([
+            self.owner,
+            dirname or self.project,
+            chroot + "_devel",
+        ])
+        response = self.client.get_repository(devel_repository_name)
+        results = response.json()["results"]
+        if not results:
+            return None
+
+        devel_repository = results[0]
+        devel_repository_version = devel_repository["latest_version_href"]
+        response = self.client.list_packages(devel_repository_version)
+        if not response.ok:
+            self.log.error(
+                "Failed to get the latest repository version: %s",
+                repository,
+            )
+            return response
+
+        hrefs = [x["pulp_href"] for x in response.json()["results"]]
+        response = self.client.copy_content(repository, hrefs)
+        if not response.ok:
+            self.log.error(
+                "Failed to copy packages into the %s repository: %s",
+                repository, hrefs,
+            )
+        return response
 
     def upload_build_results(self, chroot, results_dir, target_dir_name):
         resources = []
@@ -379,16 +427,16 @@ def repository_exists(self, dirname, chroot):
         return response.ok
 
     def _repository_name(self, chroot, dirname=None):
+        # On backend we use /devel but Pulp has a problem with that
+        suffix = "_devel" if self.devel else ""
         return "/".join([
             self.owner,
             dirname or self.project,
-            chroot,
+            chroot + suffix,
         ])
 
     def _distribution_name(self, chroot, dirname=None):
         repository = self._repository_name(chroot, dirname)
-        if self.devel:
-            return "{0}-devel".format(repository)
         return repository
 
     def _get_repository(self, chroot):

frontend/coprs_frontend/coprs/logic/complex_logic.py

@@ -8,7 +8,7 @@
 import flask
 import sqlalchemy
 
-from copr_common.enums import StatusEnum
+from copr_common.enums import StatusEnum, StorageEnum
 from coprs import app
 from coprs import db
 from coprs import helpers
@@ -610,9 +610,11 @@ def generate_build_config(cls, copr, chroot_id):
             repos[0]["module_hotfixes"] = True
 
         if not copr.auto_createrepo:
+            suffix = "_devel" if copr.storage == StorageEnum.pulp else "/devel"
+            baseurl = copr.repo_url + "/{0}{1}/".format(chroot_id, suffix)
             repos.append({
                 "id": "copr_base_devel",
-                "baseurl": copr.repo_url + "/{}/devel/".format(chroot_id),
+                "baseurl": baseurl,
                 "name": "Copr buildroot",
             })