Add special handling of multiarch repos

script/cfg.py

@@ -1,3 +1,5 @@
+import os
+
 header = '''# GENERATED FILE - DO NOT EDIT
 set -e
 '''
@@ -110,16 +112,19 @@ def rsync_commands(checksum):
 def pre_rsync_repo(repos):
     return ''
 
-rsync_repo1 = '''
-echo '# REPOOWNLIST'
+rsync_repo_buildid = '''
 [ ! -f __envsub/files_iso.lst ] || buildid=$(cat __envsub/files_iso.lst | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
 [ ! -f __envsub/files_iso.lst ] || test -n "$buildid" || buildid=$(cat __envsub/files_iso.lst | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)
 [ -z "__STAGING" ] || buildid=${buildid//Build/Build__STAGING.}
 [ ! -f __envsub/files_repo.lst ] || ! grep -q -- "-POOL-" __envsub/files_repo.lst || additional_repo_suffix=-POOL
 [ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+(.[0-9]+)?' __envsub/Media1_*.lst 2>/dev/null | head -n 1)
 
 buildid=${buildid/.iso}
+'''
 
+rsync_repo1 = '''
+echo '# REPOOWNLIST'
+''' + rsync_repo_buildid + '''
 for repo in {REPOOWNLIST,}; do
     while read src; do
         [ ! -z "$src" ] || continue
@@ -172,6 +177,29 @@ def rsync_repodir2():
 done
 '''
 
+
+def rsync_repomultiarch(destpath, debug, source):
+    destpath = destpath.rstrip("/")
+    dest = os.path.basename(destpath)
+    res = '''
+echo rsync --timeout=3600 -rtlp4 --delete --specials PRODUCTREPOPATH/'''+ destpath +'''/ /var/lib/openqa/factory/repo/''' + dest + '''-CURRENT
+echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/''' + dest + '''-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-$buildid/
+'''
+
+    if debug:
+        debugfilter = "--include=" + debug + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*"
+        res = res + '''
+echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + debugfilter + ''' PRODUCTREPOPATH/'''+ destpath +'''-Debug/ /var/lib/openqa/factory/repo/''' + dest + '''-Debug-CURRENT/
+echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/''' + dest + '''-Debug-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Debug-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Debug-$buildid/'''
+
+    if source:
+        sourcefilter = "--include=" + source + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*"
+        res = res + '''
+echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + sourcefilter + ''' PRODUCTREPOPATH/'''+ destpath +'''-Source/ /var/lib/openqa/factory/repo/''' + dest + '''-Source-CURRENT/
+echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/''' + dest + '''-Source-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Source-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Source-$buildid/'''
+
+    return res
+
 rsync_repodir1_dest = lambda dest: '''
 archs=(ARCHITECTURREPO)
 buildid=$(cat __envsub/files_iso.lst | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1)

script/scriptgen.py

@@ -175,6 +175,7 @@ def __init__(self, name, actionGenerator):
         self.ln_iso_to_repo = {}
         self.mirror_repo = ""
         self.repos = []
+        self.reposmultiarch = []  # these repos need not to be processed for each arch
         self.repolink = ""
         self.build_id_from_iso = 0
         self.repodirs = []
@@ -381,6 +382,10 @@ def doFlavor(self, node):
                 self.archs_repo = t.attrib["archs"]
 
         for t in node.findall(".//repos/*"):
+            if t.get("multiarch", ""):
+                self.reposmultiarch.append(t)
+                continue
+
             if "folder" in t.attrib:
                 self.repodirs.append(t)
             else:
@@ -864,6 +869,19 @@ def gen_print_rsync_repo(self, f):
         print(cfg.header, file=f)
         if self.ag.version:
             print("version=" + self.ag.version, file=f)
+
+        if self.reposmultiarch:
+            self.p(cfg.rsync_repo_buildid, f)
+            for repo in self.reposmultiarch:
+                destpath = repo.get("folder", repo.tag)
+                dest = os.path.basename(destpath)
+                self.p(
+                    cfg.rsync_repomultiarch(
+                        repo.get("folder", repo.tag), repo.get("debug", ""), repo.get("source", "")
+                    ),
+                    f,
+                )
+
         if self.repos:
             self.p(cfg.pre_rsync_repo(self.repos), f)
             self.p(cfg.rsync_repo1, f)
@@ -1002,6 +1020,28 @@ def gen_print_openqa(self, f):
                 ),
                 f,
             )
+
+        imultiarch = 0
+        for repo in self.reposmultiarch:
+            destpath = repo.get("folder", repo.tag)
+            destpath = destpath.rstrip("/")
+            dest = os.path.basename(destpath)
+            self.p(' echo " REPO_{}={}-$buildex" \\\\'.format(imultiarch, dest), f)
+            self.p(' echo " REPO_{}={}-$buildex" \\\\'.format(repo.tag.upper(), dest), f)
+            imultiarch = imultiarch + 1
+            if repo.get("debug", ""):
+                self.p(' echo " REPO_{}={}-Debug-$buildex" \\\\'.format(imultiarch, dest), f)
+                self.p(' echo " REPO_{}_DEBUG={}-Debug-$buildex" \\\\'.format(repo.tag.upper(), dest), f)
+                self.p(" echo \" REPO_{}_DEBUG_PACKAGES='{}'\" \\\\".format(repo.tag.upper(), repo.get("debug", "")), f)
+                imultiarch = imultiarch + 1
+            if repo.get("source", ""):
+                self.p(' echo " REPO_{}={}-Source-$buildex" \\\\'.format(imultiarch, dest), f)
+                self.p(' echo " REPO_{}_SOURCE={}-Source-$buildex" \\\\'.format(repo.tag.upper(), dest), f)
+                self.p(
+                    " echo \" REPO_{}_SOURCE_PACKAGES='{}'\" \\\\".format(repo.tag.upper(), repo.get("source", "")), f
+                )
+                imultiarch = imultiarch + 1
+
         if len(self.iso1) > 0:
             self.p(' iso1=""', f)
             self.p(

t/obs/openSUSE:Leap:16.0:ToTest/print_openqa.before

@@ -7,7 +7,10 @@
  FLAVOR=agama-installer-Leap \
  ISO=agama-installer-Leap.aarch64-16.0.0-Leap-Build1.3.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 
@@ -20,7 +23,10 @@
  FLAVOR=agama-installer-Leap \
  ISO=agama-installer-Leap.x86_64-16.0.0-Leap-Build1.3.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 
@@ -33,7 +39,10 @@
  FLAVOR=agama-installer-Leap \
  ISO=agama-installer-Leap.ppc64le-16.0.0-Leap-Build1.3.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 
@@ -46,7 +55,10 @@
  FLAVOR=agama-installer-Leap \
  ISO=agama-installer-Leap.s390x-16.0.0-Leap-Build1.3.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 
@@ -59,7 +71,10 @@
  FLAVOR=offline-installer \
  ISO=Leap-16.0-offline-installer-aarch64-Build1.3.install.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 
@@ -72,7 +87,10 @@
  FLAVOR=offline-installer \
  ISO=Leap-16.0-offline-installer-x86_64-Build1.3.install.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 
@@ -85,7 +103,10 @@
  FLAVOR=offline-installer \
  ISO=Leap-16.0-offline-installer-ppc64le-Build1.3.install.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 
@@ -98,7 +119,10 @@
  FLAVOR=offline-installer \
  ISO=Leap-16.0-offline-installer-s390x-Build1.3.install.iso \
  REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
- REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \
+ REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \
+ REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \
  VERSION=16.0 \
  _OBSOLETE=1
 

t/obs/openSUSE:Leap:16.0:ToTest/print_rsync_repo.before

@@ -1,3 +1,4 @@
-# Leap-16.0-aarch64-ppc64le-s390x-x86_64$
-rsync --timeout=3600 -rtlp4 --delete --specials obspublish::openqa/openSUSE:Leap:16.0:ToTest/product/local/000productcompose:leap_oss/Leap-16.0-aarch64-ppc64le-s390x-x86_64/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT
+rsync --timeout=3600 -rtlp4 --delete --specials obspublish::openqa/openSUSE:Leap:16.0:ToTest/product/local/*leap_oss/Leap-16.0-aarch64-ppc64le-s390x-x86_64/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT
 rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3/
+rsync --timeout=3600 -rtlp4 --delete --specials --include=java* --include=kernel-default-debug* --include=kernel-default-base-debug* --include=mraa-debug* --include=wicked-debug* --exclude=aarch64/* --exclude=armv7hl/* --exclude=i586/* --exclude=i686/* --exclude=noarch/* --exclude=nosrc/* --exclude=ppc64/* --exclude=ppc64le/* --exclude=s390x/* --exclude=src/* --exclude=x86_64/* obspublish::openqa/openSUSE:Leap:16.0:ToTest/product/local/*leap_oss/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-CURRENT/
+rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3/

xml/obs/openSUSE:Leap:16.0.xml

@@ -1,12 +1,12 @@
 <openQA
     project_pattern="openSUSE:Leap:(?P&lt;version&gt;16.0):ToTest"
     iso_path="images"
-    repo_path="product/local/000productcompose:leap_oss"
+    repo_path="product/local"
     distri="opensuse"
     archs="aarch64 x86_64 ppc64le s390x">
     <flavor name="agama-installer-Leap" folder="*/*agama-installer-Leap*" iso="1" media1="0">
         <repos>
-            <oss name="Leap-16.0-aarch64-ppc64le-s390x-x86_64$" mirror="1"/>
+            <leap_oss mirror="1" multiarch="1" folder="*leap_oss/Leap-16.0-aarch64-ppc64le-s390x-x86_64/" debug="{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}"/>
         </repos>
     </flavor>
     <flavor name="offline-installer" folder="../product/local" media1="0">