diff --git a/script/cfg.py b/script/cfg.py index 754c385..6ec272f 100644 --- a/script/cfg.py +++ b/script/cfg.py @@ -1,3 +1,5 @@ +import os + header = '''# GENERATED FILE - DO NOT EDIT set -e ''' @@ -110,8 +112,7 @@ def rsync_commands(checksum): def pre_rsync_repo(repos): return '' -rsync_repo1 = ''' -echo '# REPOOWNLIST' +rsync_repo_buildid = ''' [ ! -f __envsub/files_iso.lst ] || buildid=$(cat __envsub/files_iso.lst | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1) [ ! -f __envsub/files_iso.lst ] || test -n "$buildid" || buildid=$(cat __envsub/files_iso.lst | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1) [ -z "__STAGING" ] || buildid=${buildid//Build/Build__STAGING.} @@ -119,7 +120,11 @@ def pre_rsync_repo(repos): [ -n "$buildid" ] || buildid=$(grep -hEo 'Build[0-9]+(.[0-9]+)?' __envsub/Media1_*.lst 2>/dev/null | head -n 1) buildid=${buildid/.iso} +''' +rsync_repo1 = ''' +echo '# REPOOWNLIST' +''' + rsync_repo_buildid + ''' for repo in {REPOOWNLIST,}; do while read src; do [ ! -z "$src" ] || continue @@ -172,6 +177,29 @@ def rsync_repodir2(): done ''' + +def rsync_repomultiarch(destpath, debug, source): + destpath = destpath.rstrip("/") + dest = os.path.basename(destpath) + res = ''' +echo rsync --timeout=3600 -rtlp4 --delete --specials PRODUCTREPOPATH/'''+ destpath +'''/ /var/lib/openqa/factory/repo/''' + dest + '''-CURRENT +echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/''' + dest + '''-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-$buildid/ +''' + + if debug: + debugfilter = "--include=" + debug + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*" + res = res + ''' +echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + debugfilter + ''' PRODUCTREPOPATH/'''+ destpath +'''-Debug/ /var/lib/openqa/factory/repo/''' + dest + '''-Debug-CURRENT/ +echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/''' + dest + '''-Debug-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Debug-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Debug-$buildid/''' + + if source: + sourcefilter = "--include=" + source + " --exclude={aarch64,armv7hl,i586,i686,noarch,nosrc,ppc64,ppc64le,s390x,src,x86_64}/*" + res = res + ''' +echo rsync --timeout=3600 -rtlp4 --delete --specials ''' + sourcefilter + ''' PRODUCTREPOPATH/'''+ destpath +'''-Source/ /var/lib/openqa/factory/repo/''' + dest + '''-Source-CURRENT/ +echo rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/''' + dest + '''-Source-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Source-CURRENT/ /var/lib/openqa/factory/repo/''' + dest + '''-Source-$buildid/''' + + return res + rsync_repodir1_dest = lambda dest: ''' archs=(ARCHITECTURREPO) buildid=$(cat __envsub/files_iso.lst | grep -E 'FLAVORORS' | grep -o -E '(Build|Snapshot)[^-]*' | head -n 1) diff --git a/script/scriptgen.py b/script/scriptgen.py index 1f3c851..b7f7cf8 100644 --- a/script/scriptgen.py +++ b/script/scriptgen.py @@ -175,6 +175,7 @@ def __init__(self, name, actionGenerator): self.ln_iso_to_repo = {} self.mirror_repo = "" self.repos = [] + self.reposmultiarch = [] # these repos need not to be processed for each arch self.repolink = "" self.build_id_from_iso = 0 self.repodirs = [] @@ -381,6 +382,10 @@ def doFlavor(self, node): self.archs_repo = t.attrib["archs"] for t in node.findall(".//repos/*"): + if t.get("multiarch", ""): + self.reposmultiarch.append(t) + continue + if "folder" in t.attrib: self.repodirs.append(t) else: @@ -864,6 +869,19 @@ def gen_print_rsync_repo(self, f): print(cfg.header, file=f) if self.ag.version: print("version=" + self.ag.version, file=f) + + if self.reposmultiarch: + self.p(cfg.rsync_repo_buildid, f) + for repo in self.reposmultiarch: + destpath = repo.get("folder", repo.tag) + dest = os.path.basename(destpath) + self.p( + cfg.rsync_repomultiarch( + repo.get("folder", repo.tag), repo.get("debug", ""), repo.get("source", "") + ), + f, + ) + if self.repos: self.p(cfg.pre_rsync_repo(self.repos), f) self.p(cfg.rsync_repo1, f) @@ -1002,6 +1020,28 @@ def gen_print_openqa(self, f): ), f, ) + + imultiarch = 0 + for repo in self.reposmultiarch: + destpath = repo.get("folder", repo.tag) + destpath = destpath.rstrip("/") + dest = os.path.basename(destpath) + self.p(' echo " REPO_{}={}-$buildex" \\\\'.format(imultiarch, dest), f) + self.p(' echo " REPO_{}={}-$buildex" \\\\'.format(repo.tag.upper(), dest), f) + imultiarch = imultiarch + 1 + if repo.get("debug", ""): + self.p(' echo " REPO_{}={}-Debug-$buildex" \\\\'.format(imultiarch, dest), f) + self.p(' echo " REPO_{}_DEBUG={}-Debug-$buildex" \\\\'.format(repo.tag.upper(), dest), f) + self.p(" echo \" REPO_{}_DEBUG_PACKAGES='{}'\" \\\\".format(repo.tag.upper(), repo.get("debug", "")), f) + imultiarch = imultiarch + 1 + if repo.get("source", ""): + self.p(' echo " REPO_{}={}-Source-$buildex" \\\\'.format(imultiarch, dest), f) + self.p(' echo " REPO_{}_SOURCE={}-Source-$buildex" \\\\'.format(repo.tag.upper(), dest), f) + self.p( + " echo \" REPO_{}_SOURCE_PACKAGES='{}'\" \\\\".format(repo.tag.upper(), repo.get("source", "")), f + ) + imultiarch = imultiarch + 1 + if len(self.iso1) > 0: self.p(' iso1=""', f) self.p( diff --git a/t/obs/openSUSE:Leap:16.0:ToTest/print_openqa.before b/t/obs/openSUSE:Leap:16.0:ToTest/print_openqa.before index 966d7b3..64823f4 100644 --- a/t/obs/openSUSE:Leap:16.0:ToTest/print_openqa.before +++ b/t/obs/openSUSE:Leap:16.0:ToTest/print_openqa.before @@ -7,7 +7,10 @@ FLAVOR=agama-installer-Leap \ ISO=agama-installer-Leap.aarch64-16.0.0-Leap-Build1.3.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 @@ -20,7 +23,10 @@ FLAVOR=agama-installer-Leap \ ISO=agama-installer-Leap.x86_64-16.0.0-Leap-Build1.3.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 @@ -33,7 +39,10 @@ FLAVOR=agama-installer-Leap \ ISO=agama-installer-Leap.ppc64le-16.0.0-Leap-Build1.3.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 @@ -46,7 +55,10 @@ FLAVOR=agama-installer-Leap \ ISO=agama-installer-Leap.s390x-16.0.0-Leap-Build1.3.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 @@ -59,7 +71,10 @@ FLAVOR=offline-installer \ ISO=Leap-16.0-offline-installer-aarch64-Build1.3.install.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 @@ -72,7 +87,10 @@ FLAVOR=offline-installer \ ISO=Leap-16.0-offline-installer-x86_64-Build1.3.install.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 @@ -85,7 +103,10 @@ FLAVOR=offline-installer \ ISO=Leap-16.0-offline-installer-ppc64le-Build1.3.install.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 @@ -98,7 +119,10 @@ FLAVOR=offline-installer \ ISO=Leap-16.0-offline-installer-s390x-Build1.3.install.iso \ REPO_0=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ - REPO_SLE_LEAP_16_0_AARCH64_PPC64LE_S390X_X86_64=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_1=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3 \ + REPO_LEAP_OSS_DEBUG=Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3 \ + REPO_LEAP_OSS_DEBUG_PACKAGES='{java*,kernel-default-debug*,kernel-default-base-debug*,mraa-debug*,wicked-debug*}' \ VERSION=16.0 \ _OBSOLETE=1 diff --git a/t/obs/openSUSE:Leap:16.0:ToTest/print_rsync_repo.before b/t/obs/openSUSE:Leap:16.0:ToTest/print_rsync_repo.before index babc05f..ac37b69 100644 --- a/t/obs/openSUSE:Leap:16.0:ToTest/print_rsync_repo.before +++ b/t/obs/openSUSE:Leap:16.0:ToTest/print_rsync_repo.before @@ -1,3 +1,4 @@ -# Leap-16.0-aarch64-ppc64le-s390x-x86_64$ -rsync --timeout=3600 -rtlp4 --delete --specials obspublish::openqa/openSUSE:Leap:16.0:ToTest/product/local/000productcompose:leap_oss/Leap-16.0-aarch64-ppc64le-s390x-x86_64/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT +rsync --timeout=3600 -rtlp4 --delete --specials obspublish::openqa/openSUSE:Leap:16.0:ToTest/product/local/*leap_oss/Leap-16.0-aarch64-ppc64le-s390x-x86_64/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Build1.3/ +rsync --timeout=3600 -rtlp4 --delete --specials --include=java* --include=kernel-default-debug* --include=kernel-default-base-debug* --include=mraa-debug* --include=wicked-debug* --exclude=aarch64/* --exclude=armv7hl/* --exclude=i586/* --exclude=i686/* --exclude=noarch/* --exclude=nosrc/* --exclude=ppc64/* --exclude=ppc64le/* --exclude=s390x/* --exclude=src/* --exclude=x86_64/* obspublish::openqa/openSUSE:Leap:16.0:ToTest/product/local/*leap_oss/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-CURRENT/ +rsync --timeout=3600 -rtlp4 --delete --specials --link-dest /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-CURRENT/ /var/lib/openqa/factory/repo/Leap-16.0-aarch64-ppc64le-s390x-x86_64-Debug-Build1.3/ diff --git a/xml/obs/openSUSE:Leap:16.0.xml b/xml/obs/openSUSE:Leap:16.0.xml index 10bc276..dd5a112 100644 --- a/xml/obs/openSUSE:Leap:16.0.xml +++ b/xml/obs/openSUSE:Leap:16.0.xml @@ -1,12 +1,12 @@ - +