diff --git a/.github/workflows/generate-web-index-temp.yml b/.github/workflows/generate-web-index-temp.yml new file mode 100644 index 000000000..3dabcd110 --- /dev/null +++ b/.github/workflows/generate-web-index-temp.yml @@ -0,0 +1,187 @@ +name: "Temp: Generate download JSON and upload artifact" +on: + workflow_dispatch: + +jobs: + - name: Build JSON file + run: | + + # prepare feeds + A=$(rsync --recursive --list-only rsync://rsync.armbian.com/dl/ \ + | awk '{ + gsub(",", "", $2); # Remove commas in size + gsub("/", "-", $3); # Fix date format + url = "https://dl.armbian.com/" $5; # Construct URL + + # Include URLs matching //archive/Armbian + if (url ~ /\/[^\/]+\/archive\/Armbian/) { + # Exclude if url matches any of the unwanted substrings + if (url !~ /\.txt|homeassistant|openhab|kali|omv/) { + print $2 "|" url "|" $3 "T" $4 "Z" + } + } + }') + B=$(gh release view --json assets --repo github.com/armbian/community | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\\||g') + C=$(gh release view --json assets --repo github.com/armbian/os | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\\||g') + D=$(gh release view --json assets --repo github.com/armbian/distribution | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\\||g') + + # debug + echo "$A" > a.txt + echo "$B" > b.txt + echo "$C" > c.txt + echo "$D" > d.txt + + # join all in one file and compress + for line in $A $B $C $D; do + + BOARD=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f2) + ARMBIAN_VERSION=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f1) + IMAGE_SIZE=$(echo $line | cut -d"|" -f1 | sed "s/\.//g" | sed "s/\,//g") + KERNEL_BRANCH=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4) + IMAGE_RELEASE=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f3) + IMAGE_URL=$(echo $line | cut -d"|" -f2) + # workarounds + STEP_A=1 + STEP_B=2 + [[ $IMAGE_URL == *i3-wm* || $IMAGE_URL == *kde-* || $IMAGE_URL == *-rc* ]] && STEP_A=2 && STEP_B=3 + IMAGE_TARGET=$(echo $IMAGE_URL | cut -d"|" -f2 | grep "minimal\\|desktop" | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4- | grep -Po '_[a-z].*' | cut -d"." -f1 | sed "s/_//" | sed "s/_desktop//" | sed "s/-kisak//" | sed "s/-backported-mesa//" | cut -d"-" -f1,${STEP_A}) + IMAGE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\\|_sm8250-xiaomi-elish//g' | sed "s/rc[0-9]//g" | cut -d"_" -f4- | cut -d"-" -f${STEP_B}- | cut -d"_" -f1 | cut -d"." -f1) + [[ $IMAGE_EXTENSION == $KERNEL_BRANCH || $IMAGE_EXTENSION == ${IMAGE_TARGET} || $IMAGE_EXTENSION =~ boot|csot|boe|sms ]] && unset IMAGE_EXTENSION + [[ -z ${IMAGE_TARGET} ]] && IMAGE_TARGET="server" + FILE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\\|_sm8250-xiaomi-elish//g' | sed "s/-rc[0-9]//g" | rev | cut -d"_" -f1 | rev | sed 's/.*[^0-9]\\([0-9]*\\.[0-9]*\\.[0-9]*\\)//' | sed "s/desktop.\\|minimal.//") + #FILE_EXTENSION=$(echo $IMAGE_URL | grep -o "oowow.*\\|img.*") + + # Clean out application from extension + FILE_EXTENSION=$(echo $FILE_EXTENSION | sed 's/.*-'$IMAGE_EXTENSION'//g' | sed -e 's/^\\.//g') + + # Extract extensions for weird type of images + WEIRD_EXT=$(echo $FILE_EXTENSION | sed "s/img//g" | sed "s/^\\.//g" | sed "s/\\..*//g" | sed "s/xz//g" | sed "s/oowow//g") + + IMAGE_TYPE=$(echo $IMAGE_URL | cut -d"/" -f5) + IMAGE_CREATED=$(echo $line | cut -d"|" -f3 | sed "s/\\//-/g") + EXPOSED=false + + # Assemble redirector link + PREFIX="" + [[ "${IMAGE_TYPE}" == "os" ]] && PREFIX="nightly/" + REDI_EXT=$(echo $FILE_EXTENSION | rev | cut -d"." -f1 | rev | sed "s/xz//g" | sed "s/qcow2//g" | sed "s/zip//g") + REDI_URL="https://dl.armbian.com/${PREFIX}${BOARD,,}/${IMAGE_RELEASE^}_${KERNEL_BRANCH}${IMAGE_TARGET:+_$IMAGE_TARGET}${IMAGE_EXTENSION:+-$IMAGE_EXTENSION}${WEIRD_EXT:+-$WEIRD_EXT}${REDI_EXT:+.$REDI_EXT}" + + # bypass redirector for weirdos + # if [[ "${FILE_EXTENSION}" =~ boot|csot|boe|sms|rootfs|qcow2 ]]; then REDI_URL=${IMAGE_URL}; fi + + # check if image is exposed + while read exposed; do + [[ $IMAGE_URL =~ $exposed ]] && EXPOSED=true + done < os/exposed.map + echo "${BOARD,,}|$ARMBIAN_VERSION|$IMAGE_URL|$REDI_URL|$IMAGE_CREATED|$IMAGE_SIZE|$IMAGE_RELEASE|$KERNEL_BRANCH|$IMAGE_TARGET|$IMAGE_EXTENSION|$EXPOSED|$IMAGE_TYPE|$FILE_EXTENSION" + + done | sed '1s/^/"board_slug"|"armbian_version"|"file_url"|"redi_url"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"\\n/' | jc --csv | jq '{"assets": .}' > all-images.json + } + + for board in $(ls -1 build/config/boards/*.{conf,csc,wip,tvb}); do + if [[ $(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | sed "s/,/ /g" | wc -w) -gt 2 ]]; then + KERNEL_TARGET=$(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | awk -F, '{ print $1,$3,$2 }') + else + KERNEL_TARGET=$(grep KERNEL_TARGET -w $board | cut -d '"' -f 2 | sed "s/,/ /g") + fi + VARIANTS=0 + for targets in ${KERNEL_TARGET}; do + if [[ "${targets}" == current && $VARIANTS -lt 2 ]]; then + check_targets "$board" "current" + VARIANTS=$((VARIANTS+$?)) + fi + if [[ "${targets}" == legacy && $VARIANTS -lt 2 ]]; then + check_targets "$board" "legacy" + VARIANTS=$((VARIANTS+$?)) + fi + if [[ "${targets}" == edge && $VARIANTS -lt 2 ]]; then + check_targets "$board" "edge" + VARIANTS=$((VARIANTS+$?)) + fi + done || true + done | tee -a > exposed.map.autoproposal + + - name: Build JSON file + run: | + A=$(rsync --recursive --list-only rsync://rsync.armbian.com/dl/ \ + | awk '{\n+ gsub(",", "", $2);\n+ gsub("/", "-", $3);\n+ url = "https://dl.armbian.com/" $5;\n+ if (url ~ /\/[^\/]+\/archive\/Armbian/) {\n+ if (url !~ \.txt|homeassistant|openhab|kali|omv) {\n+ print $2 "|" url "|" $3 "T" $4 "Z"\n+ }\n+ }\n+ }') + B=$(gh release view --json assets --repo github.com/armbian/community | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') + C=$(gh release view --json assets --repo github.com/armbian/os | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') + D=$(gh release view --json assets --repo github.com/armbian/distribution | jq '.assets[] | .size, .url, .createdAt' | xargs -n3 -d'\n' | grep -v ".txt" | sed "s/\"//g" | sed -e 's| |\||g') + + for line in $A $B $C $D; do + BOARD=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f2) + ARMBIAN_VERSION=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f1) + IMAGE_SIZE=$(echo $line | cut -d"|" -f1 | sed "s/\.//g" | sed "s/\,//g") + KERNEL_BRANCH=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4) + IMAGE_RELEASE=$(echo $line | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f3) + IMAGE_URL=$(echo $line | cut -d"|" -f2) + STEP_A=1 + STEP_B=2 + [[ $IMAGE_URL == *i3-wm* || $IMAGE_URL == *kde-* || $IMAGE_URL == *-rc* ]] && STEP_A=2 && STEP_B=3 + IMAGE_TARGET=$(echo $IMAGE_URL | cut -d"|" -f2 | grep "minimal\|desktop" | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | cut -d"_" -f4- | grep -Po '_[a-z].*' | cut -d"." -f1 | sed "s/_//" | sed "s/_desktop//" | sed "s/-kisak//" | sed "s/-backported-mesa//" | cut -d"-" -f1,${STEP_A}) + IMAGE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\|_sm8250-xiaomi-elish//g' | sed "s/rc[0-9]//g" | cut -d"_" -f4- | cut -d"-" -f${STEP_B}- | cut -d"_" -f1 | cut -d"." -f1) + [[ $IMAGE_EXTENSION == $KERNEL_BRANCH || $IMAGE_EXTENSION == ${IMAGE_TARGET} || $IMAGE_EXTENSION =~ boot|csot|boe|sms ]] && unset IMAGE_EXTENSION + [[ -z ${IMAGE_TARGET} ]] && IMAGE_TARGET="server" + FILE_EXTENSION=$(echo $IMAGE_URL | cut -d"|" -f2 | cut -d"|" -f2 | grep -Po 'Armbian.*[0-9][0-9].[0-9].*' | grep -Po '[0-9][0-9].[0-9].*' | sed 's/_sm8250-xiaomi-umi\|_sm8250-xiaomi-elish//g' | sed "s/-rc[0-9]//g" | rev | cut -d"_" -f1 | rev | sed 's/.*[^0-9]\([0-9]*\.[0-9]*\.[0-9]*\)//' | sed "s/desktop.\|minimal.//") + FILE_EXTENSION=$(echo $FILE_EXTENSION | sed 's/.*-'$IMAGE_EXTENSION'//g' | sed -e 's/^\.//g') + WEIRD_EXT=$(echo $FILE_EXTENSION | sed "s/img//g" | sed "s/^\.//g" | sed "s/\..*//g" | sed "s/xz//g" | sed "s/oowow//g") + IMAGE_TYPE=$(echo $IMAGE_URL | cut -d"/" -f5) + IMAGE_CREATED=$(echo $line | cut -d"|" -f3 | sed "s/\//-/g") + EXPOSED=false + PREFIX="" + [[ "${IMAGE_TYPE}" == "os" ]] && PREFIX="nightly/" + REDI_EXT=$(echo $FILE_EXTENSION | rev | cut -d"." -f1 | rev | sed "s/xz//g" | sed "s/qcow2//g" | sed "s/zip//g") + REDI_URL="https://dl.armbian.com/${PREFIX}${BOARD,,}/${IMAGE_RELEASE^}_${KERNEL_BRANCH}${IMAGE_TARGET:+_$IMAGE_TARGET}${IMAGE_EXTENSION:+-$IMAGE_EXTENSION}${WEIRD_EXT:+-$WEIRD_EXT}${REDI_EXT:+.$REDI_EXT}" + + while read exposed; do + [[ $IMAGE_URL =~ $exposed ]] && EXPOSED=true + done < os/exposed.map + echo "${BOARD,,}|$ARMBIAN_VERSION|$IMAGE_URL|$REDI_URL|$IMAGE_CREATED|$IMAGE_SIZE|$IMAGE_RELEASE|$KERNEL_BRANCH|$IMAGE_TARGET|$IMAGE_EXTENSION|$EXPOSED|$IMAGE_TYPE|$FILE_EXTENSION" + done | sed '1s/^/"board_slug"|"armbian_version"|"file_url"|"redi_url"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"\n/' | jc --csv | jq '{"assets": .}' > all-images.json + + - name: Add cache artifact links for checksums/signatures/torrents + run: | + jq ' + .assets as $a + | ($a + ( + $a + | map(select(.file_url | test("\\.(xz|img|zip|iso|qcow2)$"))) + | map(. as $orig + | [ + ($orig + | .file_url = ("https://imola.armbian.com/cache/artifacts/" + ($orig.board_slug) + "/" + ($orig.file_url | split("/") | .[-1]) + ".sha") + | .file_extension = "sha" + | .download_repository = "cache" + | .file_size = 0 + ), + ($orig + | .file_url = ("https://imola.armbian.com/cache/artifacts/" + ($orig.board_slug) + "/" + ($orig.file_url | split("/") | .[-1]) + ".asc") + | .file_extension = "asc" + | .download_repository = "cache" + | .file_size = 0 + ), + ($orig + | .file_url = ("https://imola.armbian.com/cache/artifacts/" + ($orig.board_slug) + "/" + ($orig.file_url | split("/") | .[-1]) + ".torrent") + | .file_extension = "torrent" + | .download_repository = "cache" + | .file_size = 0 + ) + ] + ) | add + )) | {assets: .} + ' all-images.json > all-images-updated.json || ( echo "jq processing failed" && exit 1 ) + + mv all-images-updated.json all-images.json + + - name: Prepare data and upload artifact + run: | + mkdir -p armbian.github.io/data + mv all-images.json armbian.github.io/data/ + + - name: Upload all-images.json artifact + uses: actions/upload-artifact@v4 + with: + name: all-images.json + path: armbian.github.io/data/all-images.json diff --git a/.github/workflows/generate-web-index.yml b/.github/workflows/generate-web-index.yml index 5f9d55323..581e674fa 100644 --- a/.github/workflows/generate-web-index.yml +++ b/.github/workflows/generate-web-index.yml @@ -191,6 +191,45 @@ jobs: done | sed '1s/^/"board_slug"|"armbian_version"|"file_url"|"redi_url"|"file_updated"|"file_size"|"distro_release"|"kernel_branch"|"image_variant"|"preinstalled_application"|"promoted"|"download_repository"|"file_extension"\n/' | jc --csv | jq '{"assets": .}' > all-images.json + - name: "Add cache artifact links for checksums/signatures/torrents" + run: | + + # For image assets (xz/img/zip/qcow2/etc) add additional entries + # pointing to imola cache where sha/asc/torrent files are stored. + # We construct URLs as: https://imola.armbian.com/cache/artifacts/. + + jq ' + .assets as $a + | ($a + ( + $a + | map(select(.file_url | test("\\.(xz|img|zip|iso|qcow2)$"))) + | map(. as $orig + | [ + ($orig + | .file_url = ("https://imola.armbian.com/cache/artifacts/" + ($orig.board_slug) + "/" + ($orig.file_url | split("/") | .[-1]) + ".sha") + | .file_extension = "sha" + | .download_repository = "cache" + | .file_size = 0 + ), + ($orig + | .file_url = ("https://imola.armbian.com/cache/artifacts/" + ($orig.board_slug) + "/" + ($orig.file_url | split("/") | .[-1]) + ".asc") + | .file_extension = "asc" + | .download_repository = "cache" + | .file_size = 0 + ), + ($orig + | .file_url = ("https://imola.armbian.com/cache/artifacts/" + ($orig.board_slug) + "/" + ($orig.file_url | split("/") | .[-1]) + ".torrent") + | .file_extension = "torrent" + | .download_repository = "cache" + | .file_size = 0 + ) + ] + ) | add + )) | {assets: .} + ' all-images.json > all-images-updated.json || ( echo "jq processing failed" && exit 1 ) + + mv all-images-updated.json all-images.json + - name: "Download and compress torrent files" run: |