Add a self-hosted Azure pipeline that builds the internal repo for all OS, uploads debug symbols, updates LatestVersion.json, Telemetry.json, publishes GitHub releases, and notifies on Discord ("Release" posts to the release channel, "Beta" posts to the beta channel).

Author: Koi-3088

.azure-pipelines/azure-pipelines.yml

@@ -0,0 +1,18 @@
+# templates/checkout.yml
+# Template for checking out the Arduino-Source repositories
+
+steps:
+  - checkout: Arduino-Source-Internal
+    path: Arduino-Source-Internal
+    fetchDepth: 1
+    persistCredentials: true
+    sparseCheckoutPatterns: |
+      /*
+      !Repository/Public/*
+      !Repository/Deployment-SerialPrograms-Qt6.10.0-MSVC2022/
+      !Repository/Deployment-SerialPrograms-Qt6.8.3-MSVC2022/
+    workspaceRepo: true
+
+  - checkout: Arduino-Source
+    path: Arduino-Source-Internal/Repository/Public
+    persistCredentials: true

.azure-pipelines/templates/checkout.yml

@@ -0,0 +1,178 @@
+# templates/macos-build.yml
+# Template for building SerialPrograms on MacOS
+
+parameters:
+  stageName: ''
+  displayName: ''
+  targetOS: ''
+  poolName: ''
+  imageName: ''
+  architecture: ''
+  compiler: ''
+  qt_version: ''
+  qt_version_major: ''
+  qt_modules: ''
+  cmake_preset: ''
+  macos_path: ''
+  cmake_version_params: ''
+  cmake_additional_param: ''
+  condition: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+  displayName: ${{ parameters.displayName }}
+  dependsOn: []
+  condition: ${{ parameters.condition }}
+
+  jobs:
+  - job: MacOS
+    timeoutInMinutes: 60
+    cancelTimeoutInMinutes: 1
+    displayName: MacOS ${{ parameters.architecture }}
+
+    pool:
+      name: ${{ parameters.poolName }}
+      demands:
+        - Agent.OSArchitecture -equals ${{ lower(parameters.architecture) }}
+
+    variables:
+      architecture: ${{ parameters.architecture }}
+      compiler: ${{ parameters.compiler }}
+      qt_version: ${{ parameters.qt_version }}
+      cmake_preset: ${{ parameters.cmake_preset }}
+      macos_path: ${{ parameters.macos_path }}
+      cmake_version_params: ${{ parameters.cmake_version_params }}
+      cmake_additional_param: ${{ parameters.cmake_additional_param }}
+
+    steps:
+      - template: checkout.yml
+
+      #- script: |
+      #    set -e
+      #    PERSISTENT_ONNX="/Users/Shared/onnxruntime"
+      #    WORK_ONNX="$(Pipeline.Workspace)/onnxruntime"
+      #
+      #    if [ -d "$PERSISTENT_ONNX" ]; then
+      #      echo "=== Found ONNX Runtime at $PERSISTENT_ONNX ==="
+      #      if [ ! -L "$WORK_ONNX" ]; then
+      #        ln -sf "$PERSISTENT_ONNX" "$WORK_ONNX"
+      #        echo "=== Created symlink to ONNX Runtime ==="
+      #      fi
+      #    else
+      #      echo "=== ERROR: ONNX Runtime not found at $PERSISTENT_ONNX ==="
+      #      exit 1
+      #    fi
+      #  displayName: 'Link ONNX Runtime'
+      #  condition: succeeded()
+
+      - script: |
+          export PATH=$(macos_path)
+          cd "$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/SerialPrograms"
+          cmake $(cmake_additional_param) $(cmake_version_params) --preset=$(cmake_preset) --fresh
+        displayName: 'Configure CMake'
+        condition: succeeded()
+
+      - script: |
+          export PATH=$(macos_path)
+          cd "$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/SerialPrograms"
+          cmake --build --preset=$(cmake_preset)
+        displayName: 'Build'
+        condition: succeeded()
+
+      - script: |
+          set -e
+          export PATH=$(macos_path)
+          BUILD_DIR="$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/build/$(cmake_preset)"
+
+          echo "=== Generating dSYM bundle ==="
+          mkdir -p "$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/build/$(cmake_preset)/cache-symbols"
+          cd "$BUILD_DIR"
+          dsymutil SerialPrograms.app/Contents/MacOS/SerialPrograms -o cache-symbols/SerialPrograms.dSYM
+
+          echo "=== dSYM bundle created ==="
+        displayName: 'Generate Debug Symbols'
+        condition: succeeded()
+
+      - script: |
+          set -e
+          export PATH=$(macos_path)
+          APP_DIR="$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/build/$(cmake_preset)/SerialPrograms.app"
+          MACOS_DIR="$APP_DIR/Contents/MacOS"
+          FW_DIR="$APP_DIR/Contents/Frameworks"
+          BREW_PREFIX=$(brew --prefix)
+
+          mkdir -p "$FW_DIR"
+          if [ "$(architecture)" = "x64" ]; then
+              QT_BIN="/usr/local/Qt/$(qt_version)/macos/bin"
+          else
+              QT_BIN="/opt/Qt/$(qt_version)/macos/bin"
+          fi
+
+          echo "=== Copying additional deps ==="
+          cp $BREW_PREFIX/opt/gcc/lib/gcc/current/libgcc_s.1.1.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/protobuf/lib/libutf8_validity.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/little-cms2/lib/liblcms2.2.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/webp/lib/libsharpyuv.0.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/jpeg-xl/lib/libjxl_cms.0.11.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/vtk/lib/libvtkCommonComputationalGeometry-*.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/vtk/lib/libvtkFiltersVerdict-*.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/vtk/lib/libvtkfmt-*.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/vtk/lib/libvtkFiltersGeometry-*.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/vtk/lib/libvtkFiltersCore-*.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/vtk/lib/libvtkCommonCore-*.dylib $APP_DIR/Contents/Frameworks
+          cp $BREW_PREFIX/opt/vtk/lib/libvtkCommonSystem-*.dylib $APP_DIR/Contents/Frameworks
+
+          echo "=== Running macdeployqt6 ==="
+          install_name_tool -add_rpath $BREW_PREFIX/lib $MACOS_DIR/SerialPrograms
+          otool -l $MACOS_DIR/SerialPrograms | grep -A2 LC_RPATH
+          "$QT_BIN/macdeployqt6" $APP_DIR -no-strip -verbose=3
+
+          echo "=== Fixing install IDs in copied libs ==="
+          for dylib in "$FW_DIR"/*.dylib; do
+              [ -f "$dylib" ] || continue
+              base=$(basename "$dylib")
+              install_name_tool -id "@executable_path/../Frameworks/$base" "$dylib"
+          done
+
+          echo "=== Rewriting internal dylib references ==="
+          for dylib in "$FW_DIR"/*.dylib; do
+              [ -f "$dylib" ] || continue
+              for linked in $(otool -L "$dylib" | awk 'NR>1 {print $1}' | grep "$BREW_PREFIX" || true); do
+                  base=$(basename "$linked")
+                  if [ -f "$FW_DIR/$base" ]; then
+                      install_name_tool -change "$linked" "@executable_path/../Frameworks/$base" "$dylib"
+                  fi
+              done
+          done
+
+          echo "=== Rewriting main executable references ==="
+          for linked in $(otool -L "$MACOS_DIR/SerialPrograms" | awk 'NR>1 {print $1}' | grep "$BREW_PREFIX" || true); do
+              base=$(basename "$linked")
+              if [ -f "$FW_DIR/$base" ]; then
+                  install_name_tool -change "$linked" "@executable_path/../Frameworks/$base" "$MACOS_DIR/SerialPrograms"
+              fi
+          done
+
+          echo "=== Creating tarballs for build and symbols ==="
+          mkdir -p "$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/build/$(cmake_preset)/cache-build"
+          cd "$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/build/$(cmake_preset)"
+          tar -czf "cache-build/macos-build-${{ parameters.architecture }}.tar.gz" SerialPrograms.app
+          tar -czf "cache-symbols/macos-symbols-${{ parameters.architecture }}.tar.gz" cache-symbols/SerialPrograms.dSYM
+
+          echo "Deployment complete for $(architecture)"
+        displayName: 'Deploy app'
+        condition: succeeded()
+
+      - task: Cache@2
+        displayName: 'Cache the build artifact'
+        inputs:
+          key: 'macos-build-${{ parameters.architecture }} | "$(Build.BuildId)"'
+          path: '$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/build/$(cmake_preset)/cache-build'
+        condition: succeeded()
+
+      - task: Cache@2
+        displayName: 'Cache debug symbols'
+        inputs:
+          key: 'macos-symbols-${{ parameters.architecture }} | "$(Build.BuildId)"'
+          path: '$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/build/$(cmake_preset)/cache-symbols'
+        condition: succeeded()

.azure-pipelines/templates/github-release.yml

@@ -0,0 +1,191 @@
+# templates/macos-notarize.yml
+# Template for codesigning and notarizing SerialPrograms on MacOS
+
+parameters:
+  stageName: ''
+  displayName: ''
+  dependsOnStage: ''
+  buildType: ''
+  poolName: ''
+  imageName: ''
+  architecture: ''
+  compiler: ''
+  cmake_preset: ''
+  macos_path: ''
+  openssl_pkcs_args: ''
+  condition: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+  displayName: ${{ parameters.displayName }}
+  dependsOn: ${{ parameters.dependsOnStage }}
+  condition: ${{ parameters.condition }}
+
+  jobs:
+  - job: Codesign
+    displayName: Codesign ${{ parameters.architecture }}
+    timeoutInMinutes: 120
+    cancelTimeoutInMinutes: 1
+
+    pool:
+      name: ${{ parameters.poolName }}
+      demands:
+        - Agent.OSArchitecture -equals ${{ lower(parameters.architecture) }}
+
+    variables:
+      architecture: ${{ parameters.architecture }}
+      compiler: ${{ parameters.compiler }}
+      cmake_preset: ${{ parameters.cmake_preset }}
+
+    steps:
+      - template: checkout.yml
+
+      - task: Cache@2
+        displayName: 'Restore the cached build artifact'
+        inputs:
+          key: 'macos-build-${{ parameters.architecture }} | "$(Build.BuildId)"'
+          path: $(Pipeline.Workspace)
+          restoreKeys: |
+            macos-build-${{ parameters.architecture }} | "$(Build.BuildId)"
+        condition: succeeded()
+
+      - task: InstallAppleCertificate@2
+        displayName: 'Install Apple certificate'
+        inputs:
+          certSecureFile: 'CodesignCertMac.p12'
+          certPwd: $(CERT_PW)
+          keychain: custom
+          keychainPassword: $(KEYCHAIN_PW)
+          customKeychainPath: '$(HOME)/Library/Keychains/azure-signing.keychain-db'
+          deleteCert: true
+          deleteCustomKeychain: true
+          opensslPkcsArgs: ${{ parameters.openssl_pkcs_args }}
+        condition: succeeded()
+
+      - script: |
+          set -e
+          KEYCHAIN="$HOME/Library/Keychains/azure-signing.keychain-db"
+
+          echo "=== Activating keychain ==="
+          security list-keychains -d user -s "$KEYCHAIN" "$HOME/Library/Keychains/login.keychain-db"
+          security unlock-keychain -p "$(KEYCHAIN_PW)" "$KEYCHAIN"
+
+          echo "=== Allowing codesign to access private key ==="
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$(KEYCHAIN_PW)" "$KEYCHAIN"
+
+          echo "=== Verifying identities ==="
+          security find-identity -p codesigning -v
+        env:
+          KEYCHAIN_PW: $(KEYCHAIN_PW)
+        displayName: 'Authorize signing key'
+        condition: succeeded()
+
+      - script: |
+          set -euo pipefail
+          export PATH=${{ parameters.macos_path }}
+          APP_DIR="$(Pipeline.Workspace)/Arduino-Source-Internal/SerialPrograms.app"
+          ZIP_PATH="$(Pipeline.Workspace)/SerialPrograms-MacOS-$(compiler)-$(architecture).zip"
+          ENTITLEMENTS_DIR="$(Pipeline.Workspace)/Arduino-Source-Internal/Repository/Public/SerialPrograms/cmake/MacOSXEntitlements.plist"
+
+          echo "=== Extracting tarball ==="
+          tar -xzf "$(Pipeline.Workspace)/macos-build-${{ parameters.architecture }}.tar.gz"
+
+          echo "=== Signing app ==="
+          codesign --deep --force --options runtime --timestamp --entitlements "$ENTITLEMENTS_DIR" --sign "$SIGN_IDENTITY" "$APP_DIR"
+
+          echo "=== Verifying code signature ==="
+          codesign --verify --deep --strict --verbose=2 "$APP_DIR"
+
+          echo "Creating ZIP for notarization..."
+          ditto -c -k --keepParent "$APP_DIR" "$ZIP_PATH"
+        env:
+          SIGN_IDENTITY: $(SIGNING_IDENTITY)
+        displayName: 'Codesign, verify, create ZIP'
+        condition: succeeded()
+
+      - script: |
+          set -euo pipefail
+          ZIP_PATH="$(Pipeline.Workspace)/SerialPrograms-MacOS-$(compiler)-$(architecture).zip"
+
+          SUBMIT_JSON=$(xcrun notarytool submit "$ZIP_PATH" \
+            --apple-id "$APPLE_ID" \
+            --password "$APPLE_APP_PW" \
+            --team-id "$APPLE_TEAM_ID" \
+            --output-format json)
+
+          REQUEST_ID=$(echo "$SUBMIT_JSON" | jq -r '.id')
+          [ -n "$REQUEST_ID" ] && [ "$REQUEST_ID" != "null" ]
+
+          echo "##vso[task.setvariable variable=NOTARY_REQUEST_ID]$REQUEST_ID"
+        env:
+          APPLE_ID: $(APPLE_ID)
+          APPLE_APP_PW: $(APPLE_APP_PW)
+          APPLE_TEAM_ID: $(APPLE_TEAM_ID)
+        displayName: 'Submit for notarization'
+        condition: succeeded()
+
+      - script: |
+          set -euo pipefail
+          MAX_WAIT=14400
+          INTERVAL=30
+          ELAPSED=0
+
+          while true; do
+            STATUS=$(xcrun notarytool info "$(NOTARY_REQUEST_ID)" \
+              --apple-id "$APPLE_ID" \
+              --password "$APPLE_APP_PW" \
+              --team-id "$APPLE_TEAM_ID" \
+              --output-format json | jq -r '.status')
+
+            case "$STATUS" in
+              Accepted) break ;;
+              Invalid)
+                xcrun notarytool log "$(NOTARY_REQUEST_ID)" \
+                  --apple-id "$APPLE_ID" \
+                  --password "$APPLE_APP_PW" \
+                  --team-id "$APPLE_TEAM_ID"
+                exit 1 ;;
+            esac
+
+            sleep "$INTERVAL"
+            ELAPSED=$((ELAPSED + INTERVAL))
+            [ "$ELAPSED" -lt "$MAX_WAIT" ]
+          done
+        env:
+          APPLE_ID: $(APPLE_ID)
+          APPLE_APP_PW: $(APPLE_APP_PW)
+          APPLE_TEAM_ID: $(APPLE_TEAM_ID)
+        displayName: 'Wait for notarization'
+        condition: succeeded()
+
+      - script: |
+          set -euo pipefail
+          APP_DIR="$(Pipeline.Workspace)/Arduino-Source-Internal/SerialPrograms.app"
+          xcrun stapler staple "$APP_DIR"
+          spctl --assess --type execute --verbose=4 "$APP_DIR"
+        displayName: 'Staple and verify'
+        condition: succeeded()
+
+      - script: |
+          echo "=== Creating a tarball ==="
+          tar -czf "SerialPrograms-MacOS-$(compiler)-$(architecture).tar.gz" -C "$(Pipeline.Workspace)/Arduino-Source-Internal" "SerialPrograms.app"
+          echo "=== Creating cache directory and moving tarball ==="
+          CACHE_DIR="$(Pipeline.Workspace)/Arduino-Source-Internal/cache-notarized"
+          mkdir -p "$CACHE_DIR"
+          mv "$(Pipeline.Workspace)/Arduino-Source-Internal/SerialPrograms-MacOS-$(compiler)-$(architecture).tar.gz" "$CACHE_DIR/"
+        displayName: 'Create a tarball'
+        condition: succeeded()
+
+      - task: Cache@2
+        displayName: 'Cache the notarized artifact'
+        inputs:
+          key: 'macos-notarized-${{ parameters.architecture }} | "$(Build.BuildId)"'
+          path: '$(Pipeline.Workspace)/Arduino-Source-Internal/cache-notarized'
+        condition: succeeded()
+
+      - task: PublishPipelineArtifact@1
+        displayName: 'Publish notarized app'
+        inputs:
+          targetPath: '$(Pipeline.Workspace)/Arduino-Source-Internal/cache-notarized/SerialPrograms-MacOS-$(compiler)-$(architecture).tar.gz'
+          artifact: 'SerialPrograms-MacOS-$(compiler)-$(architecture)'
+        condition: succeeded()