Use registry + local cache instead of gha (#24658)

Author: mikesurowiec

.dockerignore

@@ -0,0 +1,6 @@
+.git/
+node_modules/
+.github/
+.vscode/
+docs/
+script/

.github/workflows/main-docker-cache.yml

@@ -0,0 +1,71 @@
+name: Build and Push Main Docker Cache
+
+# **What it does**: Builds and pushes the `main` Docker cache image
+# **Why we have it**: It allows PRs using the registry cache to pull a pre-built image, which should speed up the build
+# **Who does it impact**: All contributors.
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+# This allows a subsequently queued workflow run to take priority over
+# previously queued runs and interrupt currently executing runs
+concurrency:
+  group: '${{ github.workflow }}'
+  cancel-in-progress: true
+
+jobs:
+  build-and-push-nonprod-cache:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      NONPROD_REGISTRY_USERNAME: ghdocs
+      NONPROD_REGISTRY_NAME: ghdocs
+      DOCKER_IMAGE_MAIN_REF: ${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main
+
+    steps:
+      - name: 'Az CLI login'
+        uses: azure/login@66d2e78565ab7af265d2b627085bc34c73ce6abb
+        with:
+          creds: ${{ secrets.NONPROD_AZURE_CREDENTIALS }}
+
+      - name: 'Docker login'
+        uses: azure/docker-login@81744f9799e7eaa418697cb168452a2882ae844a
+        with:
+          login-server: ${{ secrets.NONPROD_REGISTRY_SERVER }}
+          username: ${{ env.NONPROD_REGISTRY_USERNAME }}
+          password: ${{ secrets.NONPROD_REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
+
+      - name: Check out repo
+        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+        with:
+          # To prevent issues with cloning early access content later
+          persist-credentials: 'false'
+          lfs: 'true'
+
+      - name: Check out LFS objects
+        run: git lfs checkout
+
+      - if: ${{ github.repository == 'github/docs-internal' }}
+        name: Clone early access
+        env:
+          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
+          GIT_BRANCH: main
+        run: npm install dotenv && node script/early-access/clone-for-build.js
+
+      - name: 'Build and push image'
+        uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
+        with:
+          context: .
+          push: true
+          target: ${{ fromJSON('["production", "production_early_access"]')[github.repository == 'github/docs-internal'] }}
+          tags: ${{ env.DOCKER_IMAGE_MAIN_REF }}
+          cache-from: type=registry,ref=${{ env.DOCKER_IMAGE_MAIN_REF }}
+          cache-to: type=registry,mode=max,ref=${{ env.DOCKER_IMAGE_MAIN_REF }}

.github/workflows/staging-build-and-deploy-azure.yml

@@ -67,6 +67,14 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
 
+      - name: Cache Docker layers
+        uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
       - name: Check out repo
         uses: actions/checkout@1e204e9a9253d643386038d443f96446fa156a97
         with:
@@ -78,12 +86,6 @@ jobs:
       - name: Check out LFS objects
         run: git lfs checkout
 
-      - name: Setup node
-        uses: actions/setup-node@04c56d2f954f1e4c69436aa54cfef261a018f458
-        with:
-          node-version: 16.13.x
-          cache: npm
-
       - if: ${{ github.repository == 'github/docs-internal' }}
         name: Clone early access
         env:
@@ -98,8 +100,12 @@ jobs:
           push: true
           target: ${{ fromJSON('["production", "production_early_access"]')[github.repository == 'github/docs-internal'] }}
           tags: ${{ env.DOCKER_IMAGE }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          # we only pull the `main` cache image
+          cache-from: |
+            type=local,src=/tmp/.buildx-cache
+            type=registry,ref=${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main
+          # `main-docker-cache.yml` handles updating the remote cache so we don't pollute it with PR specific code
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
 
       # Succeed despite any non-zero exit code (e.g. if there is no deployment to cancel)
       - name: 'Cancel any existing deployments for this PR'
@@ -124,3 +130,11 @@ jobs:
             dockerRegistryPassword="${{ secrets.NONPROD_REGISTRY_PASSWORD }}"
 
       - run: echo ${{ steps.deploy.outputs.defaultHostName }}
+
+      - # Fixes cache growth problem
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache