build profiling image in CI, add arm64 support to AWS notebooks (#68)

.dockerignore

@@ -13,5 +13,4 @@
 !LightGBM/python-package
 !LightGBM/src
 !LightGBM/swig
-!LightGBM/windows
 !LightGBM/VERSION.txt

.github/workflows/main.yml

@@ -31,6 +31,9 @@ jobs:
       - name: Build cluster image
         run: |
           make cluster-image
+      - name: Build profiling image
+        run: |
+          make profiling-image
   all-tests-successful:
     if: always()
     runs-on: ubuntu-latest

Dockerfile-cluster

@@ -3,7 +3,7 @@ ARG BASE_IMAGE=unset
 # hadolint ignore=DL3006
 FROM ${BASE_IMAGE}
 
-RUN --mount=type=bind,source=LightGBM,target=/tmp/LightGBM \
+RUN --mount=type=bind,source=LightGBM,target=/tmp/LightGBM,rw \
 <<EOF
 cd /tmp/LightGBM
 

Dockerfile-notebook

@@ -5,7 +5,7 @@ FROM ${BASE_IMAGE}
 
 COPY jupyter_notebook_config.py /root/.jupyter/jupyter_notebook_config.py
 
-RUN --mount=type=bind,source=LightGBM,target=/tmp/LightGBM \
+RUN --mount=type=bind,source=LightGBM,target=/tmp/LightGBM,rw \
 <<EOF
 cd /tmp/LightGBM
 

Dockerfile-profiling

@@ -3,12 +3,14 @@ ARG BASE_IMAGE=unset
 # hadolint ignore=DL3006
 FROM ${BASE_IMAGE}
 
-RUN pip install --no-cache-dir --prefer-binary \
-        memray \
-        pytest \
-        pytest-memray \
-        pytest-profiling \
-        snakeviz
+RUN <<EOF
+pip install --no-cache-dir --prefer-binary \
+    memray \
+    pytest \
+    pytest-memray \
+    pytest-profiling \
+    snakeviz
+EOF
 
 COPY bin/profile-examples.sh /usr/local/bin/profile-examples.sh
 COPY bin/profile-example-memory-usage.sh /usr/local/bin/profile-example-memory-usage.sh

Makefile

@@ -200,6 +200,8 @@ push-image: create-repo
 	docker push \
 		$$(cat ./ecr-details.json | jq .'repository'.'repositoryUri' | tr -d '"'):${IMAGE_TAG}
 
+# NOTE: IMAGE_TAG is in the environment here so the AWS notebooks
+#       know what image to use for the Dask cluster
 .PHONY: start-notebook
 start-notebook:
 	docker run \
@@ -208,6 +210,7 @@ start-notebook:
 		--env AWS_ACCESS_KEY_ID=$${AWS_ACCESS_KEY_ID:-notset} \
 		--env AWS_DEFAULT_REGION=${AWS_REGION} \
 		--env AWS_SECRET_ACCESS_KEY=$${AWS_SECRET_ACCESS_KEY:-notset} \
+		--env IMAGE_TAG=${IMAGE_TAG} \
 		-p 8888:8888 \
 		-p 8787:8787 \
 		--name ${NOTEBOOK_CONTAINER_NAME} \

notebooks/demo-aws.ipynb

@@ -51,18 +51,37 @@
     "with open(\"../ecr-details.json\", \"r\") as f:\n",
     "    ecr_details = json.loads(f.read())\n",
     "\n",
-    "CONTAINER_IMAGE = (\n",
-    "    ecr_details[\"repository\"][\"repositoryUri\"] + \":\" + os.environ[\"DASK_VERSION\"]\n",
-    ")\n",
-    "print(f\"scheduler and worker image: {CONTAINER_IMAGE}\")"
+    "IMAGE_REPO = ecr_details[\"repository\"][\"repositoryUri\"]\n",
+    "IMAGE_TAG = os.environ[\"IMAGE_TAG\"]\n",
+    "IMAGE_URI = f\"{IMAGE_REPO}:{IMAGE_TAG}\"\n",
+    "print(f\"scheduler and worker image: {IMAGE_URI}\")"
    ]
   },
   {
    "cell_type": "markdown",
    "id": "harmful-bosnia",
    "metadata": {},
    "source": [
-    "Before proceeding, set up your AWS credentials. If you're unsure how to do this, see [the AWS docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)."
+    "Before proceeding, set up your AWS credentials. If you're unsure how to do this, see [the AWS docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html).\n",
+    "\n",
+    "Next, determine the CPU architecture of the machine you're running on.\n",
+    "This project builds single-architecture container images matching the host system, so it's important\n",
+    "to use the same CPU architecture on AWS Fargate."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4a10d61c-5251-46a7-9f16-bd6eef606a82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import platform\n",
+    "\n",
+    "if platform.machine().lower() in {\"aarch64\", \"arm64\"}:\n",
+    "    cpu_architecture = \"ARM64\"\n",
+    "else:\n",
+    "    cpu_architecture = \"X86_64\""
    ]
   },
   {
@@ -85,7 +104,8 @@
     "\n",
     "n_workers = 3\n",
     "cluster = FargateCluster(\n",
-    "    image=CONTAINER_IMAGE,\n",
+    "    image=IMAGE_URI,\n",
+    "    cpu_architecture=cpu_architecture,\n",
     "    worker_cpu=512,\n",
     "    worker_mem=4096,\n",
     "    n_workers=n_workers,\n",
@@ -300,7 +320,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,