Merge pull request #17 from parca-dev/add_arm64_support

Add ARM64 Support

.github/workflows/generate-python.yml

@@ -35,6 +35,7 @@ jobs:
           3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.10.8, 3.10.9, 3.10.10, 3.10.11, 3.10.12, 3.10.13,
           3.11.0, 3.11.1, 3.11.2, 3.11.3, 3.11.4, 3.11.5, 3.11.6, 3.11.7, 3.11.8
         ]
+        arch: [amd64, arm64]
     steps:
       - name: Check out the code
         uses: actions/[email protected]
@@ -61,31 +62,36 @@ jobs:
       - name: Build
         run: make build
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+
       - name: Set up container image for generate
         uses: ScribeMD/[email protected]
         with:
-          key: docker-python-${{ runner.os }}-${{ runner.arch }}
+          key: docker-python-${{ runner.os }}-${{ matrix.arch }}
 
       - name: Set up Layout cache for generated files
         uses: actions/[email protected]
         with:
           path: offsets-python
-          key: python-generated-${{ runner.os }}-${{ runner.arch }}
+          key: python-generated-${{ runner.os }}-${{ matrix.arch }}
           restore-keys: |
-            python-generated-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}
-            python-generated-${{ runner.os }}-${{ runner.arch }}
+            python-generated-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.python-version }}
+            python-generated-${{ runner.os }}-${{ matrix.arch }}
 
-      - name: Generate python Offsets for ${{ matrix.python-version }}
+      - name: Generate Python Offsets for ${{ matrix.python-version }} on ${{ matrix.arch }}
         run: |
           export python_version="${{ matrix.python-version }}"
 
-          if ! ls python-binaries/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0 1> /dev/null 2>&1; then
-            mkdir -p python-binaries/${{ matrix.python-version }}
-            docker run --rm -v "${PWD}"/python-binaries/${{ matrix.python-version }}:/tmp -w /tmp docker.io/library/python:${{ matrix.python-version }} bash -c 'cp /usr/local/lib/libpython"${python_version%.*}"*.so.1.0 /tmp'
+          if ! ls python-binaries/${{ matrix.arch }}/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0 1> /dev/null 2>&1; then
+            mkdir -p python-binaries/${{ matrix.arch }}/${{ matrix.python-version }}
+            docker run --rm --platform "linux/${{ matrix.arch }}" -v "${PWD}"/python-binaries/${{ matrix.arch }}/${{ matrix.python-version }}:/tmp -w /tmp docker.io/library/python:${{ matrix.python-version }} bash -c 'cp /usr/local/lib/libpython"${python_version%.*}"*.so.1.0 /tmp'
           fi
 
-          mkdir -p offsets-python
-          ./structlayout -r python -v ${{ matrix.python-version }} -o offsets-python python-binaries/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0
+          mkdir -p offsets-python/${{ matrix.arch }}
+          ./structlayout -r python -v ${{ matrix.python-version }} -o offsets-python/${{ matrix.arch }} python-binaries/${{ matrix.arch }}/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0
 
           git add offsets-python
 
@@ -96,7 +102,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           if-no-files-found: ignore
-          name: python-${{ matrix.python-version }}
+          name: python-${{ matrix-arch }}-${{ matrix.python-version }}
           path: offsets-python
           retention-days: 1
 
@@ -143,10 +149,20 @@ jobs:
 
       - name: Merge the layout files
         run: |
-          rm pkg/python/layout/*
-          rm pkg/python/initialstate/*
-          ./mergelayout -o pkg/python/layout offsets-python/layout/python_*.yaml
-          ./mergelayout -o pkg/python/initialstate offsets-python/initialstate/python_*.yaml
+          target_archs=(
+              amd64
+              arm64
+          )
+
+          rm -rf pkg/python/layout
+          rm -rf pkg/python/initialstate
+          for arch in "${target_archs[@]}"; do
+              mkdir -p pkg/python/layout/"${arch}"
+              ./mergelayout -o pkg/python/layout/"${arch}" offsets-python/"${arch}"/layout/python_*.yaml
+
+              mkdir -p pkg/python/initialstate"/${arch}"
+              ./mergelayout -o pkg/python/initialstate/"${arch}" offsets-python/"${arch}"/initialstate/python_*.yaml
+          done
 
       # If there are no changes (i.e. no diff exists with the checked-out base branch),
       # no pull request will be created and the action exits silently.

.github/workflows/generate-ruby.yml

@@ -29,8 +29,10 @@ jobs:
           2.7.0, 2.7.1, 2.7.2, 2.7.3, 2.7.4, 2.7.5, 2.7.6, 2.7.7, 2.7.8,
           3.0.0, 3.0.1, 3.0.2, 3.0.3, 3.0.4, 3.0.5, 3.0.6,
           3.1.0, 3.1.1, 3.1.2, 3.1.3, 3.1.4,
-          3.2.0, 3.2.1, 3.2.2, 3.2.3, 3.3.0
+          3.2.0, 3.2.1, 3.2.2, 3.2.3,
+          3.3.0
         ]
+        arch: [amd64, arm64]
     steps:
       - name: Check out the code
         uses: actions/[email protected]
@@ -60,26 +62,31 @@ jobs:
       - name: Set up container image for generate
         uses: ScribeMD/[email protected]
         with:
-          key: docker-ruby-${{ runner.os }}-${{ runner.arch }}
+          key: docker-ruby-${{ runner.os }}-${{ matrix.arch }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
 
       - name: Set up Layout cache for generated files
         uses: actions/[email protected]
         with:
           path: offsets-ruby
-          key: ruby-generated-${{ runner.os }}-${{ runner.arch }}
+          key: ruby-generated-${{ runner.os }}-${{ matrix.arch }}
           restore-keys: |
-            ruby-generated-${{ runner.os }}-${{ runner.arch }}-${{ matrix.ruby-version }}
-            ruby-generated-${{ runner.os }}-${{ runner.arch }}
+            ruby-generated-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.ruby-version }}
+            ruby-generated-${{ runner.os }}-${{ matrix.arch }}
 
-      - name: Generate Ruby Offsets for ${{ matrix.ruby-version }}
+      - name: Generate Ruby Offsets for ${{ matrix.ruby-version }} on ${{ matrix.arch }}
         run: |
-          if ! ls "${PWD}"/ruby-binaries/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }} 1>/dev/null 2>&1; then
-            mkdir -p "${PWD}"/ruby-binaries/${{ matrix.ruby-version }}
-            docker run --rm -v "${PWD}"/ruby-binaries/${{ matrix.ruby-version }}:/tmp -w /tmp docker.io/library/ruby:${{ matrix.ruby-version }}-slim bash -c 'cp /usr/local/lib/libruby.so.${{ matrix.ruby-version }} /tmp'
+          if ! ls "${PWD}"/ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }} 1>/dev/null 2>&1; then
+            mkdir -p "${PWD}"/ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }}
+            docker run --rm --platform "linux/${{ matrix.arch }}" -v "${PWD}"/ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }}:/tmp -w /tmp docker.io/library/ruby:${{ matrix.ruby-version }}-slim bash -c 'cp /usr/local/lib/libruby.so.${{ matrix.ruby-version }} /tmp'
           fi
 
-          mkdir -p offsets-ruby
-          ./structlayout -r ruby -v ${{ matrix.ruby-version }} -o offsets-ruby ruby-binaries/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }}
+          mkdir -p offsets-ruby/${{ matrix.arch }}/
+          ./structlayout -r ruby -v ${{ matrix.ruby-version }} -o offsets-ruby/${{ matrix.arch }} ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }}
 
           git add offsets-ruby
 
@@ -89,7 +96,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           if-no-files-found: ignore
-          name: ruby-${{ matrix.ruby-version }}
+          name: ruby-${{ matrix.arch }}-${{ matrix.ruby-version }}
           path: offsets-ruby
           retention-days: 1
 
@@ -136,8 +143,15 @@ jobs:
 
       - name: Merge the layout files
         run: |
-          rm pkg/ruby/layout/*
-          ./mergelayout -o pkg/ruby/layout offsets-ruby/layout/ruby_*.yaml
+          target_archs=(
+              amd64
+              arm64
+          )
+          rm -rf pkg/ruby/layout
+          for arch in "${target_archs[@]}"; do
+              mkdir -p pkg/ruby/layout/"${arch}"
+              ./mergelayout -o pkg/ruby/layout/"${arch}" offsets-ruby/"${arch}"/layout/ruby_*.yaml
+          done
 
       # If there are no changes (i.e. no diff exists with the checked-out base branch),
       # no pull request will be created and the action exits silently.

.github/workflows/test-integration.yml

@@ -49,4 +49,4 @@ jobs:
             ${{ runner.arch }}-integration-runtimes
 
       - name: Run integration tests
-        run: make test/integration
+        run: make ARCH="amd64" generate test/integration

Makefile

@@ -58,13 +58,15 @@ tagalign:
 	go run github.com/4meepo/tagalign/cmd/tagalign@latest -fix -sort ./...
 
 .PHONY: test
-test: generate
+test: build
 	go test ./...
 
 .PHONY: test/integration
-test/integration: generate
+test/integration: build
+	@echo "Make sure to run 'generate' if any generation code has changed"
 	go test -tags=integration ./tests/integration/...
 
+
 .PHONY: test/integration/update
 test/integration/update:
 	go test -count=1 -race -tags=integration ./tests/integration/... -update

README.md

@@ -3,6 +3,26 @@
 This project is a collection of tools to extract runtime data from several interpreters and runtimes.
 By runtime data, we mean information about the execution of a program, especially stack unwinding and profiling.
 
+## Requirements
+
+- [Devbox](https://jetpack.io/devbox)
+  - Also see [CONTRIBUTING.md](CONTRIBUTING.md) for more information about the development environment.
+- [direnv](https://direnv.net/)
+
+```sh
+make bootstrap
+```
+
+- A container runtime (e.g. Docker, Podman)
+  - Ability to run cross-platform containers (e.g. Docker Desktop, Podman with QEMU)
+
+```sh
+sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts
+
+docker run --rm -t arm64v8/ubuntu uname -m
+```
+
 ## Usage
 
 ### Python

pkg/python/initialstate/amd64/3.7.0 - 3.7.2.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 24
+tstate_current: 1392

pkg/python/initialstate/amd64/3.8.0 - 3.8.1.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 32
+tstate_current: 1368

pkg/python/initialstate/amd64/3.9.5 - 3.10.0.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 32
+tstate_current: 568

pkg/python/initialstate/amd64/= 3.11.0.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 40
+tstate_current: 576

pkg/python/initialstate/amd64/= 3.7.4.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 24
+tstate_current: 1480

pkg/python/initialstate/arm64/3.7.0 - 3.7.2.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 24
+tstate_current: 1408

pkg/python/initialstate/arm64/3.8.0 - 3.8.1.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 32
+tstate_current: 1384

pkg/python/initialstate/arm64/3.9.5 - 3.10.0.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 32
+tstate_current: 584

pkg/python/initialstate/arm64/= 3.11.0.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 40
+tstate_current: 592

pkg/python/initialstate/arm64/= 3.7.4.yaml

@@ -0,0 +1,2 @@
+interpreter_head: 24
+tstate_current: 1496

pkg/python/layout/amd64/2.7.15 - 2.7.18.yaml

@@ -0,0 +1,32 @@
+py_cframe:
+    current_frame: 0
+py_code_object:
+    co_filename: 80
+    co_firstlineno: 96
+    co_name: 88
+    co_varnames: 56
+py_frame_object:
+    f_back: 24
+    f_code: 32
+    f_lineno: 124
+    f_localsplus: 376
+py_interpreter_state:
+    tstate_head: 8
+py_object:
+    ob_type: 8
+py_runtime_state:
+    interp_main: -1
+py_string:
+    data: 36
+    size: 16
+py_thread_state:
+    cframe: -1
+    frame: 16
+    interp: 8
+    native_thread_id: -1
+    next: 0
+    thread_id: 144
+py_tuple_object:
+    ob_item: 24
+py_type_object:
+    tp_name: 24

pkg/python/layout/amd64/3.4.8 - 3.5.5.yaml

@@ -0,0 +1,32 @@
+py_cframe:
+    current_frame: 0
+py_code_object:
+    co_filename: 96
+    co_firstlineno: 112
+    co_name: 104
+    co_varnames: 64
+py_frame_object:
+    f_back: 24
+    f_code: 32
+    f_lineno: 124
+    f_localsplus: 376
+py_interpreter_state:
+    tstate_head: 8
+py_object:
+    ob_type: 8
+py_runtime_state:
+    interp_main: -1
+py_string:
+    data: 48
+    size: 16
+py_thread_state:
+    cframe: -1
+    frame: 24
+    interp: 16
+    native_thread_id: -1
+    next: 8
+    thread_id: 152
+py_tuple_object:
+    ob_item: 24
+py_type_object:
+    tp_name: 24

pkg/python/layout/amd64/3.7.0 - 3.7.4.yaml

@@ -0,0 +1,32 @@
+py_cframe:
+    current_frame: 0
+py_code_object:
+    co_filename: 96
+    co_firstlineno: 36
+    co_name: 104
+    co_varnames: 64
+py_frame_object:
+    f_back: 24
+    f_code: 32
+    f_lineno: 108
+    f_localsplus: 360
+py_interpreter_state:
+    tstate_head: 8
+py_object:
+    ob_type: 8
+py_runtime_state:
+    interp_main: -1
+py_string:
+    data: 48
+    size: 16
+py_thread_state:
+    cframe: -1
+    frame: 24
+    interp: 16
+    native_thread_id: -1
+    next: 8
+    thread_id: 176
+py_tuple_object:
+    ob_item: 24
+py_type_object:
+    tp_name: 24

pkg/python/layout/amd64/3.8.0 - 3.9.6.yaml

@@ -0,0 +1,32 @@
+py_cframe:
+    current_frame: 0
+py_code_object:
+    co_filename: 104
+    co_firstlineno: 40
+    co_name: 112
+    co_varnames: 72
+py_frame_object:
+    f_back: 24
+    f_code: 32
+    f_lineno: 108
+    f_localsplus: 360
+py_interpreter_state:
+    tstate_head: 8
+py_object:
+    ob_type: 8
+py_runtime_state:
+    interp_main: -1
+py_string:
+    data: 48
+    size: 16
+py_thread_state:
+    cframe: -1
+    frame: 24
+    interp: 16
+    native_thread_id: -1
+    next: 8
+    thread_id: 176
+py_tuple_object:
+    ob_item: 24
+py_type_object:
+    tp_name: 24