diff --git a/.github/workflows/generate-python.yml b/.github/workflows/generate-python.yml index 38bcaa4..4ef6494 100644 --- a/.github/workflows/generate-python.yml +++ b/.github/workflows/generate-python.yml @@ -35,6 +35,7 @@ jobs: 3.10.0, 3.10.1, 3.10.2, 3.10.3, 3.10.4, 3.10.5, 3.10.6, 3.10.7, 3.10.8, 3.10.9, 3.10.10, 3.10.11, 3.10.12, 3.10.13, 3.11.0, 3.11.1, 3.11.2, 3.11.3, 3.11.4, 3.11.5, 3.11.6, 3.11.7, 3.11.8 ] + arch: [amd64, arm64] steps: - name: Check out the code uses: actions/checkout@v4.1.1 @@ -61,31 +62,36 @@ jobs: - name: Build run: make build + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 + - name: Set up container image for generate uses: ScribeMD/docker-cache@0.3.7 with: - key: docker-python-${{ runner.os }}-${{ runner.arch }} + key: docker-python-${{ runner.os }}-${{ matrix.arch }} - name: Set up Layout cache for generated files uses: actions/cache@v4.0.0 with: path: offsets-python - key: python-generated-${{ runner.os }}-${{ runner.arch }} + key: python-generated-${{ runner.os }}-${{ matrix.arch }} restore-keys: | - python-generated-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }} - python-generated-${{ runner.os }}-${{ runner.arch }} + python-generated-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.python-version }} + python-generated-${{ runner.os }}-${{ matrix.arch }} - - name: Generate python Offsets for ${{ matrix.python-version }} + - name: Generate Python Offsets for ${{ matrix.python-version }} on ${{ matrix.arch }} run: | export python_version="${{ matrix.python-version }}" - if ! ls python-binaries/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0 1> /dev/null 2>&1; then - mkdir -p python-binaries/${{ matrix.python-version }} - docker run --rm -v "${PWD}"/python-binaries/${{ matrix.python-version }}:/tmp -w /tmp docker.io/library/python:${{ matrix.python-version }} bash -c 'cp /usr/local/lib/libpython"${python_version%.*}"*.so.1.0 /tmp' + if ! ls python-binaries/${{ matrix.arch }}/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0 1> /dev/null 2>&1; then + mkdir -p python-binaries/${{ matrix.arch }}/${{ matrix.python-version }} + docker run --rm --platform "linux/${{ matrix.arch }}" -v "${PWD}"/python-binaries/${{ matrix.arch }}/${{ matrix.python-version }}:/tmp -w /tmp docker.io/library/python:${{ matrix.python-version }} bash -c 'cp /usr/local/lib/libpython"${python_version%.*}"*.so.1.0 /tmp' fi - mkdir -p offsets-python - ./structlayout -r python -v ${{ matrix.python-version }} -o offsets-python python-binaries/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0 + mkdir -p offsets-python/${{ matrix.arch }} + ./structlayout -r python -v ${{ matrix.python-version }} -o offsets-python/${{ matrix.arch }} python-binaries/${{ matrix.arch }}/${{ matrix.python-version }}/libpython"${python_version%.*}"*.so.1.0 git add offsets-python @@ -96,7 +102,7 @@ jobs: uses: actions/upload-artifact@v4 with: if-no-files-found: ignore - name: python-${{ matrix.python-version }} + name: python-${{ matrix-arch }}-${{ matrix.python-version }} path: offsets-python retention-days: 1 @@ -143,10 +149,20 @@ jobs: - name: Merge the layout files run: | - rm pkg/python/layout/* - rm pkg/python/initialstate/* - ./mergelayout -o pkg/python/layout offsets-python/layout/python_*.yaml - ./mergelayout -o pkg/python/initialstate offsets-python/initialstate/python_*.yaml + target_archs=( + amd64 + arm64 + ) + + rm -rf pkg/python/layout + rm -rf pkg/python/initialstate + for arch in "${target_archs[@]}"; do + mkdir -p pkg/python/layout/"${arch}" + ./mergelayout -o pkg/python/layout/"${arch}" offsets-python/"${arch}"/layout/python_*.yaml + + mkdir -p pkg/python/initialstate"/${arch}" + ./mergelayout -o pkg/python/initialstate/"${arch}" offsets-python/"${arch}"/initialstate/python_*.yaml + done # If there are no changes (i.e. no diff exists with the checked-out base branch), # no pull request will be created and the action exits silently. diff --git a/.github/workflows/generate-ruby.yml b/.github/workflows/generate-ruby.yml index 43ee810..5d131d2 100644 --- a/.github/workflows/generate-ruby.yml +++ b/.github/workflows/generate-ruby.yml @@ -29,8 +29,10 @@ jobs: 2.7.0, 2.7.1, 2.7.2, 2.7.3, 2.7.4, 2.7.5, 2.7.6, 2.7.7, 2.7.8, 3.0.0, 3.0.1, 3.0.2, 3.0.3, 3.0.4, 3.0.5, 3.0.6, 3.1.0, 3.1.1, 3.1.2, 3.1.3, 3.1.4, - 3.2.0, 3.2.1, 3.2.2, 3.2.3, 3.3.0 + 3.2.0, 3.2.1, 3.2.2, 3.2.3, + 3.3.0 ] + arch: [amd64, arm64] steps: - name: Check out the code uses: actions/checkout@v4.1.1 @@ -60,26 +62,31 @@ jobs: - name: Set up container image for generate uses: ScribeMD/docker-cache@0.3.7 with: - key: docker-ruby-${{ runner.os }}-${{ runner.arch }} + key: docker-ruby-${{ runner.os }}-${{ matrix.arch }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 - name: Set up Layout cache for generated files uses: actions/cache@v4.0.0 with: path: offsets-ruby - key: ruby-generated-${{ runner.os }}-${{ runner.arch }} + key: ruby-generated-${{ runner.os }}-${{ matrix.arch }} restore-keys: | - ruby-generated-${{ runner.os }}-${{ runner.arch }}-${{ matrix.ruby-version }} - ruby-generated-${{ runner.os }}-${{ runner.arch }} + ruby-generated-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.ruby-version }} + ruby-generated-${{ runner.os }}-${{ matrix.arch }} - - name: Generate Ruby Offsets for ${{ matrix.ruby-version }} + - name: Generate Ruby Offsets for ${{ matrix.ruby-version }} on ${{ matrix.arch }} run: | - if ! ls "${PWD}"/ruby-binaries/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }} 1>/dev/null 2>&1; then - mkdir -p "${PWD}"/ruby-binaries/${{ matrix.ruby-version }} - docker run --rm -v "${PWD}"/ruby-binaries/${{ matrix.ruby-version }}:/tmp -w /tmp docker.io/library/ruby:${{ matrix.ruby-version }}-slim bash -c 'cp /usr/local/lib/libruby.so.${{ matrix.ruby-version }} /tmp' + if ! ls "${PWD}"/ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }} 1>/dev/null 2>&1; then + mkdir -p "${PWD}"/ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }} + docker run --rm --platform "linux/${{ matrix.arch }}" -v "${PWD}"/ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }}:/tmp -w /tmp docker.io/library/ruby:${{ matrix.ruby-version }}-slim bash -c 'cp /usr/local/lib/libruby.so.${{ matrix.ruby-version }} /tmp' fi - mkdir -p offsets-ruby - ./structlayout -r ruby -v ${{ matrix.ruby-version }} -o offsets-ruby ruby-binaries/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }} + mkdir -p offsets-ruby/${{ matrix.arch }}/ + ./structlayout -r ruby -v ${{ matrix.ruby-version }} -o offsets-ruby/${{ matrix.arch }} ruby-binaries/${{ matrix.arch }}/${{ matrix.ruby-version }}/libruby.so.${{ matrix.ruby-version }} git add offsets-ruby @@ -89,7 +96,7 @@ jobs: uses: actions/upload-artifact@v4 with: if-no-files-found: ignore - name: ruby-${{ matrix.ruby-version }} + name: ruby-${{ matrix.arch }}-${{ matrix.ruby-version }} path: offsets-ruby retention-days: 1 @@ -136,8 +143,15 @@ jobs: - name: Merge the layout files run: | - rm pkg/ruby/layout/* - ./mergelayout -o pkg/ruby/layout offsets-ruby/layout/ruby_*.yaml + target_archs=( + amd64 + arm64 + ) + rm -rf pkg/ruby/layout + for arch in "${target_archs[@]}"; do + mkdir -p pkg/ruby/layout/"${arch}" + ./mergelayout -o pkg/ruby/layout/"${arch}" offsets-ruby/"${arch}"/layout/ruby_*.yaml + done # If there are no changes (i.e. no diff exists with the checked-out base branch), # no pull request will be created and the action exits silently. diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 49d08c1..e117b0d 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -49,4 +49,4 @@ jobs: ${{ runner.arch }}-integration-runtimes - name: Run integration tests - run: make test/integration + run: make ARCH="amd64" generate test/integration diff --git a/Makefile b/Makefile index 6709c9d..e49f2ac 100644 --- a/Makefile +++ b/Makefile @@ -58,13 +58,15 @@ tagalign: go run github.com/4meepo/tagalign/cmd/tagalign@latest -fix -sort ./... .PHONY: test -test: generate +test: build go test ./... .PHONY: test/integration -test/integration: generate +test/integration: build + @echo "Make sure to run 'generate' if any generation code has changed" go test -tags=integration ./tests/integration/... + .PHONY: test/integration/update test/integration/update: go test -count=1 -race -tags=integration ./tests/integration/... -update diff --git a/README.md b/README.md index d19c919..a91d843 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,26 @@ This project is a collection of tools to extract runtime data from several interpreters and runtimes. By runtime data, we mean information about the execution of a program, especially stack unwinding and profiling. +## Requirements + +- [Devbox](https://jetpack.io/devbox) + - Also see [CONTRIBUTING.md](CONTRIBUTING.md) for more information about the development environment. +- [direnv](https://direnv.net/) + +```sh +make bootstrap +``` + +- A container runtime (e.g. Docker, Podman) + - Ability to run cross-platform containers (e.g. Docker Desktop, Podman with QEMU) + +```sh +sudo apt-get install qemu binfmt-support qemu-user-static # Install the qemu packages +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # This step will execute the registering scripts + +docker run --rm -t arm64v8/ubuntu uname -m +``` + ## Usage ### Python diff --git a/pkg/python/initialstate/.gitkeep b/pkg/python/initialstate/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/pkg/python/initialstate/amd64/3.7.0 - 3.7.2.yaml b/pkg/python/initialstate/amd64/3.7.0 - 3.7.2.yaml new file mode 100644 index 0000000..9edd58a --- /dev/null +++ b/pkg/python/initialstate/amd64/3.7.0 - 3.7.2.yaml @@ -0,0 +1,2 @@ +interpreter_head: 24 +tstate_current: 1392 diff --git a/pkg/python/initialstate/amd64/3.8.0 - 3.8.1.yaml b/pkg/python/initialstate/amd64/3.8.0 - 3.8.1.yaml new file mode 100644 index 0000000..7b44a6b --- /dev/null +++ b/pkg/python/initialstate/amd64/3.8.0 - 3.8.1.yaml @@ -0,0 +1,2 @@ +interpreter_head: 32 +tstate_current: 1368 diff --git a/pkg/python/initialstate/amd64/3.9.5 - 3.10.0.yaml b/pkg/python/initialstate/amd64/3.9.5 - 3.10.0.yaml new file mode 100644 index 0000000..cfeb1a7 --- /dev/null +++ b/pkg/python/initialstate/amd64/3.9.5 - 3.10.0.yaml @@ -0,0 +1,2 @@ +interpreter_head: 32 +tstate_current: 568 diff --git a/pkg/python/initialstate/amd64/= 3.11.0.yaml b/pkg/python/initialstate/amd64/= 3.11.0.yaml new file mode 100644 index 0000000..a08c9a9 --- /dev/null +++ b/pkg/python/initialstate/amd64/= 3.11.0.yaml @@ -0,0 +1,2 @@ +interpreter_head: 40 +tstate_current: 576 diff --git a/pkg/python/initialstate/amd64/= 3.7.4.yaml b/pkg/python/initialstate/amd64/= 3.7.4.yaml new file mode 100644 index 0000000..e9b6a4d --- /dev/null +++ b/pkg/python/initialstate/amd64/= 3.7.4.yaml @@ -0,0 +1,2 @@ +interpreter_head: 24 +tstate_current: 1480 diff --git a/pkg/python/initialstate/arm64/3.7.0 - 3.7.2.yaml b/pkg/python/initialstate/arm64/3.7.0 - 3.7.2.yaml new file mode 100644 index 0000000..b8f22df --- /dev/null +++ b/pkg/python/initialstate/arm64/3.7.0 - 3.7.2.yaml @@ -0,0 +1,2 @@ +interpreter_head: 24 +tstate_current: 1408 diff --git a/pkg/python/initialstate/arm64/3.8.0 - 3.8.1.yaml b/pkg/python/initialstate/arm64/3.8.0 - 3.8.1.yaml new file mode 100644 index 0000000..b1220ea --- /dev/null +++ b/pkg/python/initialstate/arm64/3.8.0 - 3.8.1.yaml @@ -0,0 +1,2 @@ +interpreter_head: 32 +tstate_current: 1384 diff --git a/pkg/python/initialstate/arm64/3.9.5 - 3.10.0.yaml b/pkg/python/initialstate/arm64/3.9.5 - 3.10.0.yaml new file mode 100644 index 0000000..93ec777 --- /dev/null +++ b/pkg/python/initialstate/arm64/3.9.5 - 3.10.0.yaml @@ -0,0 +1,2 @@ +interpreter_head: 32 +tstate_current: 584 diff --git a/pkg/python/initialstate/arm64/= 3.11.0.yaml b/pkg/python/initialstate/arm64/= 3.11.0.yaml new file mode 100644 index 0000000..727fe43 --- /dev/null +++ b/pkg/python/initialstate/arm64/= 3.11.0.yaml @@ -0,0 +1,2 @@ +interpreter_head: 40 +tstate_current: 592 diff --git a/pkg/python/initialstate/arm64/= 3.7.4.yaml b/pkg/python/initialstate/arm64/= 3.7.4.yaml new file mode 100644 index 0000000..3eb63a0 --- /dev/null +++ b/pkg/python/initialstate/arm64/= 3.7.4.yaml @@ -0,0 +1,2 @@ +interpreter_head: 24 +tstate_current: 1496 diff --git a/pkg/python/layout/.gitkeep b/pkg/python/layout/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/pkg/python/layout/amd64/2.7.15 - 2.7.18.yaml b/pkg/python/layout/amd64/2.7.15 - 2.7.18.yaml new file mode 100644 index 0000000..65595d6 --- /dev/null +++ b/pkg/python/layout/amd64/2.7.15 - 2.7.18.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 80 + co_firstlineno: 96 + co_name: 88 + co_varnames: 56 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 36 + size: 16 +py_thread_state: + cframe: -1 + frame: 16 + interp: 8 + native_thread_id: -1 + next: 0 + thread_id: 144 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/amd64/3.4.8 - 3.5.5.yaml b/pkg/python/layout/amd64/3.4.8 - 3.5.5.yaml new file mode 100644 index 0000000..b96494a --- /dev/null +++ b/pkg/python/layout/amd64/3.4.8 - 3.5.5.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 112 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 152 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/amd64/3.7.0 - 3.7.4.yaml b/pkg/python/layout/amd64/3.7.0 - 3.7.4.yaml new file mode 100644 index 0000000..89efcd5 --- /dev/null +++ b/pkg/python/layout/amd64/3.7.0 - 3.7.4.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 36 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 108 + f_localsplus: 360 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 176 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/amd64/3.8.0 - 3.9.6.yaml b/pkg/python/layout/amd64/3.8.0 - 3.9.6.yaml new file mode 100644 index 0000000..222ec17 --- /dev/null +++ b/pkg/python/layout/amd64/3.8.0 - 3.9.6.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 104 + co_firstlineno: 40 + co_name: 112 + co_varnames: 72 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 108 + f_localsplus: 360 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 176 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/amd64/= 3.10.0.yaml b/pkg/python/layout/amd64/= 3.10.0.yaml new file mode 100644 index 0000000..e9187e7 --- /dev/null +++ b/pkg/python/layout/amd64/= 3.10.0.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 104 + co_firstlineno: 40 + co_name: 112 + co_varnames: 72 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 100 + f_localsplus: 352 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: -1 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 176 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/amd64/= 3.11.0.yaml b/pkg/python/layout/amd64/= 3.11.0.yaml new file mode 100644 index 0000000..d710c1b --- /dev/null +++ b/pkg/python/layout/amd64/= 3.11.0.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 8 +py_code_object: + co_filename: 112 + co_firstlineno: 72 + co_name: 120 + co_varnames: 96 +py_frame_object: + f_back: 48 + f_code: 32 + f_lineno: -1 + f_localsplus: 72 +py_interpreter_state: + tstate_head: 16 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: 48 +py_string: + data: 48 + size: -1 +py_thread_state: + cframe: 56 + frame: -1 + interp: 16 + native_thread_id: 160 + next: 8 + thread_id: 152 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/amd64/= 3.3.7.yaml b/pkg/python/layout/amd64/= 3.3.7.yaml new file mode 100644 index 0000000..c70fe1d --- /dev/null +++ b/pkg/python/layout/amd64/= 3.3.7.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 112 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 16 + interp: 8 + native_thread_id: -1 + next: 0 + thread_id: 144 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/amd64/= 3.6.6.yaml b/pkg/python/layout/amd64/= 3.6.6.yaml new file mode 100644 index 0000000..d91c24d --- /dev/null +++ b/pkg/python/layout/amd64/= 3.6.6.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 36 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 152 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/2.7.15 - 2.7.18.yaml b/pkg/python/layout/arm64/2.7.15 - 2.7.18.yaml new file mode 100644 index 0000000..65595d6 --- /dev/null +++ b/pkg/python/layout/arm64/2.7.15 - 2.7.18.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 80 + co_firstlineno: 96 + co_name: 88 + co_varnames: 56 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 36 + size: 16 +py_thread_state: + cframe: -1 + frame: 16 + interp: 8 + native_thread_id: -1 + next: 0 + thread_id: 144 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/3.4.8 - 3.5.5.yaml b/pkg/python/layout/arm64/3.4.8 - 3.5.5.yaml new file mode 100644 index 0000000..b96494a --- /dev/null +++ b/pkg/python/layout/arm64/3.4.8 - 3.5.5.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 112 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 152 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/3.7.0 - 3.7.4.yaml b/pkg/python/layout/arm64/3.7.0 - 3.7.4.yaml new file mode 100644 index 0000000..89efcd5 --- /dev/null +++ b/pkg/python/layout/arm64/3.7.0 - 3.7.4.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 36 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 108 + f_localsplus: 360 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 176 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/3.8.0 - 3.9.6.yaml b/pkg/python/layout/arm64/3.8.0 - 3.9.6.yaml new file mode 100644 index 0000000..222ec17 --- /dev/null +++ b/pkg/python/layout/arm64/3.8.0 - 3.9.6.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 104 + co_firstlineno: 40 + co_name: 112 + co_varnames: 72 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 108 + f_localsplus: 360 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 176 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/= 3.10.0.yaml b/pkg/python/layout/arm64/= 3.10.0.yaml new file mode 100644 index 0000000..e9187e7 --- /dev/null +++ b/pkg/python/layout/arm64/= 3.10.0.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 104 + co_firstlineno: 40 + co_name: 112 + co_varnames: 72 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 100 + f_localsplus: 352 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: -1 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 176 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/= 3.11.0.yaml b/pkg/python/layout/arm64/= 3.11.0.yaml new file mode 100644 index 0000000..d710c1b --- /dev/null +++ b/pkg/python/layout/arm64/= 3.11.0.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 8 +py_code_object: + co_filename: 112 + co_firstlineno: 72 + co_name: 120 + co_varnames: 96 +py_frame_object: + f_back: 48 + f_code: 32 + f_lineno: -1 + f_localsplus: 72 +py_interpreter_state: + tstate_head: 16 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: 48 +py_string: + data: 48 + size: -1 +py_thread_state: + cframe: 56 + frame: -1 + interp: 16 + native_thread_id: 160 + next: 8 + thread_id: 152 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/= 3.3.7.yaml b/pkg/python/layout/arm64/= 3.3.7.yaml new file mode 100644 index 0000000..c70fe1d --- /dev/null +++ b/pkg/python/layout/arm64/= 3.3.7.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 112 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 16 + interp: 8 + native_thread_id: -1 + next: 0 + thread_id: 144 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/layout/arm64/= 3.6.6.yaml b/pkg/python/layout/arm64/= 3.6.6.yaml new file mode 100644 index 0000000..d91c24d --- /dev/null +++ b/pkg/python/layout/arm64/= 3.6.6.yaml @@ -0,0 +1,32 @@ +py_cframe: + current_frame: 0 +py_code_object: + co_filename: 96 + co_firstlineno: 36 + co_name: 104 + co_varnames: 64 +py_frame_object: + f_back: 24 + f_code: 32 + f_lineno: 124 + f_localsplus: 376 +py_interpreter_state: + tstate_head: 8 +py_object: + ob_type: 8 +py_runtime_state: + interp_main: -1 +py_string: + data: 48 + size: 16 +py_thread_state: + cframe: -1 + frame: 24 + interp: 16 + native_thread_id: -1 + next: 8 + thread_id: 152 +py_tuple_object: + ob_item: 24 +py_type_object: + tp_name: 24 diff --git a/pkg/python/python.go b/pkg/python/python.go index 82e0b1a..aaf15d9 100644 --- a/pkg/python/python.go +++ b/pkg/python/python.go @@ -16,6 +16,7 @@ import ( "embed" "errors" "path/filepath" + "runtime" "strings" "sync" @@ -24,8 +25,8 @@ import ( ) const ( - layouts = "layout" - initialState = "initialstate" + layoutDir = "layout" + initialStateDir = "initialstate" ) type Key struct { @@ -34,7 +35,7 @@ type Key struct { } var ( - //go:embed layout/*.yaml + //go:embed layout/*/*.yaml generatedLayouts embed.FS //go:embed initialstate/* generatedState embed.FS @@ -54,7 +55,7 @@ func init() { func loadLayouts() (map[Key]*Layout, error) { var err error once.Do(func() { - entries, err := generatedLayouts.ReadDir(layouts) + entries, err := generatedLayouts.ReadDir(filepath.Join(layoutDir, runtime.GOARCH)) if err != nil { return } @@ -64,7 +65,7 @@ func loadLayouts() (map[Key]*Layout, error) { continue } var data []byte - data, err = generatedLayouts.ReadFile(filepath.Join(layouts, entry.Name())) + data, err = generatedLayouts.ReadFile(filepath.Join(layoutDir, runtime.GOARCH, entry.Name())) if err != nil { return } @@ -90,6 +91,44 @@ func loadLayouts() (map[Key]*Layout, error) { return structLayouts, err } +func getLayoutForArch(v *semver.Version, arch string) (Key, *Layout, error) { + entries, err := generatedLayouts.ReadDir(filepath.Join(layoutDir, arch)) + if err != nil { + return Key{}, nil, err + } + var i int + for _, entry := range entries { + if entry.IsDir() { + continue + } + var data []byte + data, err = generatedLayouts.ReadFile(filepath.Join(layoutDir, arch, entry.Name())) + if err != nil { + return Key{}, nil, err + } + ext := filepath.Ext(entry.Name()) + // Filter out non-yaml files. + if ext != ".yaml" && ext != ".yml" { + continue + } + var lyt Layout + if err = yaml.Unmarshal(data, &lyt); err != nil { + return Key{}, nil, err + } + rawConstraint := strings.TrimSuffix(entry.Name(), ext) + constr, err := semver.NewConstraint(rawConstraint) + if err != nil { + return Key{}, nil, err + } + if constr.Check(v) { + key := Key{index: i, constraint: constr.String()} + return key, &lyt, nil + } + i++ + } + return Key{}, nil, errors.New("not found") +} + // GetLayout returns the matching layout for the given version. func GetLayout(v *semver.Version) (Key, *Layout, error) { layouts, err := loadLayouts() @@ -119,7 +158,7 @@ func GetLayouts() (map[Key]*Layout, error) { // loadInitialState loads the initial state for the supported versions. func loadInitialState() (map[Key]*InitialState, error) { - entries, err := generatedState.ReadDir(initialState) + entries, err := generatedState.ReadDir(filepath.Join(initialStateDir, runtime.GOARCH)) if err != nil { return nil, err } @@ -128,7 +167,7 @@ func loadInitialState() (map[Key]*InitialState, error) { if entry.IsDir() { continue } - data, err := generatedState.ReadFile(filepath.Join(initialState, entry.Name())) + data, err := generatedState.ReadFile(filepath.Join(initialStateDir, runtime.GOARCH, entry.Name())) if err != nil { return nil, err } @@ -152,6 +191,41 @@ func loadInitialState() (map[Key]*InitialState, error) { return initialStates, nil } +func getInitialStateForArch(v *semver.Version, arch string) (Key, *InitialState, error) { + entries, err := generatedState.ReadDir(filepath.Join(initialStateDir, arch)) + if err != nil { + return Key{}, nil, err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + data, err := generatedState.ReadFile(filepath.Join(initialStateDir, arch, entry.Name())) + if err != nil { + return Key{}, nil, err + } + ext := filepath.Ext(entry.Name()) + // Filter out non-yaml files. + if ext != ".yaml" && ext != ".yml" { + continue + } + var initState InitialState + if err := yaml.Unmarshal(data, &initState); err != nil { + return Key{}, nil, err + } + rawConstraint := strings.TrimSuffix(entry.Name(), ext) + constr, err := semver.NewConstraint(rawConstraint) + if err != nil { + return Key{}, nil, err + } + if constr.Check(v) { + key := Key{constraint: constr.String()} + return key, &initState, nil + } + } + return Key{}, nil, errors.New("not found") +} + // GetInitialState returns the initial state for the given version. func GetInitialState(v *semver.Version) (Key, *InitialState, error) { state, err := loadInitialState() diff --git a/pkg/python/python_test.go b/pkg/python/python_test.go index 144b6b6..7ed110a 100644 --- a/pkg/python/python_test.go +++ b/pkg/python/python_test.go @@ -13,6 +13,8 @@ package python import ( + "fmt" + "runtime" "testing" "github.com/Masterminds/semver/v3" @@ -27,14 +29,18 @@ func TestGetLayouts(t *testing.T) { t.Log(layouts) } +var allSupportedArchs = []string{"amd64", "arm64"} + func TestGetLayout(t *testing.T) { tests := []struct { version string + archs []string want *Layout wantErr bool }{ { version: "2.7.15", + archs: allSupportedArchs, want: &Layout{ PyCodeObject: PyCodeObject{ CoFilename: 80, @@ -63,6 +69,7 @@ func TestGetLayout(t *testing.T) { }, { version: "3.6.6", + archs: allSupportedArchs, want: &Layout{ PyCodeObject: PyCodeObject{ CoFilename: 96, @@ -97,6 +104,7 @@ func TestGetLayout(t *testing.T) { }, { version: "3.11.0", + archs: allSupportedArchs, want: &Layout{ PyObject: PyObject{ ObType: 8, @@ -144,51 +152,54 @@ func TestGetLayout(t *testing.T) { }, } for _, tt := range tests { - t.Run(tt.version, func(t *testing.T) { - version, err := semver.StrictNewVersion(tt.version) - if err != nil { - t.Errorf("StrictNewVersion() error = %v", err) - return - } - _, got, err := GetLayout(version) - if (err != nil) != tt.wantErr { - t.Errorf("GetLayout() error = %v, wantErr %v", err, tt.wantErr) - return - } - if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(Layout{})); diff != "" { - t.Errorf("GetLayout() mismatch (-want +got):\n%s", diff) - } - }) + var ( + version = tt.version + want = tt.want + ) + for _, arch := range tt.archs { + arch := arch + t.Run(version, func(t *testing.T) { + v, err := semver.StrictNewVersion(version) + if err != nil { + t.Errorf("StrictNewVersion() error = %v", err) + return + } + _, got, err := getLayoutForArch(v, arch) + if (err != nil) != tt.wantErr { + t.Errorf("GetLayout(%s on %s) error = %v, wantErr %v", version, arch, err, tt.wantErr) + return + } + if diff := cmp.Diff(want, got, cmp.AllowUnexported(Layout{})); diff != "" { + t.Errorf("GetLayout(%s on %s) mismatch (-want +got):\n%s", version, arch, diff) + } + }) + } } } func TestGetInitialState(t *testing.T) { tests := []struct { - name string version string + arch string want *InitialState wantErr bool }{ { - name: "2.7.15", version: "2.7.15", want: nil, wantErr: true, }, { - name: "3.3.7", version: "3.3.7", want: nil, wantErr: true, }, { - name: "3.6.6", version: "3.6.6", want: nil, wantErr: true, }, { - name: "3.7.2", version: "3.7.2", want: &InitialState{ InterpreterHead: 24, @@ -196,7 +207,6 @@ func TestGetInitialState(t *testing.T) { }, }, { - name: "3.7.4", version: "3.7.4", want: &InitialState{ InterpreterHead: 24, @@ -204,7 +214,6 @@ func TestGetInitialState(t *testing.T) { }, }, { - name: "3.8.0", version: "3.8.0", want: &InitialState{ InterpreterHead: 32, @@ -212,7 +221,6 @@ func TestGetInitialState(t *testing.T) { }, }, { - name: "3.9.6", version: "3.9.6", want: &InitialState{ InterpreterHead: 32, @@ -220,7 +228,6 @@ func TestGetInitialState(t *testing.T) { }, }, { - name: "3.10.0", version: "3.10.0", want: &InitialState{ InterpreterHead: 32, @@ -228,28 +235,103 @@ func TestGetInitialState(t *testing.T) { }, }, { - name: "3.11.0", version: "3.11.0", want: &InitialState{ InterpreterHead: 40, ThreadStateCurrent: 576, }, }, + // arm64 + { + version: "2.7.15", + arch: "arm64", + want: nil, + wantErr: true, + }, + { + version: "3.3.7", + arch: "arm64", + want: nil, + wantErr: true, + }, + { + version: "3.6.6", + arch: "arm64", + want: nil, + wantErr: true, + }, + { + version: "3.7.2", + arch: "arm64", + want: &InitialState{ + InterpreterHead: 24, + ThreadStateCurrent: 1408, + }, + }, + { + version: "3.7.4", + arch: "arm64", + want: &InitialState{ + InterpreterHead: 24, + ThreadStateCurrent: 1496, + }, + }, + { + version: "3.8.0", + arch: "arm64", + want: &InitialState{ + InterpreterHead: 32, + ThreadStateCurrent: 1384, + }, + }, + { + version: "3.9.6", + arch: "arm64", + want: &InitialState{ + InterpreterHead: 32, + ThreadStateCurrent: 584, + }, + }, + { + version: "3.10.0", + arch: "arm64", + want: &InitialState{ + InterpreterHead: 32, + ThreadStateCurrent: 584, + }, + }, + { + version: "3.11.0", + arch: "arm64", + want: &InitialState{ + InterpreterHead: 40, + ThreadStateCurrent: 592, + }, + }, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - v, err := semver.StrictNewVersion(tt.version) + var ( + arch = tt.arch + version = tt.version + want = tt.want + ) + if tt.arch == "" { + arch = runtime.GOARCH + } + name := fmt.Sprintf("%s on %s", tt.version, arch) + t.Run(name, func(t *testing.T) { + v, err := semver.StrictNewVersion(version) if err != nil { t.Errorf("StrictNewVersion() error = %v", err) return } - _, got, err := GetInitialState(v) + _, got, err := getInitialStateForArch(v, arch) if (err != nil) != tt.wantErr { - t.Errorf("GetInitialState(%s) error = %v, wantErr %v", tt.name, err, tt.wantErr) + t.Errorf("GetInitialState(%s) error = %v, wantErr %v", name, err, tt.wantErr) return } - if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(InitialState{})); diff != "" { - t.Errorf("GetInitialState(%s) mismatch (-want +got):\n%s", tt.name, diff) + if diff := cmp.Diff(want, got, cmp.AllowUnexported(InitialState{})); diff != "" { + t.Errorf("GetInitialState(%s) mismatch (-want +got):\n%s", name, diff) } }) } diff --git a/pkg/ruby/layout/.gitkeep b/pkg/ruby/layout/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/pkg/ruby/layout/amd64/2.6.0 - 2.7.6.yaml b/pkg/ruby/layout/amd64/2.6.0 - 2.7.6.yaml new file mode 100644 index 0000000..89eb01d --- /dev/null +++ b/pkg/ruby/layout/amd64/2.6.0 - 2.7.6.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 56 +ec_offset: 32 +label_offset: 16 +line_info_size_offset: 136 +line_info_table_offset: 120 +lineno_offset: 0 +main_thread_offset: 192 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/layout/amd64/3.0.0 - 3.0.4.yaml b/pkg/ruby/layout/amd64/3.0.0 - 3.0.4.yaml new file mode 100644 index 0000000..fefa327 --- /dev/null +++ b/pkg/ruby/layout/amd64/3.0.0 - 3.0.4.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 56 +ec_offset: 520 +label_offset: 16 +line_info_size_offset: 136 +line_info_table_offset: 120 +lineno_offset: 0 +main_thread_offset: 40 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/layout/amd64/3.1.2 - 3.1.3.yaml b/pkg/ruby/layout/amd64/3.1.2 - 3.1.3.yaml new file mode 100644 index 0000000..7b3cd1a --- /dev/null +++ b/pkg/ruby/layout/amd64/3.1.2 - 3.1.3.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 64 +ec_offset: 520 +label_offset: 16 +line_info_size_offset: 136 +line_info_table_offset: 120 +lineno_offset: 0 +main_thread_offset: 40 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/layout/amd64/3.2.0 - 3.2.1.yaml b/pkg/ruby/layout/amd64/3.2.0 - 3.2.1.yaml new file mode 100644 index 0000000..f333f4c --- /dev/null +++ b/pkg/ruby/layout/amd64/3.2.0 - 3.2.1.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 64 +ec_offset: 520 +label_offset: 16 +line_info_size_offset: 128 +line_info_table_offset: 112 +lineno_offset: 0 +main_thread_offset: 40 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/layout/arm64/2.6.0 - 2.7.6.yaml b/pkg/ruby/layout/arm64/2.6.0 - 2.7.6.yaml new file mode 100644 index 0000000..36feea7 --- /dev/null +++ b/pkg/ruby/layout/arm64/2.6.0 - 2.7.6.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 56 +ec_offset: 32 +label_offset: 16 +line_info_size_offset: 136 +line_info_table_offset: 120 +lineno_offset: 0 +main_thread_offset: 200 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/layout/arm64/3.0.0 - 3.0.4.yaml b/pkg/ruby/layout/arm64/3.0.0 - 3.0.4.yaml new file mode 100644 index 0000000..99ec89c --- /dev/null +++ b/pkg/ruby/layout/arm64/3.0.0 - 3.0.4.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 56 +ec_offset: 536 +label_offset: 16 +line_info_size_offset: 136 +line_info_table_offset: 120 +lineno_offset: 0 +main_thread_offset: 40 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/layout/arm64/3.1.2 - 3.1.3.yaml b/pkg/ruby/layout/arm64/3.1.2 - 3.1.3.yaml new file mode 100644 index 0000000..32868b1 --- /dev/null +++ b/pkg/ruby/layout/arm64/3.1.2 - 3.1.3.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 64 +ec_offset: 536 +label_offset: 16 +line_info_size_offset: 136 +line_info_table_offset: 120 +lineno_offset: 0 +main_thread_offset: 40 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/layout/arm64/3.2.0 - 3.2.1.yaml b/pkg/ruby/layout/arm64/3.2.0 - 3.2.1.yaml new file mode 100644 index 0000000..f55f54e --- /dev/null +++ b/pkg/ruby/layout/arm64/3.2.0 - 3.2.1.yaml @@ -0,0 +1,11 @@ +cfp_offset: 16 +control_frame_t_sizeof: 64 +ec_offset: 536 +label_offset: 16 +line_info_size_offset: 128 +line_info_table_offset: 112 +lineno_offset: 0 +main_thread_offset: 40 +path_flavour: 1 +vm_offset: 0 +vm_size_offset: 8 diff --git a/pkg/ruby/ruby.go b/pkg/ruby/ruby.go index d93dce9..6c70044 100644 --- a/pkg/ruby/ruby.go +++ b/pkg/ruby/ruby.go @@ -17,6 +17,7 @@ import ( "embed" "errors" "path/filepath" + "runtime" "strings" "sync" @@ -29,10 +30,10 @@ type Key struct { constraint string } -const versionDir = "layout" +const layoutDir = "layout" var ( - //go:embed layout/*.yaml + //go:embed layout/*/*.yaml generatedLayouts embed.FS structLayouts = map[Key]*Layout{} once = &sync.Once{} @@ -49,7 +50,7 @@ func init() { func loadLayouts() (map[Key]*Layout, error) { var err error once.Do(func() { - entries, err := generatedLayouts.ReadDir(versionDir) + entries, err := generatedLayouts.ReadDir(filepath.Join(layoutDir, runtime.GOARCH)) if err != nil { return } @@ -59,7 +60,7 @@ func loadLayouts() (map[Key]*Layout, error) { continue } var data []byte - data, err = generatedLayouts.ReadFile(filepath.Join(versionDir, entry.Name())) + data, err = generatedLayouts.ReadFile(filepath.Join(layoutDir, runtime.GOARCH, entry.Name())) if err != nil { return } diff --git a/scripts/download/python.sh b/scripts/download/python.sh index 73f1cf9..2f44be0 100755 --- a/scripts/download/python.sh +++ b/scripts/download/python.sh @@ -20,6 +20,7 @@ set -euo pipefail CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-docker} TARGET_DIR=${TARGET_DIR:-tests/integration/binaries/python} +ARCH=${ARCH} # https://devguide.python.org/versions/ python_versions=( @@ -42,6 +43,14 @@ python_versions=( 3.11.0 # bugfix ) +target_archs=( + amd64 + arm64 +) +if [ -n "${ARCH}" ]; then + target_archs=("${ARCH}") +fi + # Check if CONTAINER_RUNTIME is installed. if ! command -v "${CONTAINER_RUNTIME}" &>/dev/null; then echo "ERROR: ${CONTAINER_RUNTIME} is not installed." @@ -50,18 +59,27 @@ fi # Install libpython for each python version under python_runtimes directory. for python_version in "${python_versions[@]}"; do - echo "Checking if python ${python_version} runtime is already downloaded..." - if ls "${PWD}"/"${TARGET_DIR}"/${python_version}/libpython"${python_version%.*}"*.so.1.0 1> /dev/null 2>&1; then - echo "Python ${python_version} runtime is already downloaded." - continue - fi - echo "Downloading python ${python_version} runtime..." - "${CONTAINER_RUNTIME}" run --rm -v "${PWD}"/"${TARGET_DIR}"/${python_version}:/tmp -w /tmp python:${python_version} bash -c 'cp /usr/local/lib/libpython"${python_version%.*}"*.so.1.0 /tmp' - echo "Changing the owner of the file to the current user..." - sudo chown -R $(whoami) "${TARGET_DIR}" - echo "Done." + for arch in "${target_archs[@]}"; do + target="${PWD}"/"${TARGET_DIR}"/"${arch}"/${python_version} + echo "Checking if python ${python_version} runtime for ${arch} is already downloaded..." + if ls "${target}"/libpython"${python_version%.*}"*.so.1.0 1>/dev/null 2>&1; then + echo "Python ${python_version} runtime for ${arch} is already downloaded." + continue + fi + echo "Downloading python ${python_version} runtime..." + "${CONTAINER_RUNTIME}" run --rm \ + --platform "linux/${arch}" \ + -v "${target}":/tmp -w /tmp \ + python:${python_version} \ + bash -c 'cp /usr/local/lib/libpython"${python_version%.*}"*.so.1.0 \ + /tmp' + echo "Changing the owner of the file to the current user..." + sudo chown -R $(whoami) "${TARGET_DIR}" + echo "Done." + for i in "${target}"/libpython*; do file "$i"; done + done done echo "All python runtimes downloaded successfully." - -for i in "${TARGET_DIR}"/libpython*; do file "$i"; done +dir="${PWD}"/"${TARGET_DIR}"/*/*/libpython* +for i in $dir; do file "$i"; done diff --git a/scripts/download/ruby.sh b/scripts/download/ruby.sh index 9042c31..3468e59 100755 --- a/scripts/download/ruby.sh +++ b/scripts/download/ruby.sh @@ -20,6 +20,7 @@ set -euo pipefail CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-docker} TARGET_DIR=${TARGET_DIR:-tests/integration/binaries/ruby} +ARCH=${ARCH} ruby_versions=( 2.6.0 @@ -35,6 +36,14 @@ ruby_versions=( 3.2.1 ) +target_archs=( + amd64 + arm64 +) +if [ -n "${ARCH}" ]; then + target_archs=("${ARCH}") +fi + # Check if CONTAINER_RUNTIME is installed. if ! command -v "${CONTAINER_RUNTIME}" &>/dev/null; then echo "ERROR: ${CONTAINER_RUNTIME} is not installed." @@ -43,18 +52,27 @@ fi # Install libpython for each ruby version under python_runtimes directory. for ruby_version in "${ruby_versions[@]}"; do - echo "Checking if ruby ${ruby_version} runtime is already downloaded..." - if ls "${PWD}"/"${TARGET_DIR}"/libruby.so.${ruby_version} 1>/dev/null 2>&1; then - echo "Ruby ${ruby_version} runtime is already downloaded." - continue - fi - echo "Downloading ruby ${ruby_version} runtime..." - "${CONTAINER_RUNTIME}" run --rm -v "${PWD}"/"${TARGET_DIR}":/tmp -w /tmp ruby:${ruby_version}-slim cp /usr/local/lib/libruby.so.${ruby_version} /tmp - echo "Changing the owner of the file to the current user..." - sudo chown -R $(whoami) "${TARGET_DIR}" - echo "Done." + for arch in "${target_archs[@]}"; do + target="${PWD}"/"${TARGET_DIR}"/"${arch}" + echo "Checking if ruby ${ruby_version} runtime for ${arch} is already downloaded..." + if ls "${target}"/libruby.so.${ruby_version} 1>/dev/null 2>&1; then + echo "Ruby ${ruby_version} runtime for ${arch} is already downloaded." + continue + fi + echo "Downloading ruby ${ruby_version} runtime..." + "${CONTAINER_RUNTIME}" run --rm \ + --platform "linux/${arch}" \ + -v "${target}":/tmp -w /tmp \ + ruby:${ruby_version}-slim \ + cp /usr/local/lib/libruby.so.${ruby_version} \ + /tmp + echo "Changing the owner of the file to the current user..." + sudo chown -R $(whoami) "${TARGET_DIR}" + echo "Done." + + done done echo "All ruby runtimes downloaded successfully." - -for i in "${TARGET_DIR}"/libruby*; do file "$i"; done +dir="${PWD}"/"${TARGET_DIR}"/**/libruby* +for i in $dir; do file "$i"; done diff --git a/scripts/mergelayout/python.sh b/scripts/mergelayout/python.sh index 9b7f70b..ac9d6ee 100755 --- a/scripts/mergelayout/python.sh +++ b/scripts/mergelayout/python.sh @@ -17,8 +17,22 @@ set -euo pipefail # This script helps to merge structlayout outputs in specified directory for integration tests. -rm -f pkg/python/layout/*.yaml -./mergelayout -o pkg/python/layout 'tmp/python/layout/python_*.yaml' -rm -f pkg/python/initialstate/*.yaml -./mergelayout -o pkg/python/initialstate 'tmp/python/initialstate/python_*.yaml' +ARCH=${ARCH} +target_archs=( + amd64 + arm64 +) +if [ -n "${ARCH}" ]; then + target_archs=("${ARCH}") +fi + +rm -rf pkg/python/layout +rm -rf pkg/python/initialstate +for arch in "${target_archs[@]}"; do + mkdir -p pkg/python/layout/"${arch}" + ./mergelayout -o pkg/python/layout/"${arch}" tmp/python/"${arch}"/layout/'python_*.yaml' + + mkdir -p pkg/python/initialstate"/${arch}" + ./mergelayout -o pkg/python/initialstate/"${arch}" tmp/python/"${arch}"/initialstate/'python_*.yaml' +done diff --git a/scripts/mergelayout/ruby.sh b/scripts/mergelayout/ruby.sh index 2283883..77eaa90 100755 --- a/scripts/mergelayout/ruby.sh +++ b/scripts/mergelayout/ruby.sh @@ -17,5 +17,18 @@ set -euo pipefail # This script helps to merge structlayout outputs in specified directory for integration tests. -rm -f pkg/ruby/layout/*.yaml -./mergelayout -o pkg/ruby/layout 'tmp/ruby/layout/ruby_*.yaml' + +ARCH=${ARCH} +target_archs=( + amd64 + arm64 +) +if [ -n "${ARCH}" ]; then + target_archs=("${ARCH}") +fi + +rm -rf pkg/ruby/layout +for arch in "${target_archs[@]}"; do + mkdir -p pkg/ruby/layout/"${arch}" + ./mergelayout -o pkg/ruby/layout/"${arch}" tmp/ruby/"${arch}"/layout/'ruby_*.yaml' +done diff --git a/scripts/structlayout/python.sh b/scripts/structlayout/python.sh index f6ee9f7..b727c43 100755 --- a/scripts/structlayout/python.sh +++ b/scripts/structlayout/python.sh @@ -39,8 +39,19 @@ python_versions=( 3.11.0 # bugfix ) +ARCH=${ARCH} +target_archs=( + amd64 + arm64 +) +if [ -n "${ARCH}" ]; then + target_archs=("${ARCH}") +fi + mkdir -p tmp/python/ for python_version in "${python_versions[@]}"; do - echo "Running structlayout for python ${python_version} runtime..." - ./structlayout -r python -v "${python_version}" -o tmp/python tests/integration/binaries/python/"${python_version}"/libpython"${python_version%.*}"*.so.1.0 + for arch in "${target_archs[@]}"; do + echo "Running structlayout against python ${python_version} runtime for ${arch}..." + ./structlayout -r python -v "${python_version}" -o tmp/python/${arch} tests/integration/binaries/python/${arch}/"${python_version}"/libpython"${python_version%.*}"*.so.1.0 + done done diff --git a/scripts/structlayout/ruby.sh b/scripts/structlayout/ruby.sh index fe7fb76..99d35bf 100755 --- a/scripts/structlayout/ruby.sh +++ b/scripts/structlayout/ruby.sh @@ -32,8 +32,20 @@ ruby_versions=( 3.2.1 ) +ARCH=${ARCH} +target_archs=( + amd64 + arm64 +) +if [ -n "${ARCH}" ]; then + target_archs=("${ARCH}") +fi + + mkdir -p tmp/ruby for ruby_version in "${ruby_versions[@]}"; do - echo "Running structlayout for ruby ${ruby_version} runtime..." - ./structlayout -r ruby -v "${ruby_version}" -o tmp/ruby tests/integration/binaries/ruby/libruby.so.${ruby_version} + for arch in "${target_archs[@]}"; do + echo "Running structlayout againt ruby ${ruby_version} runtime for ${arch}..." + ./structlayout -r ruby -v "${ruby_version}" -o tmp/ruby/${arch} tests/integration/binaries/ruby/${arch}/libruby.so.${ruby_version} + done done diff --git a/tests/integration/python_test.go b/tests/integration/python_test.go index 7990e62..4dce67a 100644 --- a/tests/integration/python_test.go +++ b/tests/integration/python_test.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "testing" @@ -53,7 +54,9 @@ func TestPythonIntegration(t *testing.T) { } parts := strings.Split(version, ".") - matches, err := filepath.Glob(fmt.Sprintf("%s/%s/libpython%s.%s*.so.1.0", TargetDirPython, version, parts[0], parts[1])) + matches, err := filepath.Glob( + fmt.Sprintf("%s/%s/%s/libpython%s.%s*.so.1.0", TargetDirPython, runtime.GOARCH, version, parts[0], parts[1]), + ) if err != nil { t.Fatalf("filepath.Glob() = %v", err) } diff --git a/tests/integration/ruby_test.go b/tests/integration/ruby_test.go index ad3cd12..d176f67 100644 --- a/tests/integration/ruby_test.go +++ b/tests/integration/ruby_test.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "testing" "github.com/google/go-cmp/cmp" @@ -51,7 +52,7 @@ func TestRubyIntegration(t *testing.T) { t.Fatalf("ruby.GenerateDataMap(%s) = %v", version, err) } - input := fmt.Sprintf("%s/libruby.so.%s", TargetDirRuby, version) + input := fmt.Sprintf("%s/%s/libruby.so.%s", TargetDirRuby, runtime.GOARCH, version) f, err := elf.Open(input) if err != nil {