From f8802a2d5ecfddc9267d99ab30d13aff3282e4b8 Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 6 Nov 2024 13:03:53 +0100 Subject: [PATCH 1/6] Add support for partition-tables. --- .github/workflows/build.yml | 48 ++++++++- README.md | 109 ++------------------- partition-tables/README.md | 15 +++ partition-tables/ota-1c0000/README.md | 7 ++ partition-tables/ota-1c0000/partitions.csv | 17 ++++ variants/Contributing.md | 51 ++++++++++ variants/README.md | 56 +++++++++++ 7 files changed, 198 insertions(+), 105 deletions(-) create mode 100644 partition-tables/README.md create mode 100644 partition-tables/ota-1c0000/README.md create mode 100644 partition-tables/ota-1c0000/partitions.csv create mode 100644 variants/Contributing.md create mode 100644 variants/README.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 279cb8e..359d088 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,11 @@ on: required: false type: boolean default: true + build-tables: + description: Build partition tables + required: false + type: boolean + default: true toit-version: description: Toit SDK version to check out required: true @@ -192,7 +197,7 @@ jobs: artifacts: | envelopes/* body: | - Envelopes for Toit SDK ${{ github.event.inputs.toit-version }} + Envelopes and partition tables for Toit SDK ${{ github.event.inputs.toit-version }} allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.event.inputs.toit-version }} @@ -300,7 +305,7 @@ jobs: artifacts: | envelopes/* body: | - Envelopes for Toit SDK ${{ github.event.inputs.toit-version }} + Envelopes and partition tables for Toit SDK ${{ github.event.inputs.toit-version }} allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.event.inputs.toit-version }} @@ -371,7 +376,44 @@ jobs: artifacts: | envelopes/* body: | - Envelopes for Toit SDK ${{ github.event.inputs.toit-version }} + Envelopes and partition tables for Toit SDK ${{ github.event.inputs.toit-version }} + allowUpdates: true + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.event.inputs.toit-version }} + commit: main + + release-partition-tables: + name: "Release partition tables - ${{ github.event.inputs.toit-version }}" + if: ${{ github.event.inputs.build-tables == 'true' }} + runs-on: ubuntu-latest + + steps: + - name: Show inputs + run: echo "${{ toJSON(github.event.inputs) }}" + + - uses: actions/checkout@v4 + + - name: Set the git version + run: | + echo "TOIT_GIT_VERSION=${{ github.event.inputs.toit-version }}" >> $GITHUB_ENV + + - name: Prepare partition tables + run: | + mkdir -p build + for variant in $(ls partition-tables); do + if [ -e "partition-tables/$variant/partitions.csv" ]; then + cp "partition-tables/$variant/partitions.csv" "build/partition-table-$variant.csv" + fi + done + + - name: Release + if: ${{ github.event.inputs.upload-release == 'true' }} + uses: ncipollo/release-action@v1 + with: + artifacts: | + build/* + body: | + Envelopes and partition tables for Toit SDK ${{ github.event.inputs.toit-version }} allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.event.inputs.toit-version }} diff --git a/README.md b/README.md index 3a68fbd..1df689d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Envelopes -Envelopes for Toit. +Envelopes and partition tables for Toit. This repository contains a tool to generate envelopes for Toit. It also automatically builds envelopes for the variants that are stored in @@ -19,105 +19,10 @@ compiled with and for SDK v2.0.0-alpha.90 are found on the [v2.0.0-alpha.90 release](https://github.com/toitlang/envelopes/releases/tag/v2.0.0-alpha.90) page. -## Variants +See the [variants/README.md](variants/README.md) for a list of the +most commonly used variants. You can also browse the `variants` +directory for more variants. -### esp32 - -A generic ESP32 variant. This is the default variant when using Toit. -It is built for maximum compatibility. - -This variant supports Ethernet, but without the clock output. - -### esp32-spiram - -An ESP32 variant for boards with SPIRAM. Otherwise the same as the ESP32 variant. - -### esp32-eth-clk-out0 and esp32-eth-clk-out17 - -A variant for ESP32 boards with Ethernet and a clock output on pin 0/17. - -Olimex boards with Ethernet should use this variant. The WROOM versions need -`esp32-eth-clk-out17` and the WROVER versions need `esp32-eth-clk-out0`. - -### esp32c3 - -A generic [ESP32-C3 variant](variants/esp32c). This is the default variant -when using Toit on ESP32-C3 boards. - -### esp32s2 - -A generic [ESP32-S2 variant](variants/esp32s2). This is the default variant -when using Toit on ESP32-S2 boards. - -### esp32s3 - -A generic [ESP32-S3 variant](variants/esp32s3). This is the default variant -when using Toit on ESP32-S3 boards. - -This variant is configured for external Quad PSRAM. - -### esp32s3-spiram-octo - -A [variant](variants/esp32s3-spiram-octo/) for ESP32-S3 boards with external -octal PSRAM. - -These boards are faster, but often more expensive. - -### esp32-no-ble - -A [variant](variants/esp32-no-ble/) for ESP32 boards. This variant -saves some RAM and flash space by removing the Bluetooth stack. -The saved IRAM enables us to make the Toit interpreter a little faster -and add support for external RAM (PSRAM, aka SPIRAM). - -## Contributing - -Feel free to open issues and pull requests with new variants. Make sure -they have a description (README.md) with the purpose and the changes. -We will automatically build them whenever a new Toit version is released. - -Note that some variants are featured here. Consult the -Toit team before adding new variants to this README. - -### Creating a variant - -Variants are created by copying an existing variant and making the -necessary changes. These consists of either overwriting existing files -or by applying patches. - -#### Partition changes - -For partition changes, simply copy the new `partitions.csv` into the -variant directory. The `esp32-ota-1c0000` is an example of this where -the OTA partition size has been increased. - -#### sdkconfig changes - -For `sdkconfig` changes, a patch to the original `sdkconfig.defaults` -file is typically preferred. - -For example, to create a variant `esp32s3-foo`. -* Check out Toit (or use an existing checkout). -* Copy the existing `toolchains/esp32s3` directory to `toolchains/esp32s3-foo`: - `cp -r toolchains/esp32s3 toolchains/esp32s3-foo`. -* Run `make IDF_TARGET=esp32s3 ESP32_CHIP=esp32s3-foo menuconfig` and make the changes you want. - This automatically updates the `sdkconfig.defaults` as well. -* Create patch by running: - ``` - diff -aur \ - --label toit/toolchains/esp32s3/sdkconfig.defaults \ - --label synthesized/esp32s3-foo/sdkconfig.defaults \ - toolchains/esp32s3/sdkconfig.defaults \ - toolchains/esp32s3-foo/sdkconfig.defaults \ - > toolchains/esp32s3-foo/sdkconfig.defaults.patch - ``` - The labels are not crucial, but make it easier for us to update the - patch at a later time. -* Create a new variant in this (`envelopes`) repository and copy the patch file into it. - -#### Main changes - -For changes to the `main` directory (be it the `toit.c` or the `CMakelists.txt` in it), -use a recursive diff to create a patch on the original `main` directory. - -The synthetization tool will use the flag `-p1` when applying the patch. +See the [partition-tables/README.md](partition-tables/README.md) for +a list of the most commonly used partition tables. You can also browse +the `partition-tables` directory for more partition tables. diff --git a/partition-tables/README.md b/partition-tables/README.md new file mode 100644 index 0000000..98eb80c --- /dev/null +++ b/partition-tables/README.md @@ -0,0 +1,15 @@ +# Partition tables + +This directory contains partition tables for ESP32 devices and their variants. + +During flashing, users can override the default partition table that comes +with the envelope that they are flashing. + +Here is a list of the most commonly used partition tables: + +## OTA-1C0000 + +A partition table that sets the size of the OTA partitions +to 0x1C0000 (1835008) bytes. This table is typically used for +use-cases where programs are bundled with the firmware (like with +[Artemis](https://github.com/toitware/artemis)). diff --git a/partition-tables/ota-1c0000/README.md b/partition-tables/ota-1c0000/README.md new file mode 100644 index 0000000..dae7637 --- /dev/null +++ b/partition-tables/ota-1c0000/README.md @@ -0,0 +1,7 @@ +# OTA-1C0000 + +A partition table that sets the size of the OTA partitions +to 0x1C0000 (1835008) bytes. + +As a consequence the size of the programs partition is set to +0x60000 (393216) bytes. diff --git a/partition-tables/ota-1c0000/partitions.csv b/partition-tables/ota-1c0000/partitions.csv new file mode 100644 index 0000000..27411f3 --- /dev/null +++ b/partition-tables/ota-1c0000/partitions.csv @@ -0,0 +1,17 @@ +# Copyright (C) 2023 Toitware ApS. +# +# Use of this source code is governed by a BSD0-style license that can be +# found in the LICENSE_BSD0 file. + +# Partition Table for Toit. + +# Name, Type, SubType, Offset, Size +# bootloader,, , 0x001000, 0x007000 +# partitions,, , 0x008000, 0x000c00 +secure, 0x42, 0x00, 0x009000, 0x004000, +otadata, data, ota, 0x00d000, 0x002000, +phy_init, data, phy, 0x00f000, 0x001000, +ota_0, app, ota_0, 0x010000, 0x1c0000, +ota_1, app, ota_1, 0x1d0000, 0x1c0000, +nvs, data, nvs, 0x390000, 0x010000, +programs, 0x40, 0x00, 0x3A0000, 0x060000, encrypted diff --git a/variants/Contributing.md b/variants/Contributing.md new file mode 100644 index 0000000..efccba3 --- /dev/null +++ b/variants/Contributing.md @@ -0,0 +1,51 @@ +# Contributing + +Feel free to open issues and pull requests with new variants. Make sure +they have a description (README.md) with the purpose and the changes. If +they are generally useful, we will add them to the repository. + +Variants in this repository are automatically built whenever a new Toit +version is released. + +## Creating a variant + +Variants are created by copying an existing variant and making the +necessary changes. These consists of either overwriting existing files +or by applying patches. + +### Partition changes + +For partition changes, simply copy the new `partitions.csv` into the +variant directory. The `esp32-ota-1c0000` is an example of this where +the OTA partition size has been increased. + +### sdkconfig changes + +For `sdkconfig` changes, a patch to the original `sdkconfig.defaults` +file is typically preferred. + +For example, to create a variant `esp32s3-foo`. +* Check out Toit (or use an existing checkout). +* Copy the existing `toolchains/esp32s3` directory to `toolchains/esp32s3-foo`: + `cp -r toolchains/esp32s3 toolchains/esp32s3-foo`. +* Run `make IDF_TARGET=esp32s3 ESP32_CHIP=esp32s3-foo menuconfig` and make the changes you want. + This automatically updates the `sdkconfig.defaults` as well. +* Create patch by running: + ``` + diff -aur \ + --label toit/toolchains/esp32s3/sdkconfig.defaults \ + --label synthesized/esp32s3-foo/sdkconfig.defaults \ + toolchains/esp32s3/sdkconfig.defaults \ + toolchains/esp32s3-foo/sdkconfig.defaults \ + > toolchains/esp32s3-foo/sdkconfig.defaults.patch + ``` + The labels are not crucial, but make it easier for us to update the + patch at a later time. +* Create a new variant in this (`envelopes`) repository and copy the patch file into it. + +### Main changes + +For changes to the `main` directory (be it the `toit.c` or the `CMakelists.txt` in it), +use a recursive diff to create a patch on the original `main` directory. + +The synthetization tool will use the flag `-p1` when applying the patch. diff --git a/variants/README.md b/variants/README.md new file mode 100644 index 0000000..0da87dc --- /dev/null +++ b/variants/README.md @@ -0,0 +1,56 @@ +# Variants + +Some commonly used variants are featured here. You can also browse this +directory for more variants. + +See the [Contributing](Contributing.md) guide for how to create a new variant. + + +## esp32 + +A generic ESP32 variant. This is the default variant when using Toit. +It is built for maximum compatibility. + +This variant supports Ethernet, but without the clock output. + +## esp32-spiram + +An ESP32 variant for boards with SPIRAM. Otherwise the same as the ESP32 variant. + +## esp32-eth-clk-out0 and esp32-eth-clk-out17 + +A variant for ESP32 boards with Ethernet and a clock output on pin 0/17. + +Olimex boards with Ethernet should use this variant. The WROOM versions need +`esp32-eth-clk-out17` and the WROVER versions need `esp32-eth-clk-out0`. + +## esp32c3 + +A generic [ESP32-C3 variant](esp32c). This is the default variant +when using Toit on ESP32-C3 boards. + +## esp32s2 + +A generic [ESP32-S2 variant](esp32s2). This is the default variant +when using Toit on ESP32-S2 boards. + +## esp32s3 + +A generic [ESP32-S3 variant](esp32s3). This is the default variant +when using Toit on ESP32-S3 boards. + +This variant is configured for external Quad PSRAM. + +## esp32s3-spiram-octo + +A [variant](esp32s3-spiram-octo/) for ESP32-S3 boards with external +octal PSRAM. + +These boards are faster, but often more expensive. + +## esp32-no-ble + +A [variant](esp32-no-ble/) for ESP32 boards. This variant +saves some RAM and flash space by removing the Bluetooth stack. +The saved IRAM enables us to make the Toit interpreter a little faster +and add support for external RAM (PSRAM, aka SPIRAM). From c54737f03f283c0b6e9d6f401bf31c87d15acaf9 Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 6 Nov 2024 13:39:08 +0100 Subject: [PATCH 2/6] Improve. --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 359d088..42b6135 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -398,11 +398,14 @@ jobs: echo "TOIT_GIT_VERSION=${{ github.event.inputs.toit-version }}" >> $GITHUB_ENV - name: Prepare partition tables + shell: bash run: | mkdir -p build - for variant in $(ls partition-tables); do + shopt -s nullglob + for csv_file in partition-tables/*/partitions.csv; do + variant=$(basename $(dirname $csv_file)) if [ -e "partition-tables/$variant/partitions.csv" ]; then - cp "partition-tables/$variant/partitions.csv" "build/partition-table-$variant.csv" + cp "$csv_file" "build/partition-table-$variant.csv" fi done From 4564f0a8a7bc12d246145d5a5415738592a0e4e4 Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 6 Nov 2024 13:46:45 +0100 Subject: [PATCH 3/6] Update .github/workflows/build.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42b6135..30d671c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -414,7 +414,7 @@ jobs: uses: ncipollo/release-action@v1 with: artifacts: | - build/* + build/partition-table-*.csv body: | Envelopes and partition tables for Toit SDK ${{ github.event.inputs.toit-version }} allowUpdates: true From 636360d20d53751c9ded23151bc52f191161c19f Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 6 Nov 2024 13:51:44 +0100 Subject: [PATCH 4/6] Update .github/workflows/build.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30d671c..b8c9505 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -403,7 +403,7 @@ jobs: mkdir -p build shopt -s nullglob for csv_file in partition-tables/*/partitions.csv; do - variant=$(basename $(dirname $csv_file)) + variant=$(basename "$(dirname "$csv_file")") if [ -e "partition-tables/$variant/partitions.csv" ]; then cp "$csv_file" "build/partition-table-$variant.csv" fi From 093360bd3129f869f8506c6d52390fb749fc338c Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 20 Nov 2024 13:24:58 +0100 Subject: [PATCH 5/6] Feedback. --- .github/workflows/build.yml | 14 +++++++------- README.md | 4 ++-- {partition-tables => partitions/esp32}/README.md | 0 .../esp32}/ota-1c0000/README.md | 0 .../esp32}/ota-1c0000/partitions.csv | 0 5 files changed, 9 insertions(+), 9 deletions(-) rename {partition-tables => partitions/esp32}/README.md (100%) rename {partition-tables => partitions/esp32}/ota-1c0000/README.md (100%) rename {partition-tables => partitions/esp32}/ota-1c0000/partitions.csv (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42b6135..649114e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,8 +23,8 @@ on: required: false type: boolean default: true - build-tables: - description: Build partition tables + build-partitions: + description: Build partitions required: false type: boolean default: true @@ -197,7 +197,7 @@ jobs: artifacts: | envelopes/* body: | - Envelopes and partition tables for Toit SDK ${{ github.event.inputs.toit-version }} + Envelopes and partitions for Toit SDK ${{ github.event.inputs.toit-version }} allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.event.inputs.toit-version }} @@ -384,7 +384,7 @@ jobs: release-partition-tables: name: "Release partition tables - ${{ github.event.inputs.toit-version }}" - if: ${{ github.event.inputs.build-tables == 'true' }} + if: ${{ github.event.inputs.build-partitions == 'true' }} runs-on: ubuntu-latest steps: @@ -402,10 +402,10 @@ jobs: run: | mkdir -p build shopt -s nullglob - for csv_file in partition-tables/*/partitions.csv; do + for csv_file in partitions/esp32/*/partitions.csv; do variant=$(basename $(dirname $csv_file)) - if [ -e "partition-tables/$variant/partitions.csv" ]; then - cp "$csv_file" "build/partition-table-$variant.csv" + if [ -e "partitions/esp32/$variant/partitions.csv" ]; then + cp "$csv_file" "build/partitions-esp32-$variant.csv" fi done diff --git a/README.md b/README.md index 1df689d..2f80ebc 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,6 @@ See the [variants/README.md](variants/README.md) for a list of the most commonly used variants. You can also browse the `variants` directory for more variants. -See the [partition-tables/README.md](partition-tables/README.md) for +See the [partitions/esp32/README.md](partitions/esp32/README.md) for a list of the most commonly used partition tables. You can also browse -the `partition-tables` directory for more partition tables. +the `partitions/esp32` directory for more partition tables. diff --git a/partition-tables/README.md b/partitions/esp32/README.md similarity index 100% rename from partition-tables/README.md rename to partitions/esp32/README.md diff --git a/partition-tables/ota-1c0000/README.md b/partitions/esp32/ota-1c0000/README.md similarity index 100% rename from partition-tables/ota-1c0000/README.md rename to partitions/esp32/ota-1c0000/README.md diff --git a/partition-tables/ota-1c0000/partitions.csv b/partitions/esp32/ota-1c0000/partitions.csv similarity index 100% rename from partition-tables/ota-1c0000/partitions.csv rename to partitions/esp32/ota-1c0000/partitions.csv From 8bf9acaf612bfe66bb9a86f452fb736f12ffde97 Mon Sep 17 00:00:00 2001 From: Florian Loitsch Date: Wed, 20 Nov 2024 13:35:48 +0100 Subject: [PATCH 6/6] Update path. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15fbf64..8d92915 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -414,7 +414,7 @@ jobs: uses: ncipollo/release-action@v1 with: artifacts: | - build/partition-table-*.csv + build/partitions-*.csv body: | Envelopes and partition tables for Toit SDK ${{ github.event.inputs.toit-version }} allowUpdates: true