diff --git a/interpreters/python/.gitignore b/interpreters/python/.gitignore index ad58719cdd4..33efb9cbe30 100644 --- a/interpreters/python/.gitignore +++ b/interpreters/python/.gitignore @@ -4,3 +4,5 @@ /Python/ /romfs_cpython_modules.h /romfs_cpython_modules.img +/config.site +/Setup.local diff --git a/interpreters/python/Kconfig b/interpreters/python/Kconfig index da20c8ec43f..947a71e6de2 100644 --- a/interpreters/python/Kconfig +++ b/interpreters/python/Kconfig @@ -27,7 +27,7 @@ config INTERPRETER_CPYTHON_STACKSIZE config INTERPRETER_CPYTHON_PRIORITY int "CPython task priority" - default 150 + default 100 ---help--- This is the priority of the CPython task. @@ -37,22 +37,4 @@ config INTERPRETER_CPYTHON_PROGNAME ---help--- This is the name of the program that will be used from the nsh. -config INTERPRETER_CPYTHON_MOUNT_MODULES_STACKSIZE - int "CPython's Modules Mount stack size" - default 4096 - ---help--- - This is the stack size allocated when the CPython's Modules Mount task runs. - -config INTERPRETER_CPYTHON_MOUNT_MODULES_PRIORITY - int "CPython's Modules Mount task priority" - default 150 - ---help--- - This is the priority of the CPython's Modules Mount task. - -config INTERPRETER_CPYTHON_MOUNT_MODULES_PROGNAME - string "CPython's Modules Mount app name" - default "python_mount_modules" - ---help--- - This is the name of the program that will be used from the nsh. - endif diff --git a/interpreters/python/Makefile b/interpreters/python/Makefile index 4b6c42eac39..af5c8b89f6a 100644 --- a/interpreters/python/Makefile +++ b/interpreters/python/Makefile @@ -32,6 +32,7 @@ UNPACK ?= unzip -q -o MACHDEP=nuttx CONFIG_SITE=${CURDIR}/config.site +SETUP_LOCAL=${CURDIR}/Setup.local CPYTHON_PATH=$(CURDIR)/$(CPYTHON_UNPACKNAME) BUILDIR=$(CURDIR)/build @@ -80,6 +81,8 @@ $(CPYTHON_UNPACKNAME): $(CPYTHON_ZIP) $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0009-include-nuttx-sys-select-header-to-define-FD_SETSIZE.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0010-check-for-the-d_ino-member-of-the-structure-dirent.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0011-avoid-redefinition-warning-if-UNUSED-is-already-defi.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0013-transform-functions-used-by-NuttX-to-lowercase.patch $(HOSTPYTHON): mkdir -p $(HOSTBUILD) @@ -91,6 +94,35 @@ $(HOSTPYTHON): ) $(MAKE) -C $(HOSTBUILD) install +# The `config.site` file contains settings that override the configuration +# settings provided by the `configure` script. Depending on the features +# enabled on NuttX, this file may need to be adjusted. + +$(CONFIG_SITE): + $(Q) ( cp $(CONFIG_SITE).in $(CONFIG_SITE)) +ifeq ($(CONFIG_ARCH_HAVE_FORK),y) + @echo "export ac_cv_func_fork=\"yes\"" >> $@ +else + @echo "export ac_cv_func_fork=\"no\"" >> $@ +endif +ifeq ($(CONFIG_SYSTEM_SYSTEM),y) + @echo "export ac_cv_func_system=\"yes\"" >> $@ +else + @echo "export ac_cv_func_system=\"no\"" >> $@ +endif + +# The `Setup.local` file enables or disables Python modules. +# Depending on the features enabled on NuttX, this file may need to be +# adjusted. Please note that the base `Setup.local.in` file only contains +# a section to disable Python modules. Inserting lines to it will disable +# such modules. + +$(SETUP_LOCAL): + $(Q) ( cp $(SETUP_LOCAL).in $(SETUP_LOCAL)) +ifneq ($(CONFIG_ARCH_HAVE_FORK),y) + @echo "_posixsubprocess" >> $@ +endif + # For the Python's `configure` script, please consider the following # when building for NuttX: # @@ -102,7 +134,7 @@ $(HOSTPYTHON): # Python/Modules/getpath.c (issue will be filed soon to track this # problem). -$(TARGETBUILD)/Makefile: $(HOSTPYTHON) +$(TARGETBUILD)/Makefile: $(HOSTPYTHON) $(CONFIG_SITE) $(SETUP_LOCAL) $(Q) mkdir -p $(TARGETBUILD)/Modules $(Q) mkdir -p $(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR) $(Q) ( cp Setup.local $(TARGETBUILD)/Modules/Setup.local ) @@ -144,13 +176,7 @@ PROGNAME += $(CONFIG_INTERPRETER_CPYTHON_PROGNAME) PRIORITY += $(CONFIG_INTERPRETER_CPYTHON_PRIORITY) STACKSIZE += $(CONFIG_INTERPRETER_CPYTHON_STACKSIZE) -MAINSRC += python.c - -PROGNAME += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_PROGNAME) -PRIORITY += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_PRIORITY) -STACKSIZE += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_STACKSIZE) - -MAINSRC += mount_modules.c +MAINSRC += python_wrapper.c checkgenromfs: @genromfs -h 1>/dev/null 2>&1 || { \ @@ -163,7 +189,7 @@ romfs_cpython_modules.img : $(TARGETLIBPYTHON) checkgenromfs @genromfs -f $@ -d $(TARGETMODULES) -V "ROMFS_Test" || { echo "genromfs failed" ; exit 1 ; } romfs_cpython_modules.h : romfs_cpython_modules.img - @xxd -i $< >$@ || { echo "xxd of $< failed" ; exit 1 ; } + @xxd -i $< | sed -e "s/^unsigned/static const unsigned/g" >$@ || { echo "xxd of $< failed" ; exit 1 ; } context:: $(CPYTHON_UNPACKNAME) @@ -176,5 +202,7 @@ distclean:: $(call DELFILE, $(CPYTHON_ZIP)) $(call DELFILE, romfs_cpython_modules.img) $(call DELFILE, romfs_cpython_modules.h) + $(call DELFILE, config.site) + $(call DELFILE, Setup.local) include $(APPDIR)/Application.mk diff --git a/interpreters/python/Setup.local b/interpreters/python/Setup.local.in similarity index 100% rename from interpreters/python/Setup.local rename to interpreters/python/Setup.local.in diff --git a/interpreters/python/config.site b/interpreters/python/config.site.in similarity index 95% rename from interpreters/python/config.site rename to interpreters/python/config.site.in index 28313d75e7e..eb37e5a88c9 100644 --- a/interpreters/python/config.site +++ b/interpreters/python/config.site.in @@ -1,3 +1,4 @@ +export MODULE_BUILDTYPE="static" export ac_cv_file__dev_ptmx="no" export ac_cv_file__dev_ptc="no" export ac_cv_buggy_getaddrinfo="no" @@ -15,10 +16,9 @@ export ac_cv_func_clock_gettime="yes" export ac_cv_header_sys_syscall_h="no" export ac_cv_func_timegm="yes" export ac_cv_func_clock="yes" -export ac_cv_func_fork="yes" export ac_cv_func_waitpid="yes" export ac_cv_func_pipe="yes" export ac_cv_enable_strict_prototypes_warning="no" export ac_cv_func_getnameinfo="yes" export ac_cv_func_poll="yes" -export MODULE_BUILDTYPE="static" +export ac_cv_func_gethostname="yes" \ No newline at end of file diff --git a/interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch b/interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch new file mode 100644 index 00000000000..d64342e551c --- /dev/null +++ b/interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch @@ -0,0 +1,54 @@ +From d1e903f516849c535455904b3c3f8a33665c1a88 Mon Sep 17 00:00:00 2001 +From: Ivan Grokhotkov +Date: Wed, 23 Oct 2024 16:52:52 +0200 +Subject: [PATCH 12/12] hack: place _PyRuntime structure into PSRAM bss region, + initialize later + +_PyRuntime occupies around 100kB of RAM in .data region, making it +hard to fit the interpreter into the available static RAM. +This patch moves it into PSRAM using section attribute. Normally +we shouldn't need this as we can specify placements in ldfragments, +however in this specific case I couldn't get it to work. +Since the structure is now in .bss, add a function which will assign +it the initial value. + +The proper fix might be to support .data segment on PSRAM in IDF, +as well as to fix whatever ldgen issue prevents this variable from +being moved to PSRAM. + +Co-authored-by: Tiago Medicci Serrano +--- + Python/pylifecycle.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c +index 1701a1cd217..2a8e544f0ac 100644 +--- a/Python/pylifecycle.c ++++ b/Python/pylifecycle.c +@@ -102,12 +102,23 @@ __attribute__(( + _PyRuntimeState _PyRuntime + #if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) + __attribute__ ((section (".PyRuntime"))) ++#elif defined(ESP_PLATFORM) ++__attribute__ ((section (".PyRuntime"))) + #endif + = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); + _Py_COMP_DIAG_POP + + static int runtime_initialized = 0; + ++void _PyRuntime_Early_Init(void) { ++#if defined(ESP_PLATFORM) ++ // Normally, _PyRuntime is in .data and is initialized by the C runtime. ++ // This function allows us to place it into external RAM .bss section ++ // and initialize it manually, saving some internal RAM. ++ _PyRuntime = (struct pyruntimestate) _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); ++#endif ++} ++ + PyStatus + _PyRuntime_Initialize(void) + { +-- +2.47.1 + diff --git a/interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch b/interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch new file mode 100644 index 00000000000..c1e83aad640 --- /dev/null +++ b/interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch @@ -0,0 +1,33 @@ +From 914c80b7969d73840bc1b573b478d9148999b7d0 Mon Sep 17 00:00:00 2001 +From: Tiago Medicci +Date: Fri, 31 Jan 2025 14:06:21 -0300 +Subject: [PATCH 13/13] transform functions used by NuttX to lowercase + +--- + Include/pylifecycle.h | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h +index de1bcb1d2cb..044780ee188 100644 +--- a/Include/pylifecycle.h ++++ b/Include/pylifecycle.h +@@ -33,6 +33,16 @@ PyAPI_FUNC(void) _Py_NO_RETURN Py_Exit(int); + PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); + PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); + ++#if defined(__NuttX__) ++#define py_bytesmain Py_BytesMain ++#endif ++ ++void _PyRuntime_Early_Init(void); ++ ++#if defined(__NuttX__) ++#define _pyruntime_early_init _PyRuntime_Early_Init ++#endif ++ + /* In pathconfig.c */ + Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); + Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); +-- +2.47.1 + diff --git a/interpreters/python/mount_modules.c b/interpreters/python/python_wrapper.c similarity index 72% rename from interpreters/python/mount_modules.c rename to interpreters/python/python_wrapper.c index 681031579c4..1eb1d3cc3e0 100644 --- a/interpreters/python/mount_modules.c +++ b/interpreters/python/python_wrapper.c @@ -1,5 +1,5 @@ /**************************************************************************** - * apps/interpreters/python/mount_modules.c + * apps/interpreters/python/python_wrapper.c * * SPDX-License-Identifier: Apache-2.0 * @@ -41,11 +41,14 @@ #include #include #include +#include #include #include "romfs_cpython_modules.h" +#include "Python.h" + /**************************************************************************** * Pre-processor Definitions ****************************************************************************/ @@ -61,7 +64,7 @@ #endif #ifndef CONFIG_CPYTHON_ROMFS_MOUNTPOINT -# define CONFIG_CPYTHON_ROMFS_MOUNTPOINT "/usr/local/lib/" +# define CONFIG_CPYTHON_ROMFS_MOUNTPOINT "/usr/local/lib" #endif #ifdef CONFIG_DISABLE_MOUNTPOINT @@ -90,19 +93,57 @@ ****************************************************************************/ /**************************************************************************** - * Public Functions - ****************************************************************************/ - -/**************************************************************************** - * Name: mount_modules + * Name: check_and_mount_romfs + * + * Description: + * Check if the ROMFS is already mounted, and if not, mount it. + * + * Input Parameters: + * None + * + * Returned Value: + * 0 on success, 1 on failure + * ****************************************************************************/ -int main(int argc, FAR char *argv[]) +static int check_and_mount_romfs(void) { - int ret; + int ret = OK; struct boardioc_romdisk_s desc; + FILE *fp; + char line[256]; + int is_mounted = 0; - /* Create a RAM disk for the test */ + /* Check if the device is already mounted */ + + fp = fopen("/proc/fs/mount", "r"); + if (fp == NULL) + { + printf("ERROR: Failed to open /proc/fs/mount\n"); + UNUSED(desc); + return ret = ERROR; + } + + while (fgets(line, sizeof(line), fp)) + { + if (strstr(line, CONFIG_CPYTHON_ROMFS_MOUNTPOINT) != NULL) + { + is_mounted = 1; + break; + } + } + + fclose(fp); + + if (is_mounted) + { + _info("Device is already mounted at %s\n", + CONFIG_CPYTHON_ROMFS_MOUNTPOINT); + UNUSED(desc); + return ret; + } + + /* Create a RAM disk */ desc.minor = CONFIG_CPYTHON_ROMFS_RAMDEVNO; /* Minor device number of the ROM disk. */ desc.nsectors = NSECTORS(romfs_cpython_modules_img_len); /* The number of sectors in the ROM disk */ @@ -119,8 +160,8 @@ int main(int argc, FAR char *argv[]) /* Mount the test file system */ - printf("Mounting ROMFS filesystem at target=%s with source=%s\n", - CONFIG_CPYTHON_ROMFS_MOUNTPOINT, MOUNT_DEVNAME); + _info("Mounting ROMFS filesystem at target=%s with source=%s\n", + CONFIG_CPYTHON_ROMFS_MOUNTPOINT, MOUNT_DEVNAME); ret = mount(MOUNT_DEVNAME, CONFIG_CPYTHON_ROMFS_MOUNTPOINT, "romfs", MS_RDONLY, NULL); @@ -132,3 +173,30 @@ int main(int argc, FAR char *argv[]) return 0; } + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: python_wrapper_main + ****************************************************************************/ + +int main(int argc, FAR char *argv[]) +{ + int ret; + + ret = check_and_mount_romfs(); + if (ret != 0) + { + return ret; + } + + _pyruntime_early_init(); + + setenv("PYTHONHOME", "/usr/local", 1); + + setenv("PYTHON_BASIC_REPL", "1", 1); + + return py_bytesmain(argc, argv); +}