interpreters/python: create Python's config files dynamically and other tweaks

interpreters/python/.gitignore

@@ -4,3 +4,5 @@
 /Python/
 /romfs_cpython_modules.h
 /romfs_cpython_modules.img
+/config.site
+/Setup.local

interpreters/python/Kconfig

@@ -27,7 +27,7 @@ config INTERPRETER_CPYTHON_STACKSIZE
 
 config INTERPRETER_CPYTHON_PRIORITY
 	int "CPython task priority"
-	default 150
+	default 100
 	---help---
 		This is the priority of the CPython task.
 
@@ -37,22 +37,4 @@ config INTERPRETER_CPYTHON_PROGNAME
 	---help---
 		This is the name of the program that will be used from the nsh.
 
-config INTERPRETER_CPYTHON_MOUNT_MODULES_STACKSIZE
-	int "CPython's Modules Mount stack size"
-	default 4096
-	---help---
-		This is the stack size allocated when the CPython's Modules Mount task runs.
-
-config INTERPRETER_CPYTHON_MOUNT_MODULES_PRIORITY
-	int "CPython's Modules Mount task priority"
-	default 150
-	---help---
-		This is the priority of the CPython's Modules Mount task.
-
-config INTERPRETER_CPYTHON_MOUNT_MODULES_PROGNAME
-	string "CPython's Modules Mount app name"
-	default "python_mount_modules"
-	---help---
-		This is the name of the program that will be used from the nsh.
-
 endif

interpreters/python/Makefile

@@ -32,6 +32,7 @@ UNPACK ?= unzip -q -o
 
 MACHDEP=nuttx
 CONFIG_SITE=${CURDIR}/config.site
+SETUP_LOCAL=${CURDIR}/Setup.local
 CPYTHON_PATH=$(CURDIR)/$(CPYTHON_UNPACKNAME)
 
 BUILDIR=$(CURDIR)/build
@@ -80,6 +81,8 @@ $(CPYTHON_UNPACKNAME): $(CPYTHON_ZIP)
 	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0009-include-nuttx-sys-select-header-to-define-FD_SETSIZE.patch
 	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0010-check-for-the-d_ino-member-of-the-structure-dirent.patch
 	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0011-avoid-redefinition-warning-if-UNUSED-is-already-defi.patch
+	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch
+	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0013-transform-functions-used-by-NuttX-to-lowercase.patch
 
 $(HOSTPYTHON):
 	mkdir -p $(HOSTBUILD)
@@ -91,6 +94,35 @@ $(HOSTPYTHON):
 		 )
 	$(MAKE) -C $(HOSTBUILD) install
 
+# The `config.site` file contains settings that override the configuration
+# settings provided by the `configure` script. Depending on the features
+# enabled on NuttX, this file may need to be adjusted.
+
+$(CONFIG_SITE):
+	$(Q) ( cp $(CONFIG_SITE).in $(CONFIG_SITE))
+ifeq ($(CONFIG_ARCH_HAVE_FORK),y)
+	@echo "export ac_cv_func_fork=\"yes\"" >> $@
+else
+	@echo "export ac_cv_func_fork=\"no\"" >> $@
+endif
+ifeq ($(CONFIG_SYSTEM_SYSTEM),y)
+	@echo "export ac_cv_func_system=\"yes\"" >> $@
+else
+	@echo "export ac_cv_func_system=\"no\"" >> $@
+endif
+
+# The `Setup.local` file enables or disables Python modules.
+# Depending on the features enabled on NuttX, this file may need to be
+# adjusted. Please note that the base `Setup.local.in` file  only contains
+# a section to disable Python modules. Inserting lines to it will disable
+# such modules.
+
+$(SETUP_LOCAL):
+	$(Q) ( cp $(SETUP_LOCAL).in $(SETUP_LOCAL))
+ifneq ($(CONFIG_ARCH_HAVE_FORK),y)
+	@echo "_posixsubprocess" >> $@
+endif
+
 # For the Python's `configure` script, please consider the following
 # when building for NuttX:
 #
@@ -102,7 +134,7 @@ $(HOSTPYTHON):
 # Python/Modules/getpath.c (issue will be filed soon to track this
 # problem).
 
-$(TARGETBUILD)/Makefile: $(HOSTPYTHON)
+$(TARGETBUILD)/Makefile: $(HOSTPYTHON) $(CONFIG_SITE) $(SETUP_LOCAL)
 	$(Q) mkdir -p $(TARGETBUILD)/Modules
 	$(Q) mkdir -p $(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)
 	$(Q) ( cp Setup.local $(TARGETBUILD)/Modules/Setup.local )
@@ -144,13 +176,7 @@ PROGNAME  += $(CONFIG_INTERPRETER_CPYTHON_PROGNAME)
 PRIORITY  += $(CONFIG_INTERPRETER_CPYTHON_PRIORITY)
 STACKSIZE += $(CONFIG_INTERPRETER_CPYTHON_STACKSIZE)
 
-MAINSRC += python.c
-
-PROGNAME  += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_PROGNAME)
-PRIORITY  += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_PRIORITY)
-STACKSIZE += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_STACKSIZE)
-
-MAINSRC += mount_modules.c
+MAINSRC += python_wrapper.c
 
 checkgenromfs:
 	@genromfs -h 1>/dev/null 2>&1 || { \
@@ -163,7 +189,7 @@ romfs_cpython_modules.img : $(TARGETLIBPYTHON) checkgenromfs
 	@genromfs -f $@ -d $(TARGETMODULES) -V "ROMFS_Test" || { echo "genromfs failed" ; exit 1 ; }
 
 romfs_cpython_modules.h : romfs_cpython_modules.img
-	@xxd -i $< >$@ || { echo "xxd of $< failed" ; exit 1 ; }
+	@xxd -i $< | sed -e "s/^unsigned/static const unsigned/g" >$@ || { echo "xxd of $< failed" ; exit 1 ; }
 
 context:: $(CPYTHON_UNPACKNAME)
 
@@ -176,5 +202,7 @@ distclean::
 	$(call DELFILE, $(CPYTHON_ZIP))
 	$(call DELFILE, romfs_cpython_modules.img)
 	$(call DELFILE, romfs_cpython_modules.h)
+	$(call DELFILE, config.site)
+	$(call DELFILE, Setup.local)
 
 include $(APPDIR)/Application.mk

interpreters/python/config.site → interpreters/python/config.site.in

@@ -1,3 +1,4 @@
+export MODULE_BUILDTYPE="static"
 export ac_cv_file__dev_ptmx="no"
 export ac_cv_file__dev_ptc="no"
 export ac_cv_buggy_getaddrinfo="no"
@@ -15,10 +16,9 @@ export ac_cv_func_clock_gettime="yes"
 export ac_cv_header_sys_syscall_h="no"
 export ac_cv_func_timegm="yes"
 export ac_cv_func_clock="yes"
-export ac_cv_func_fork="yes"
 export ac_cv_func_waitpid="yes"
 export ac_cv_func_pipe="yes"
 export ac_cv_enable_strict_prototypes_warning="no"
 export ac_cv_func_getnameinfo="yes"
 export ac_cv_func_poll="yes"
-export MODULE_BUILDTYPE="static"
+export ac_cv_func_gethostname="yes"

interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch

@@ -0,0 +1,54 @@
+From d1e903f516849c535455904b3c3f8a33665c1a88 Mon Sep 17 00:00:00 2001
+From: Ivan Grokhotkov <[email protected]>
+Date: Wed, 23 Oct 2024 16:52:52 +0200
+Subject: [PATCH 12/12] hack: place _PyRuntime structure into PSRAM bss region,
+ initialize later
+
+_PyRuntime occupies around 100kB of RAM in .data region, making it
+hard to fit the interpreter into the available static RAM.
+This patch moves it into PSRAM using section attribute. Normally
+we shouldn't need this as we can specify placements in ldfragments,
+however in this specific case I couldn't get it to work.
+Since the structure is now in .bss, add a function which will assign
+it the initial value.
+
+The proper fix might be to support .data segment on PSRAM in IDF,
+as well as to fix whatever ldgen issue prevents this variable from
+being moved to PSRAM.
+
+Co-authored-by: Tiago Medicci Serrano <[email protected]>
+---
+ Python/pylifecycle.c | 11 +++++++++++
+ 1 file changed, 11 insertions(+)
+
+diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
+index 1701a1cd217..2a8e544f0ac 100644
+--- a/Python/pylifecycle.c
++++ b/Python/pylifecycle.c
+@@ -102,12 +102,23 @@ __attribute__((
+ _PyRuntimeState _PyRuntime
+ #if defined(__linux__) && (defined(__GNUC__) || defined(__clang__))
+ __attribute__ ((section (".PyRuntime")))
++#elif defined(ESP_PLATFORM)
++__attribute__ ((section (".PyRuntime")))
+ #endif
+ = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie);
+ _Py_COMP_DIAG_POP
+
+ static int runtime_initialized = 0;
+
++void _PyRuntime_Early_Init(void) {
++#if defined(ESP_PLATFORM)
++    // Normally, _PyRuntime is in .data and is initialized by the C runtime.
++    // This function allows us to place it into external RAM .bss section
++    // and initialize it manually, saving some internal RAM.
++    _PyRuntime = (struct pyruntimestate) _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie);
++#endif
++}
++
+ PyStatus
+ _PyRuntime_Initialize(void)
+ {
+-- 
+2.47.1
+

interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch

@@ -0,0 +1,33 @@
+From 914c80b7969d73840bc1b573b478d9148999b7d0 Mon Sep 17 00:00:00 2001
+From: Tiago Medicci <[email protected]>
+Date: Fri, 31 Jan 2025 14:06:21 -0300
+Subject: [PATCH 13/13] transform functions used by NuttX to lowercase
+
+---
+ Include/pylifecycle.h | 10 ++++++++++
+ 1 file changed, 10 insertions(+)
+
+diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h
+index de1bcb1d2cb..044780ee188 100644
+--- a/Include/pylifecycle.h
++++ b/Include/pylifecycle.h
+@@ -33,6 +33,16 @@ PyAPI_FUNC(void) _Py_NO_RETURN Py_Exit(int);
+ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
+ PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv);
+
++#if defined(__NuttX__)
++#define py_bytesmain Py_BytesMain
++#endif
++
++void _PyRuntime_Early_Init(void);
++
++#if defined(__NuttX__)
++#define _pyruntime_early_init _PyRuntime_Early_Init
++#endif
++
+ /* In pathconfig.c */
+ Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *);
+ Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
+-- 
+2.47.1
+

interpreters/python/mount_modules.c → interpreters/python/python_wrapper.c

@@ -1,5 +1,5 @@
 /****************************************************************************
- * apps/interpreters/python/mount_modules.c
+ * apps/interpreters/python/python_wrapper.c
  *
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -41,11 +41,14 @@
 #include <fcntl.h>
 #include <dirent.h>
 #include <errno.h>
+#include <debug.h>
 
 #include <nuttx/drivers/ramdisk.h>
 
 #include "romfs_cpython_modules.h"
 
+#include "Python.h"
+
 /****************************************************************************
  * Pre-processor Definitions
  ****************************************************************************/
@@ -61,7 +64,7 @@
 #endif
 
 #ifndef CONFIG_CPYTHON_ROMFS_MOUNTPOINT
-#  define CONFIG_CPYTHON_ROMFS_MOUNTPOINT "/usr/local/lib/"
+#  define CONFIG_CPYTHON_ROMFS_MOUNTPOINT "/usr/local/lib"
 #endif
 
 #ifdef CONFIG_DISABLE_MOUNTPOINT
@@ -90,19 +93,57 @@
  ****************************************************************************/
 
 /****************************************************************************
- * Public Functions
- ****************************************************************************/
-
-/****************************************************************************
- * Name: mount_modules
+ * Name: check_and_mount_romfs
+ *
+ * Description:
+ *   Check if the ROMFS is already mounted, and if not, mount it.
+ *
+ * Input Parameters:
+ *   None
+ *
+ * Returned Value:
+ *   0 on success, 1 on failure
+ *
  ****************************************************************************/
 
-int main(int argc, FAR char *argv[])
+static int check_and_mount_romfs(void)
 {
-  int ret;
+  int ret = OK;
   struct boardioc_romdisk_s desc;
+  FILE *fp;
+  char line[256];
+  int is_mounted = 0;
 
-  /* Create a RAM disk for the test */
+  /* Check if the device is already mounted */
+
+  fp = fopen("/proc/fs/mount", "r");
+  if (fp == NULL)
+    {
+      printf("ERROR: Failed to open /proc/fs/mount\n");
+      UNUSED(desc);
+      return ret = ERROR;
+    }
+
+  while (fgets(line, sizeof(line), fp))
+    {
+      if (strstr(line, CONFIG_CPYTHON_ROMFS_MOUNTPOINT) != NULL)
+        {
+          is_mounted = 1;
+          break;
+        }
+    }
+
+  fclose(fp);
+
+  if (is_mounted)
+    {
+      _info("Device is already mounted at %s\n",
+            CONFIG_CPYTHON_ROMFS_MOUNTPOINT);
+      UNUSED(desc);
+      return ret;
+    }
+
+  /* Create a RAM disk */
 
   desc.minor    = CONFIG_CPYTHON_ROMFS_RAMDEVNO;            /* Minor device number of the ROM disk. */
   desc.nsectors = NSECTORS(romfs_cpython_modules_img_len);  /* The number of sectors in the ROM disk */
@@ -119,8 +160,8 @@ int main(int argc, FAR char *argv[])
 
   /* Mount the test file system */
 
-  printf("Mounting ROMFS filesystem at target=%s with source=%s\n",
-         CONFIG_CPYTHON_ROMFS_MOUNTPOINT, MOUNT_DEVNAME);
+  _info("Mounting ROMFS filesystem at target=%s with source=%s\n",
+        CONFIG_CPYTHON_ROMFS_MOUNTPOINT, MOUNT_DEVNAME);
 
   ret = mount(MOUNT_DEVNAME, CONFIG_CPYTHON_ROMFS_MOUNTPOINT, "romfs",
               MS_RDONLY, NULL);
@@ -132,3 +173,30 @@ int main(int argc, FAR char *argv[])
 
   return 0;
 }
+
+/****************************************************************************
+ * Public Functions
+ ****************************************************************************/
+
+/****************************************************************************
+ * Name: python_wrapper_main
+ ****************************************************************************/
+
+int main(int argc, FAR char *argv[])
+{
+  int ret;
+
+  ret = check_and_mount_romfs();
+  if (ret != 0)
+    {
+      return ret;
+    }
+
+  _pyruntime_early_init();
+
+  setenv("PYTHONHOME", "/usr/local", 1);
+
+  setenv("PYTHON_BASIC_REPL", "1", 1);
+
+  return py_bytesmain(argc, argv);
+}