ADFA-4851 | Python-Tools: syntax highlighting and code completion for .py files

plugins/Python-Tools/.gitignore

@@ -17,3 +17,4 @@ local.properties
 
 # macOS
 .DS_Store
+.cxx/

plugins/Python-Tools/README.md

@@ -11,7 +11,18 @@ Two Python project templates:
 - **Python Flask App** — a Flask web app with routes, HTML templates, static files, and error handling.
 - **Python Starter** — a minimal Python project with a single entry point.
 
-It also installs Python on-device (via Termux) the first time it is needed, and provides built-in run, install-dependencies, and test actions for Python projects.
+It also installs Python on-device the first time it is needed, and provides built-in run, install-dependencies, and test actions for Python projects.
+
+`.py` files get syntax highlighting from a bundled tree-sitter Python grammar, and code completion, diagnostics (pyflakes), and formatting (yapf) from `pylsp`, which runs on the device. Both are contributed through the plugin API's `LanguageExtension`, so the plugin needs Code on the Go 26.41 or later.
+
+## Offline bundle
+
+Python 3.12 and pip ship in Code on the Go's Termux bootstrap, so the plugin only bundles what used to need a network connection:
+
+- **Wheelhouse** - python-lsp-server with pyflakes and yapf, Flask, gunicorn, and everything they depend on. On first use the plugin installs them with `pip install --no-index` from the bundled files. `ujson` and `markupsafe` have no Android wheels, so they ship as source and are compiled on the device by the bootstrap's `clang` (with `setuptools`, `setuptools-scm`, and `wheel` bundled for the build).
+- **Project requirements** are installed automatically when a Python project opens, looking in the wheelhouse first, so the Flask template (Flask and gunicorn) needs no network. Other packages in a project's `requirements.txt` still download from PyPI.
+
+`python-bundle.lock` pins every file by URL and SHA-256. `downloadPythonBundle` fetches them at build time (resuming interrupted transfers) and refuses any file whose checksum does not match. To update, regenerate the lock from a `pip install --dry-run --report` run on a device.
 
 ## In-app help
 

plugins/Python-Tools/build.gradle.kts

@@ -1,3 +1,10 @@
+import java.io.FileOutputStream
+import java.io.IOException
+import java.net.HttpURLConnection
+import java.net.URI
+import java.security.MessageDigest
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
 import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
@@ -9,6 +16,164 @@ pluginBuilder {
     pluginName = "python-tools"
 }
 
+data class GrammarSource(
+    val name: String,
+    val version: String,
+    val sha256: String,
+) {
+    val url = "https://registry.npmjs.org/tree-sitter-$name/-/tree-sitter-$name-$version.tgz"
+}
+
+val grammarSource = GrammarSource("python", "0.23.6", "aab5b860d93dbf84b37fed532f7dacc0468ede5387d0c74911d68f964287662a")
+val grammarDir = layout.buildDirectory.dir("tree-sitter/${grammarSource.name}")
+
+val DOWNLOAD_ATTEMPTS = 40
+
+fun sha256Of(file: File): String {
+    val digest = MessageDigest.getInstance("SHA-256")
+    file.inputStream().use { stream ->
+        val buffer = ByteArray(1 shl 16)
+        while (true) {
+            val read = stream.read(buffer)
+            if (read <= 0) break
+            digest.update(buffer, 0, read)
+        }
+    }
+    return digest.digest().joinToString("") { "%02x".format(it) }
+}
+
+fun fetchVerified(
+    url: String,
+    sha256: String,
+    target: File,
+) {
+    if (target.exists() && sha256Of(target) == sha256) return
+    target.parentFile.mkdirs()
+    target.delete()
+    logger.lifecycle("Downloading $url")
+    var attempt = 0
+    while (true) {
+        attempt++
+        val offset = target.length()
+        val connection = URI(url).toURL().openConnection() as HttpURLConnection
+        connection.connectTimeout = 30_000
+        connection.readTimeout = 120_000
+        if (offset > 0) connection.setRequestProperty("Range", "bytes=$offset-")
+        val interrupted =
+            try {
+                val append = offset > 0 && connection.responseCode == HttpURLConnection.HTTP_PARTIAL
+                connection.inputStream.use { input ->
+                    FileOutputStream(target, append).use { output -> input.copyTo(output) }
+                }
+                null
+            } catch (e: IOException) {
+                e
+            } finally {
+                connection.disconnect()
+            }
+        if (interrupted == null && connection.contentLengthLong in listOf(-1L, target.length() - offset, target.length())) break
+        if (attempt >= DOWNLOAD_ATTEMPTS) {
+            target.delete()
+            throw GradleException("Failed to download $url after $attempt attempts: ${interrupted?.message}", interrupted)
+        }
+        logger.lifecycle("Resuming $url at ${target.length()} bytes (attempt ${attempt + 1})")
+    }
+    val actual = sha256Of(target)
+    if (actual != sha256) {
+        target.delete()
+        throw GradleException("Checksum mismatch for $url (expected $sha256, got $actual)")
+    }
+}
+
+data class BundleEntry(
+    val directory: String,
+    val fileName: String,
+    val url: String,
+    val sha256: String,
+)
+
+val pythonBundleLock = layout.projectDirectory.file("python-bundle.lock")
+val pythonBundleCache = layout.buildDirectory.dir("python-bundle-cache")
+val pythonBundleDir = layout.buildDirectory.dir("generated/pythonBundle")
+
+val downloadPythonBundle =
+    tasks.register("downloadPythonBundle") {
+        group = "setup"
+        description = "Download and verify the Python wheels bundled into the plugin"
+
+        val lock = pythonBundleLock
+        val cacheDir = pythonBundleCache
+        val outputDir = pythonBundleDir
+        inputs.file(lock)
+        outputs.dir(outputDir)
+
+        doLast {
+            val entries =
+                lock.asFile.readLines().filter { it.isNotBlank() }.map { line ->
+                    val fields = line.split('\t')
+                    if (fields.size != 4) throw GradleException("Malformed python-bundle.lock line: $line")
+                    BundleEntry(fields[0], fields[1], fields[2], fields[3])
+                }
+            val cache = cacheDir.get().asFile
+            entries.forEach { fetchVerified(it.url, it.sha256, File(cache, "${it.directory}/${it.fileName}")) }
+
+            val zipFile = File(outputDir.get().asFile, "python/wheelhouse.zip")
+            zipFile.parentFile.deleteRecursively()
+            zipFile.parentFile.mkdirs()
+            ZipOutputStream(zipFile.outputStream()).use { zip ->
+                entries.filter { it.directory == "wheelhouse" }.sortedBy { it.fileName }.forEach { entry ->
+                    zip.putNextEntry(ZipEntry(entry.fileName).apply { time = 0L })
+                    File(cache, "${entry.directory}/${entry.fileName}").inputStream().use { it.copyTo(zip) }
+                    zip.closeEntry()
+                }
+            }
+        }
+    }
+
+val downloadGrammar =
+    tasks.register("downloadTreeSitterGrammar") {
+        group = "setup"
+        description = "Download, verify, and unpack the tree-sitter grammar compiled into the plugin"
+
+        val source = grammarSource
+        val outputDir = grammarDir
+        val archive = layout.buildDirectory.file("tree-sitter/${source.name}-${source.version}.tgz")
+        inputs.property("grammarVersion", source.version)
+        inputs.property("grammarUrl", source.url)
+        inputs.property("grammarSha256", source.sha256)
+        outputs.dir(outputDir)
+
+        doLast {
+            val archiveFile = archive.get().asFile
+            fetchVerified(source.url, source.sha256, archiveFile)
+            val target = outputDir.get().asFile
+            target.deleteRecursively()
+            project.copy {
+                from(project.tarTree(project.resources.gzip(archiveFile)))
+                include("package/src/**")
+                eachFile { path = path.removePrefix("package/") }
+                includeEmptyDirs = false
+                into(target)
+            }
+        }
+    }
+
+tasks.named("preBuild") {
+    dependsOn(downloadGrammar, downloadPythonBundle)
+}
+
+tasks.matching { it.name.startsWith("configureCMake") || it.name.startsWith("buildCMake") }.configureEach {
+    dependsOn(downloadGrammar)
+}
+
+downloadGrammar.configure {
+    mustRunAfter(tasks.named("clean"))
+}
+
+downloadPythonBundle.configure {
+    mustRunAfter(tasks.named("clean"))
+}
+
 android {
     namespace = "com.appdevforall.python.plugin"
     compileSdk = 36
@@ -17,8 +182,38 @@ android {
         applicationId = "com.appdevforall.python.plugin"
         minSdk = 26
         targetSdk = 36
-        versionCode = 1
-        versionName = "1.0.0"
+        versionCode = 3
+        versionName = "1.2.0"
+
+        ndk {
+            abiFilters += listOf("arm64-v8a", "armeabi-v7a")
+        }
+
+        externalNativeBuild {
+            cmake {
+                arguments += "-DTREE_SITTER_GRAMMAR_NAME=${grammarSource.name}"
+                arguments += "-DTREE_SITTER_GRAMMAR_DIR=${grammarDir.get().asFile.absolutePath}"
+                arguments += "-DCMAKE_BUILD_TYPE=Release"
+                arguments += "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON"
+            }
+        }
+    }
+
+    androidResources {
+        noCompress += listOf("zip")
+    }
+
+    sourceSets {
+        named("main") {
+            assets.srcDir(pythonBundleDir.get().asFile)
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path("src/main/cpp/CMakeLists.txt")
+            version = "3.22.1"
+        }
     }
 
     buildTypes {

plugins/Python-Tools/python-bundle.lock

@@ -0,0 +1,31 @@
+wheelhouse	black-26.5.1-py3-none-any.whl	https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl	4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2
+wheelhouse	blinker-1.9.0-py3-none-any.whl	https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl	ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc
+wheelhouse	click-8.5.0-py3-none-any.whl	https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl	255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360
+wheelhouse	docstring_to_markdown-0.17-py3-none-any.whl	https://files.pythonhosted.org/packages/56/7b/af3d0da15bed3a8665419bb3a630585756920f4ad67abfdfef26240ebcc0/docstring_to_markdown-0.17-py3-none-any.whl	fd7d5094aa83943bf5f9e1a13701866b7c452eac19765380dead666e36d3711c
+wheelhouse	flask-3.1.3-py3-none-any.whl	https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl	f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c
+wheelhouse	gunicorn-26.2.0-py3-none-any.whl	https://files.pythonhosted.org/packages/fe/85/7522a52e5e2f42faf1a129113ab63e548c42e103e9af395b7bfe65e403e2/gunicorn-26.2.0-py3-none-any.whl	bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3
+wheelhouse	importlib_metadata-9.0.1-py3-none-any.whl	https://files.pythonhosted.org/packages/b3/55/ecca97ae19075f1fac62def77731e7f535e6c1fb8f92ff08160c5e6dade8/importlib_metadata-9.0.1-py3-none-any.whl	bba5600596a7e21f3eef53281cf28d6a5195634d2f2b78ff9501a3272c6eaab0
+wheelhouse	itsdangerous-2.2.0-py3-none-any.whl	https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl	c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef
+wheelhouse	jedi-0.20.0-py2.py3-none-any.whl	https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl	7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67
+wheelhouse	jinja2-3.1.6-py3-none-any.whl	https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl	85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
+wheelhouse	markupsafe-3.0.3.tar.gz	https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz	722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698
+wheelhouse	mypy_extensions-1.1.0-py3-none-any.whl	https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl	1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505
+wheelhouse	packaging-26.3-py3-none-any.whl	https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl	d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c
+wheelhouse	parso-0.8.7-py2.py3-none-any.whl	https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl	a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c
+wheelhouse	pathspec-1.1.1-py3-none-any.whl	https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl	a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189
+wheelhouse	platformdirs-4.12.2-py3-none-any.whl	https://files.pythonhosted.org/packages/d0/89/446044f33aba0348d35e433f56d12206d010a5281a1df54054d4cfb82388/platformdirs-4.12.2-py3-none-any.whl	29dbf06d96c500bc6bdbce75fb0a14d63279c93b1842f97e72a135b33e856983
+wheelhouse	pluggy-1.6.0-py3-none-any.whl	https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl	e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746
+wheelhouse	pyflakes-3.2.0-py2.py3-none-any.whl	https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl	84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a
+wheelhouse	python_lsp_jsonrpc-1.1.2-py3-none-any.whl	https://files.pythonhosted.org/packages/cb/d9/656659d5b5d5f402b2b174cd0ba9bc827e07ce3c0bf88da65424baf64af8/python_lsp_jsonrpc-1.1.2-py3-none-any.whl	7339c2e9630ae98903fdaea1ace8c47fba0484983794d6aafd0bd8989be2b03c
+wheelhouse	python_lsp_server-1.15.0-py3-none-any.whl	https://files.pythonhosted.org/packages/c2/4a/b4abc04dcc65c985fa66d6486393ad3051ff485ccabd9727aea43a242f88/python_lsp_server-1.15.0-py3-none-any.whl	d6ac11b467021310498f2dffb2edf09629bbcfe0d3073156db2337334a60463f
+wheelhouse	pytokens-0.4.1-py3-none-any.whl	https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl	26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de
+wheelhouse	setuptools-84.0.0-py3-none-any.whl	https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl	51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670
+wheelhouse	setuptools_scm-10.3.4-py3-none-any.whl	https://files.pythonhosted.org/packages/da/f5/54538a1f17ea753c42b0928dc2e62986d463f8c393340b9a1e602f2d4dd9/setuptools_scm-10.3.4-py3-none-any.whl	82f34c3e3084fc2b57d397200637cc13f2338004052d2fee47baa5f0e902464e
+wheelhouse	typing_extensions-4.16.0-py3-none-any.whl	https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl	481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8
+wheelhouse	ujson-6.0.0.tar.gz	https://files.pythonhosted.org/packages/64/7c/e1fa3fb70b53192436d751b5cb671f0ee960baa188b8351a7fec735223d3/ujson-6.0.0.tar.gz	80e23393feb707582e0ad495c397a4477b646d08094d2df64f7316f9fafd8aae
+wheelhouse	vcs_versioning-2.5.0-py3-none-any.whl	https://files.pythonhosted.org/packages/e4/e6/b4dedd1efea8a1e432123575896328dec83c26327164f7098db415d13f6f/vcs_versioning-2.5.0-py3-none-any.whl	dbf44f6303dc817e5792cb7d0e9cc90e3046ee1e32eaa357b20ad368811313d0
+wheelhouse	werkzeug-3.1.9-py3-none-any.whl	https://files.pythonhosted.org/packages/a1/38/df03f564f43cec2684823f3cccae1a652ee7face1cbaa76fb223096e64d7/werkzeug-3.1.9-py3-none-any.whl	6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab
+wheelhouse	whatthepatch-1.0.7-py3-none-any.whl	https://files.pythonhosted.org/packages/8e/93/af1d6ccb69ab6b5a00e03fa0cefa563f9862412667776ea15dd4eece3a90/whatthepatch-1.0.7-py3-none-any.whl	1b6f655fd31091c001c209529dfaabbabdbad438f5de14e3951266ea0fc6e7ed
+wheelhouse	wheel-0.48.0-py3-none-any.whl	https://files.pythonhosted.org/packages/2e/29/69cfbb602cd91690c55d38ba9fe53e6a7e76a6fa647bf38f19c138d25449/wheel-0.48.0-py3-none-any.whl	3217dcc807155e45db462d7ef2431f5ddda0d7273b700d05a67b271ceb1287ab
+wheelhouse	yapf-0.43.0-py3-none-any.whl	https://files.pythonhosted.org/packages/37/81/6acd6601f61e31cfb8729d3da6d5df966f80f374b78eff83760714487338/yapf-0.43.0-py3-none-any.whl	224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca
+wheelhouse	zipp-4.1.0-py3-none-any.whl	https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl	25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f

plugins/Python-Tools/python-tools.html

@@ -231,6 +231,12 @@ <h3>Hiding the Gradle actions</h3>
   </p>
   <pre><code>override fun toolbarActionsToHide(): Set&lt;String&gt; =
     if (isPythonProjectOpen()) ToolbarActionIds.BUILD_HIDEABLE else emptySet()</code></pre>
+  <p>
+    Quick Run is not in that set. The plugin hides it through
+    <code>UIExtension</code>, which accepts any toolbar action id:
+  </p>
+  <pre><code>override fun getHiddenToolbarActionIds(): Set&lt;String&gt; =
+    if (isPythonProjectOpen()) setOf("ide.editor.build.quickBuild") else emptySet()</code></pre>
 
   <h3>Process lifecycle</h3>
   <p>

plugins/Python-Tools/src/main/AndroidManifest.xml

@@ -27,11 +27,11 @@
 
         <meta-data
             android:name="plugin.min_ide_version"
-            android:value="1.0.0" />
+            android:value="26.41" />
 
         <meta-data
             android:name="plugin.permissions"
-            android:value="filesystem.read,filesystem.write,system.commands,project.structure" />
+            android:value="filesystem.read,filesystem.write,system.commands,project.structure,native.code" />
 
         <meta-data
             android:name="plugin.main_class"

plugins/Python-Tools/src/main/assets/treesitter/python/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.