From 6a18f9626f53e0ae880a720c051b1240961c521d Mon Sep 17 00:00:00 2001 From: Carsten Behring Date: Sat, 30 Aug 2025 18:40:08 +0200 Subject: [PATCH 1/9] uv integration --- .gitignore | 3 +- deps.edn | 178 ++++++++++++++++++++++++++++++- python.edn | 3 + src/libpython_clj2/python.clj | 1 + src/libpython_clj2/python/uv.clj | 97 +++++++++++++++++ 5 files changed, 278 insertions(+), 4 deletions(-) create mode 100644 python.edn create mode 100644 src/libpython_clj2/python/uv.clj diff --git a/.gitignore b/.gitignore index 6bd4f64..0b26a2e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ a.out *.iml .lsp .clj-kondo -pregen-ffi-test \ No newline at end of file +pregen-ffi-test +.calva diff --git a/deps.edn b/deps.edn index 77a6c5f..f6a9a0d 100644 --- a/deps.edn +++ b/deps.edn @@ -1,13 +1,17 @@ {:paths ["src"] - :deps {org.clojure/clojure {:mvn/version "1.11.1" :scope "provided"} + :deps {org.clojure/clojure {:mvn/version "1.12.0" } cnuernber/dtype-next {:mvn/version "10.111"} net.java.dev.jna/jna {:mvn/version "5.12.1"} org.clojure/data.json {:mvn/version "1.0.0"} + cheshire/cheshire {:mvn/version "5.13.0"} + scicloj/clojisr {:mvn/version "1.0.0"} ;;Replace me with caffeine... com.google.guava/guava {:mvn/version "31.1-jre"}} + + :aliases {:dev - {:extra-deps {criterium/criterium {:mvn/version"0.4.5"} + {:extra-deps {criterium/criterium {:mvn/version "0.4.5"} ch.qos.logback/logback-classic {:mvn/version "1.1.3"}}} :fastcall {:jvm-opts ["-Dlibpython_clj.manual_gil=true"]} @@ -77,4 +81,172 @@ :exec-args {:installer :local :artifact "target/libpython-clj.jar"}} - }} + } + + :python-version "3.13.7" + :python-deps [ + "openai==1.58.1" + ] + +:renv-deps {:R + {:Version "4.4.1", + :Repositories [{:Name "CRAN", :URL "https://cloud.r-project.org"}]}, + :Packages + {:tidyselect + {:Package "tidyselect", + :Version "1.2.1", + :Source "Repository", + :Repository "CRAN", + :Requirements + ["R" "cli" "glue" "lifecycle" "rlang" "vctrs" "withr"], + :Hash "829f27b9c4919c16b593794a6344d6c0"}, + :withr + {:Package "withr", + :Version "3.0.1", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "grDevices" "graphics"], + :Hash "07909200e8bbe90426fbfeb73e1e27aa"}, + :utf8 + {:Package "utf8", + :Version "1.2.4", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R"], + :Hash "62b65c52671e6665f803ff02954446e9"}, + :rlang + {:Package "rlang", + :Version "1.1.4", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "utils"], + :Hash "3eec01f8b1dee337674b2e34ab1f9bc1"}, + :glue + {:Package "glue", + :Version "1.7.0", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "methods"], + :Hash "e0b3a53876554bd45879e596cdb10a52"}, + :vctrs + {:Package "vctrs", + :Version "0.6.5", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "cli" "glue" "lifecycle" "rlang"], + :Hash "c03fa420630029418f7e6da3667aac4a"}, + :tibble + {:Package "tibble", + :Version "3.2.1", + :Source "Repository", + :Repository "CRAN", + :Requirements + ["R" + "fansi" + "lifecycle" + "magrittr" + "methods" + "pillar" + "pkgconfig" + "rlang" + "utils" + "vctrs"], + :Hash "a84e2cc86d07289b3b6f5069df7a004c"}, + :magrittr + {:Package "magrittr", + :Version "2.0.3", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R"], + :Hash "7ce2733a9826b3aeb1775d56fd305472"}, + :lifecycle + {:Package "lifecycle", + :Version "1.0.4", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "cli" "glue" "rlang"], + :Hash "b8552d117e1b808b09a832f589b79035"}, + :dplyr + {:Package "dplyr", + :Version "1.1.4", + :Source "Repository", + :Repository "CRAN", + :Requirements + ["R" + "R6" + "cli" + "generics" + "glue" + "lifecycle" + "magrittr" + "methods" + "pillar" + "rlang" + "tibble" + "tidyselect" + "utils" + "vctrs"], + :Hash "fedd9d00c2944ff00a0e2696ccf048ec"}, + :pkgconfig + {:Package "pkgconfig", + :Version "2.0.3", + :Source "Repository", + :Repository "CRAN", + :Requirements ["utils"], + :Hash "01f28d4278f15c76cddbea05899c5d6f"}, + :fansi + {:Package "fansi", + :Version "1.0.6", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "grDevices" "utils"], + :Hash "962174cf2aeb5b9eea581522286a911f"}, + :renv + {:Package "renv", + :Version "1.0.7", + :Source "Repository", + :Repository "CRAN", + :Requirements ["utils"], + :Hash "397b7b2a265bc5a7a06852524dabae20"}, + :pillar + {:Package "pillar", + :Version "1.9.0", + :Source "Repository", + :Repository "CRAN", + :Requirements + ["cli" "fansi" "glue" "lifecycle" "rlang" "utf8" "utils" "vctrs"], + :Hash "15da5a8412f317beeee6175fbc76f4bb"}, + :generics + {:Package "generics", + :Version "0.1.3", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "methods"], + :Hash "15e9634c0fcd294799e9b2e929ed1b86"}, + :cli + {:Package "cli", + :Version "3.6.3", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R" "utils"], + :Hash "b21916dd77a27642b447374a5d30ecf3"}, + :R6 + {:Package "R6", + :Version "2.5.1", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R"], + :Hash "470851b6d5d0ac559e9d01bb352b4021"} +:Rserve +{:Package "Rserve", + :Version "1.8-13", + :Source "Repository", + :Repository "CRAN", + :Requirements ["R"], + :Hash "589ebd25cffe93ab5ab61e66936d2678"} + }} + + + + + } diff --git a/python.edn b/python.edn new file mode 100644 index 0000000..c269a6e --- /dev/null +++ b/python.edn @@ -0,0 +1,3 @@ +{:python-executable ".venv/bin/python" + :pre-initialize-fn libpython-clj2.python.uv/setup!} + diff --git a/src/libpython_clj2/python.clj b/src/libpython_clj2/python.clj index 98ab73b..63b21fd 100644 --- a/src/libpython_clj2/python.clj +++ b/src/libpython_clj2/python.clj @@ -113,6 +113,7 @@ user> (py/py. np linspace 2 3 :num 10) (let [python-edn-opts (-> (try (slurp "python.edn") (catch java.io.FileNotFoundException _ "{}")) clojure.edn/read-string) + _ (log/infof "Pre-initialize-fn %s" (some-> python-edn-opts :pre-initialize-fn)) _ (some-> python-edn-opts :pre-initialize-fn requiring-resolve (apply [])) options (merge python-edn-opts options) info (py-info/detect-startup-info options) diff --git a/src/libpython_clj2/python/uv.clj b/src/libpython_clj2/python/uv.clj new file mode 100644 index 0000000..799df7c --- /dev/null +++ b/src/libpython_clj2/python/uv.clj @@ -0,0 +1,97 @@ +(ns libpython-clj2.python.uv + (:require + [clojure.edn :as edn] + [clojure.java.process :as process] + [clojure.string :as str] + [cheshire.core :as json] + )) + +(defn write-pyproject-toml! [deps-edn] + + (let [ + + python-deps + (:python-deps deps-edn) + + ;python-version (:python-version deps-edn) + + py-project-header-lines ["[project]" + "name = \"temp\"" + "version = \"0.0\"" + ;(format "requires-python = \"==%s\"" python-version) + ] + python-deps-lines + (map + (fn [dep] + (format "\"%s\"," dep)) + python-deps) + + py-project-lines + (concat + py-project-header-lines + [ "dependencies = ["] + python-deps-lines + "]\n") + ] + + (spit "pyproject.toml" + (str/join "\n" py-project-lines)))) + +(defn char-seq + [^java.io.Reader rdr] + (let [chr (.read rdr)] + (if (>= chr 0) + (do + ;(def chr chr) + (print (char (Integer. chr))) + (flush) + (cons chr (lazy-seq (char-seq rdr))))))) + + +(defn start-and-print! [process-args] + (let [args + (concat [{:env { "RENV_CONFIG_INSTALL_VERBOSE" "TRUE"}}] + process-args) + p + (apply process/start args)] + + (with-open [in-rdr (java.io.InputStreamReader. (.getInputStream p)) + err-rdr (java.io.InputStreamReader. (.getErrorStream p))] + + + + (dorun (char-seq in-rdr)) + (dorun (char-seq err-rdr))))) + + +(defn setup-python! [] + (println "Synchronize python venv with uv sync. This might take a few minutes") + (let [deps-edn + (-> + (slurp "deps.edn") + edn/read-string) + + ] + (write-pyproject-toml! deps-edn) + (start-and-print! [ "unbuffer" "uv" "sync" "--python" (-> deps-edn :python-version)])) + ) + +(defn setup-r! [] + (let [renv-lock + (-> + (slurp "deps.edn") + edn/read-string + :renv-deps + (json/generate-string {:pretty true}))] + (spit "renv.lock" renv-lock)) + + (start-and-print! ["rm" "-rf" "renv"]) + (start-and-print! ["rm" ".Rprofile"]) + (start-and-print! ["Rscript" "-e" "renv::init()"]) + (start-and-print! ["Rscript" "-e" "renv::restore()"])) + +(defn setup! [& args] + (setup-python!) + (setup-r!) + ) + From 724a76c0fde8cfb93c7fb229943a0d8954e385ff Mon Sep 17 00:00:00 2001 From: Carsten Behring Date: Sat, 30 Aug 2025 19:55:11 +0000 Subject: [PATCH 2/9] refcatored executable --- .gitignore | 1 + python.edn | 3 --- src/libpython_clj2/python/uv.clj | 21 ++++++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) delete mode 100644 python.edn diff --git a/.gitignore b/.gitignore index 0b26a2e..f38e7c7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ a.out .clj-kondo pregen-ffi-test .calva +.lock diff --git a/python.edn b/python.edn deleted file mode 100644 index c269a6e..0000000 --- a/python.edn +++ /dev/null @@ -1,3 +0,0 @@ -{:python-executable ".venv/bin/python" - :pre-initialize-fn libpython-clj2.python.uv/setup!} - diff --git a/src/libpython_clj2/python/uv.clj b/src/libpython_clj2/python/uv.clj index 799df7c..dafecae 100644 --- a/src/libpython_clj2/python/uv.clj +++ b/src/libpython_clj2/python/uv.clj @@ -1,10 +1,9 @@ (ns libpython-clj2.python.uv - (:require - [clojure.edn :as edn] - [clojure.java.process :as process] - [clojure.string :as str] - [cheshire.core :as json] - )) + (:require + [clojure.edn :as edn] + [clojure.java.process :as process] + [clojure.string :as str] + [cheshire.core :as json])) (defn write-pyproject-toml! [deps-edn] @@ -92,6 +91,14 @@ (defn setup! [& args] (setup-python!) - (setup-r!) + ;(setup-r!) ) +(comment + (require 'libpython-clj2.python) + (require 'libpython-clj2.python.uv) + (libpython-clj2.python.uv/setup!) + (libpython-clj2.python/initialize! :python-executable ".venv/bin/python") + (libpython-clj2.python/run-simple-string "import sys; print(sys.path)") + (libpython-clj2.python/import-module "openai") + ) \ No newline at end of file From 65fa30a4724489e8d10b0a2e7bd86856e23771f1 Mon Sep 17 00:00:00 2001 From: BEHRING Carsten Date: Sun, 31 Aug 2025 12:07:50 +0200 Subject: [PATCH 3/9] use python.edn --- deps.edn | 168 +------------------------------ python.edn | 8 ++ src/libpython_clj2/python.clj | 20 ++++ src/libpython_clj2/python/uv.clj | 71 ++++--------- 4 files changed, 49 insertions(+), 218 deletions(-) create mode 100644 python.edn diff --git a/deps.edn b/deps.edn index f6a9a0d..720ca01 100644 --- a/deps.edn +++ b/deps.edn @@ -82,171 +82,5 @@ :artifact "target/libpython-clj.jar"}} } - - :python-version "3.13.7" - :python-deps [ - "openai==1.58.1" - ] -:renv-deps {:R - {:Version "4.4.1", - :Repositories [{:Name "CRAN", :URL "https://cloud.r-project.org"}]}, - :Packages - {:tidyselect - {:Package "tidyselect", - :Version "1.2.1", - :Source "Repository", - :Repository "CRAN", - :Requirements - ["R" "cli" "glue" "lifecycle" "rlang" "vctrs" "withr"], - :Hash "829f27b9c4919c16b593794a6344d6c0"}, - :withr - {:Package "withr", - :Version "3.0.1", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "grDevices" "graphics"], - :Hash "07909200e8bbe90426fbfeb73e1e27aa"}, - :utf8 - {:Package "utf8", - :Version "1.2.4", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R"], - :Hash "62b65c52671e6665f803ff02954446e9"}, - :rlang - {:Package "rlang", - :Version "1.1.4", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "utils"], - :Hash "3eec01f8b1dee337674b2e34ab1f9bc1"}, - :glue - {:Package "glue", - :Version "1.7.0", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "methods"], - :Hash "e0b3a53876554bd45879e596cdb10a52"}, - :vctrs - {:Package "vctrs", - :Version "0.6.5", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "cli" "glue" "lifecycle" "rlang"], - :Hash "c03fa420630029418f7e6da3667aac4a"}, - :tibble - {:Package "tibble", - :Version "3.2.1", - :Source "Repository", - :Repository "CRAN", - :Requirements - ["R" - "fansi" - "lifecycle" - "magrittr" - "methods" - "pillar" - "pkgconfig" - "rlang" - "utils" - "vctrs"], - :Hash "a84e2cc86d07289b3b6f5069df7a004c"}, - :magrittr - {:Package "magrittr", - :Version "2.0.3", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R"], - :Hash "7ce2733a9826b3aeb1775d56fd305472"}, - :lifecycle - {:Package "lifecycle", - :Version "1.0.4", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "cli" "glue" "rlang"], - :Hash "b8552d117e1b808b09a832f589b79035"}, - :dplyr - {:Package "dplyr", - :Version "1.1.4", - :Source "Repository", - :Repository "CRAN", - :Requirements - ["R" - "R6" - "cli" - "generics" - "glue" - "lifecycle" - "magrittr" - "methods" - "pillar" - "rlang" - "tibble" - "tidyselect" - "utils" - "vctrs"], - :Hash "fedd9d00c2944ff00a0e2696ccf048ec"}, - :pkgconfig - {:Package "pkgconfig", - :Version "2.0.3", - :Source "Repository", - :Repository "CRAN", - :Requirements ["utils"], - :Hash "01f28d4278f15c76cddbea05899c5d6f"}, - :fansi - {:Package "fansi", - :Version "1.0.6", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "grDevices" "utils"], - :Hash "962174cf2aeb5b9eea581522286a911f"}, - :renv - {:Package "renv", - :Version "1.0.7", - :Source "Repository", - :Repository "CRAN", - :Requirements ["utils"], - :Hash "397b7b2a265bc5a7a06852524dabae20"}, - :pillar - {:Package "pillar", - :Version "1.9.0", - :Source "Repository", - :Repository "CRAN", - :Requirements - ["cli" "fansi" "glue" "lifecycle" "rlang" "utf8" "utils" "vctrs"], - :Hash "15da5a8412f317beeee6175fbc76f4bb"}, - :generics - {:Package "generics", - :Version "0.1.3", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "methods"], - :Hash "15e9634c0fcd294799e9b2e929ed1b86"}, - :cli - {:Package "cli", - :Version "3.6.3", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R" "utils"], - :Hash "b21916dd77a27642b447374a5d30ecf3"}, - :R6 - {:Package "R6", - :Version "2.5.1", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R"], - :Hash "470851b6d5d0ac559e9d01bb352b4021"} -:Rserve -{:Package "Rserve", - :Version "1.8-13", - :Source "Repository", - :Repository "CRAN", - :Requirements ["R"], - :Hash "589ebd25cffe93ab5ab61e66936d2678"} - }} - - - - - } + } diff --git a/python.edn b/python.edn new file mode 100644 index 0000000..10265ca --- /dev/null +++ b/python.edn @@ -0,0 +1,8 @@ +{ + :python-version "3.10.16" + :python-deps ["openai==1.58.1" + "langextract"] + :python-executable ".venv/Scripts/python" + :pre-initialize-fn libpython-clj2.python.uv/sync-python-setup! + +} \ No newline at end of file diff --git a/src/libpython_clj2/python.clj b/src/libpython_clj2/python.clj index 63b21fd..7141517 100644 --- a/src/libpython_clj2/python.clj +++ b/src/libpython_clj2/python.clj @@ -114,7 +114,17 @@ user> (py/py. np linspace 2 3 :num 10) (catch java.io.FileNotFoundException _ "{}")) clojure.edn/read-string) _ (log/infof "Pre-initialize-fn %s" (some-> python-edn-opts :pre-initialize-fn)) + _ (def python-edn-opts python-edn-opts) _ (some-> python-edn-opts :pre-initialize-fn requiring-resolve (apply [])) + _ (some-> python-edn-opts :pre-initialize-fn) + ;_ ( (requiring-resolve 'libpython-clj2.python.uv/sync-python-setup)) + + ;(require 'libpython-clj2.python.uv) + + ;(requiring-resolve ) + + ;(-> 'libpython-clj2.python.uv/char-seq namespace symbol require) + ;(resolve 'libpython-clj2.python.uv/char-seq) options (merge python-edn-opts options) info (py-info/detect-startup-info options) _ (log/infof "Startup info %s" info) @@ -820,3 +830,13 @@ user> c [symbols input] `(let [~symbols ~input] ~@(for [s symbols] `(def ~s ~s)))) + + +(comment + (require 'libpython-clj2.python) + (require 'libpython-clj2.python.uv) + + (libpython-clj2.python/initialize!) + (libpython-clj2.python/run-simple-string "import sys; print(sys.path)") + (libpython-clj2.python/import-module "openai") + (libpython-clj2.python/import-module "langextract")) \ No newline at end of file diff --git a/src/libpython_clj2/python/uv.clj b/src/libpython_clj2/python/uv.clj index dafecae..87cd3b3 100644 --- a/src/libpython_clj2/python/uv.clj +++ b/src/libpython_clj2/python/uv.clj @@ -7,18 +7,16 @@ (defn write-pyproject-toml! [deps-edn] - (let [ - - python-deps - (:python-deps deps-edn) - + (let [python-deps + (:python-deps deps-edn) + ;python-version (:python-version deps-edn) - + py-project-header-lines ["[project]" "name = \"temp\"" "version = \"0.0\"" ;(format "requires-python = \"==%s\"" python-version) - ] + ] python-deps-lines (map (fn [dep] @@ -28,10 +26,9 @@ py-project-lines (concat py-project-header-lines - [ "dependencies = ["] + ["dependencies = ["] python-deps-lines - "]\n") - ] + "]\n")] (spit "pyproject.toml" (str/join "\n" py-project-lines)))) @@ -39,7 +36,7 @@ (defn char-seq [^java.io.Reader rdr] (let [chr (.read rdr)] - (if (>= chr 0) + (when (>= chr 0) (do ;(def chr chr) (print (char (Integer. chr))) @@ -48,12 +45,12 @@ (defn start-and-print! [process-args] - (let [args - (concat [{:env { "RENV_CONFIG_INSTALL_VERBOSE" "TRUE"}}] - process-args) - p + (let [args + (concat [{:env {"RENV_CONFIG_INSTALL_VERBOSE" "TRUE"}}] + process-args) + p (apply process/start args)] - + (with-open [in-rdr (java.io.InputStreamReader. (.getInputStream p)) err-rdr (java.io.InputStreamReader. (.getErrorStream p))] @@ -63,42 +60,14 @@ (dorun (char-seq err-rdr))))) -(defn setup-python! [] - (println "Synchronize python venv with uv sync. This might take a few minutes") +(defn sync-python-setup! [] + (println "Synchronize python venv at .venv with 'uv sync'. This might take a few minutes") (let [deps-edn (-> - (slurp "deps.edn") - edn/read-string) - - ] + (slurp "python.edn") + edn/read-string)] (write-pyproject-toml! deps-edn) - (start-and-print! [ "unbuffer" "uv" "sync" "--python" (-> deps-edn :python-version)])) - ) + (start-and-print! ["uv" "sync" "--python" (-> deps-edn :python-version)]))) + + -(defn setup-r! [] - (let [renv-lock - (-> - (slurp "deps.edn") - edn/read-string - :renv-deps - (json/generate-string {:pretty true}))] - (spit "renv.lock" renv-lock)) - - (start-and-print! ["rm" "-rf" "renv"]) - (start-and-print! ["rm" ".Rprofile"]) - (start-and-print! ["Rscript" "-e" "renv::init()"]) - (start-and-print! ["Rscript" "-e" "renv::restore()"])) - -(defn setup! [& args] - (setup-python!) - ;(setup-r!) - ) - -(comment - (require 'libpython-clj2.python) - (require 'libpython-clj2.python.uv) - (libpython-clj2.python.uv/setup!) - (libpython-clj2.python/initialize! :python-executable ".venv/bin/python") - (libpython-clj2.python/run-simple-string "import sys; print(sys.path)") - (libpython-clj2.python/import-module "openai") - ) \ No newline at end of file From 862845eb5f538b89869c3db5483e72bd072b22f1 Mon Sep 17 00:00:00 2001 From: BEHRING Carsten Date: Sun, 31 Aug 2025 18:28:15 +0200 Subject: [PATCH 4/9] added required-python --- python.edn | 8 ++++---- src/libpython_clj2/python.clj | 6 +++++- src/libpython_clj2/python/uv.clj | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/python.edn b/python.edn index 10265ca..926344d 100644 --- a/python.edn +++ b/python.edn @@ -1,8 +1,8 @@ { - :python-version "3.10.16" - :python-deps ["openai==1.58.1" - "langextract"] + ;:python-version "3.10.16" + ;:python-deps ["openai==1.58.1" + ; "langextract"] :python-executable ".venv/Scripts/python" - :pre-initialize-fn libpython-clj2.python.uv/sync-python-setup! + ;:pre-initialize-fn libpython-clj2.python.uv/sync-python-setup! } \ No newline at end of file diff --git a/src/libpython_clj2/python.clj b/src/libpython_clj2/python.clj index 7141517..ee84922 100644 --- a/src/libpython_clj2/python.clj +++ b/src/libpython_clj2/python.clj @@ -837,6 +837,10 @@ user> c (require 'libpython-clj2.python.uv) (libpython-clj2.python/initialize!) + (libpython-clj2.python/run-simple-string "import sys; sys.path.append('.venv/Lib/site-packages')") (libpython-clj2.python/run-simple-string "import sys; print(sys.path)") + (libpython-clj2.python/run-simple-string "import sys; print(sys.prefix)") + (libpython-clj2.python/run-simple-string "import sys; print(sys.base_prefix)") (libpython-clj2.python/import-module "openai") - (libpython-clj2.python/import-module "langextract")) \ No newline at end of file + (libpython-clj2.python/import-module "langextract") + ) \ No newline at end of file diff --git a/src/libpython_clj2/python/uv.clj b/src/libpython_clj2/python/uv.clj index 87cd3b3..6cf39c5 100644 --- a/src/libpython_clj2/python/uv.clj +++ b/src/libpython_clj2/python/uv.clj @@ -10,12 +10,12 @@ (let [python-deps (:python-deps deps-edn) - ;python-version (:python-version deps-edn) + python-version (:python-version deps-edn) py-project-header-lines ["[project]" "name = \"temp\"" "version = \"0.0\"" - ;(format "requires-python = \"==%s\"" python-version) + (format "requires-python = \"==%s\"" python-version) ] python-deps-lines (map From 5dfc96c8c97bf2927b2cc3472dbc494ddf3948df Mon Sep 17 00:00:00 2001 From: Carsten Behring Date: Sun, 31 Aug 2025 21:47:44 +0200 Subject: [PATCH 5/9] documented uv support --- .gitignore | 2 ++ topics/environments.md | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/.gitignore b/.gitignore index f38e7c7..a875db5 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ a.out pregen-ffi-test .calva .lock +pyproject.toml +uv.lock diff --git a/topics/environments.md b/topics/environments.md index 4a5693b..9b5415e 100644 --- a/topics/environments.md +++ b/topics/environments.md @@ -14,3 +14,50 @@ Conda requires that we set the LD_LIBRARY_PATH to the conda install. * [example conda repl launcher](https://github.com/clj-python/libpython-clj/blob/master/scripts/conda-repl) * [libpython-clj issue for Conda](https://github.com/clj-python/libpython-clj/issues/18) + + + +## uv + +When you are using the (awesome !) python package manager [uv](https://docs.astral.sh/uv/) we provide a nice integration, which allows to auto-mange +declarative python environments. + +Assuming that you have 'uv' installed (it exists for Linux, Windows , Mac) you can specify and auto-setup a local python venv incl. python version by adding the following to `python.edn` +(Linux example) + +``` +:python-version "3.10.16" +:python-deps ["openai==1.58.1" + "langextract"] +:python-executable ".venv/bin/python" +:pre-initialize-fn libpython-clj2.python.uv/sync-python-setup! +``` + +The versions specification takes the same values as in uv, so would allow ranges a swell, for examples. We suggest to use precise versions, if possible + +Having this, on a call to `(py/initialize!)` a python venv will be created/updated to match the python version and the specified packages. This calls behind the scenes `uv sync` so the spec and the venv are "brought in sync". + +Re-syncing can be as well called manually (while the Clojure repl runs), invoking directly `(libpython-clj2.python.uv/sync-python-setup!)` + +### ux on Windows + +On Windows we need to use: +`:python-executable ".venv/Scripts/python"` + +as the python executable. + +### Caveat + +We have noticed that under Windows for some python versions `libpython-clj` does not setup the python library path correctly, resulting in python libraries not found using for example: `(py/import-module "xxx")` + +This is visible by inspecting python `sys.path`, which should contain `.venv/` via `(py/run-simple-string "import sys; print(sys.path)")`, + +`sys.path` should contain something like +`c:\\Users\\behrica\\Repos\\tryLangExtract\\.venv` + +This can be fixed as by running after `(py/initialize!)` the following: + +Windows: `(py/run-simple-string "import sys; sys.path.append('.venv/Lib/site-packages')")` +Linux: `(py/run-simple-string "import sys; sys.path.append('/.venv/lib//site-packages')")` + +Not sure, if the precise paths can change across python versions. They can be discovered by looking into `.venv` directory and see where precisely the "site-packages" directory is located. \ No newline at end of file From 6b682af8cfe332fca74dcb6264bbe47e874d20a0 Mon Sep 17 00:00:00 2001 From: Carsten Behring Date: Sun, 31 Aug 2025 21:58:21 +0200 Subject: [PATCH 6/9] cleaned up --- python.edn => python.edn.example | 0 src/libpython_clj2/python.clj | 24 +----------------------- src/libpython_clj2/python/uv.clj | 19 +++++++++---------- 3 files changed, 10 insertions(+), 33 deletions(-) rename python.edn => python.edn.example (100%) diff --git a/python.edn b/python.edn.example similarity index 100% rename from python.edn rename to python.edn.example diff --git a/src/libpython_clj2/python.clj b/src/libpython_clj2/python.clj index ee84922..fd798be 100644 --- a/src/libpython_clj2/python.clj +++ b/src/libpython_clj2/python.clj @@ -113,18 +113,8 @@ user> (py/py. np linspace 2 3 :num 10) (let [python-edn-opts (-> (try (slurp "python.edn") (catch java.io.FileNotFoundException _ "{}")) clojure.edn/read-string) - _ (log/infof "Pre-initialize-fn %s" (some-> python-edn-opts :pre-initialize-fn)) - _ (def python-edn-opts python-edn-opts) + _ (log/debugf "Pre-initialize-fn %s" (some-> python-edn-opts :pre-initialize-fn)) _ (some-> python-edn-opts :pre-initialize-fn requiring-resolve (apply [])) - _ (some-> python-edn-opts :pre-initialize-fn) - ;_ ( (requiring-resolve 'libpython-clj2.python.uv/sync-python-setup)) - - ;(require 'libpython-clj2.python.uv) - - ;(requiring-resolve ) - - ;(-> 'libpython-clj2.python.uv/char-seq namespace symbol require) - ;(resolve 'libpython-clj2.python.uv/char-seq) options (merge python-edn-opts options) info (py-info/detect-startup-info options) _ (log/infof "Startup info %s" info) @@ -832,15 +822,3 @@ user> c ~@(for [s symbols] `(def ~s ~s)))) -(comment - (require 'libpython-clj2.python) - (require 'libpython-clj2.python.uv) - - (libpython-clj2.python/initialize!) - (libpython-clj2.python/run-simple-string "import sys; sys.path.append('.venv/Lib/site-packages')") - (libpython-clj2.python/run-simple-string "import sys; print(sys.path)") - (libpython-clj2.python/run-simple-string "import sys; print(sys.prefix)") - (libpython-clj2.python/run-simple-string "import sys; print(sys.base_prefix)") - (libpython-clj2.python/import-module "openai") - (libpython-clj2.python/import-module "langextract") - ) \ No newline at end of file diff --git a/src/libpython_clj2/python/uv.clj b/src/libpython_clj2/python/uv.clj index 6cf39c5..c500507 100644 --- a/src/libpython_clj2/python/uv.clj +++ b/src/libpython_clj2/python/uv.clj @@ -3,9 +3,10 @@ [clojure.edn :as edn] [clojure.java.process :as process] [clojure.string :as str] - [cheshire.core :as json])) + [cheshire.core :as json] + [clojure.java.process :as proc])) -(defn write-pyproject-toml! [deps-edn] +(defn- write-pyproject-toml! [deps-edn] (let [python-deps (:python-deps deps-edn) @@ -33,7 +34,7 @@ (spit "pyproject.toml" (str/join "\n" py-project-lines)))) -(defn char-seq +(defn- char-seq [^java.io.Reader rdr] (let [chr (.read rdr)] (when (>= chr 0) @@ -44,12 +45,8 @@ (cons chr (lazy-seq (char-seq rdr))))))) -(defn start-and-print! [process-args] - (let [args - (concat [{:env {"RENV_CONFIG_INSTALL_VERBOSE" "TRUE"}}] - process-args) - p - (apply process/start args)] +(defn- start-and-print! [process-args] + (let [p(apply process/start process-args)] (with-open [in-rdr (java.io.InputStreamReader. (.getInputStream p)) err-rdr (java.io.InputStreamReader. (.getErrorStream p))] @@ -60,7 +57,9 @@ (dorun (char-seq err-rdr))))) -(defn sync-python-setup! [] +(defn sync-python-setup! + "Synchronize python venv at .venv with 'uv sync'." + [] (println "Synchronize python venv at .venv with 'uv sync'. This might take a few minutes") (let [deps-edn (-> From ab1a1914430f145da9df07fabf025b9389e84d14 Mon Sep 17 00:00:00 2001 From: Carsten Behring Date: Sun, 31 Aug 2025 22:01:51 +0200 Subject: [PATCH 7/9] reverted Clojure version change --- .devcontainer/Dockerfile | 25 ++++++++++++++++++++++ .devcontainer/devcontainer.json | 38 +++++++++++++++++++++++++++++++++ deps.edn | 2 +- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..e5089c0 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,25 @@ +FROM clojure:temurin-21-tools-deps-1.11.3.1456-jammy + +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + # + # [Optional] Add sudo support. Omit if you don't need to install software after connecting. + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +# ******************************************************** +# * Anything else you want to do like clean up goes here * +# ******************************************************** + +# [Optional] Set the default user. Omit if you want to keep the default as root. +USER $USERNAME +SHELL ["/bin/bash", "-ec"] +ENTRYPOINT ["bash"] + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..c5be6aa --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +{ + "build": { + "dockerfile": "Dockerfile" + }, + + "features": { + "ghcr.io/devcontainers/features/desktop-lite:1": {}, + "ghcr.io/devcontainers-contrib/features/apt-get-packages:1": { + "packages": "r-base-dev,rlwrap,expect" + }, + "ghcr.io/devcontainers/features/python:1": {}, + "ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": {}, + "ghcr.io/rocker-org/devcontainer-features/r-apt:0": {}, + "ghcr.io/rocker-org/devcontainer-features/r-packages:1": {}, + "ghcr.io/wxw-matt/devcontainer-features/command_runner:latest": { + "command1": "bash < <(curl -s https://raw.githubusercontent.com/clojure-lsp/clojure-lsp/master/install)", + "command2": "bash < <(curl -s https://raw.githubusercontent.com/babashka/babashka/master/install)", + "command3": "bash -c 'wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein -O /usr/local/bin/lein && chmod +x /usr/local/bin/lein'" + }, + "ghcr.io/wxw-matt/devcontainer-features/command_runner:0": {}, + "ghcr.io/va-h/devcontainers-features/uv:1": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rocker-org/devcontainer-features/r-apt", + "ghcr.io/devcontainers-contrib/features/apt-get-packages", + "ghcr.io/rocker-org/devcontainer-features/r-packages" + ], + + "customizations": { + "vscode": { + "extensions": [ + "betterthantomorrow.calva" + ] + } + } + + +} diff --git a/deps.edn b/deps.edn index 720ca01..1bc41ce 100644 --- a/deps.edn +++ b/deps.edn @@ -1,5 +1,5 @@ {:paths ["src"] - :deps {org.clojure/clojure {:mvn/version "1.12.0" } + :deps {org.clojure/clojure {:mvn/version "1.11.1" :scope "provided"} cnuernber/dtype-next {:mvn/version "10.111"} net.java.dev.jna/jna {:mvn/version "5.12.1"} org.clojure/data.json {:mvn/version "1.0.0"} From 9e86562d43682961aa29da6f3bf813b12989e8d9 Mon Sep 17 00:00:00 2001 From: Carsten Behring Date: Sun, 31 Aug 2025 22:39:03 +0200 Subject: [PATCH 8/9] Update environments.md comment about performance and missing progress logging --- topics/environments.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/topics/environments.md b/topics/environments.md index 9b5415e..9da6e50 100644 --- a/topics/environments.md +++ b/topics/environments.md @@ -46,6 +46,15 @@ On Windows we need to use: as the python executable. +### Performance and progress logging + +uv is very fast, usualy package installations and even python installations run in under one minute. + +It was not possible to print progress logging output from the 'uv sync' run, so please be patient if nothings happens for a while, after changing python version or adding (larger) python packages. + +If you don't want to wait, you can just "kill the clojure process" and run `uv sync` by hand (which prints the logging). +The integration creates an updated "pyproject.toml" file on disk, which `uv sync` will process. + ### Caveat We have noticed that under Windows for some python versions `libpython-clj` does not setup the python library path correctly, resulting in python libraries not found using for example: `(py/import-module "xxx")` @@ -60,4 +69,4 @@ This can be fixed as by running after `(py/initialize!)` the following: Windows: `(py/run-simple-string "import sys; sys.path.append('.venv/Lib/site-packages')")` Linux: `(py/run-simple-string "import sys; sys.path.append('/.venv/lib//site-packages')")` -Not sure, if the precise paths can change across python versions. They can be discovered by looking into `.venv` directory and see where precisely the "site-packages" directory is located. \ No newline at end of file +Not sure, if the precise paths can change across python versions. They can be discovered by looking into `.venv` directory and see where precisely the "site-packages" directory is located. From 6220606582081294b328a3ffef8d93b4d14163f1 Mon Sep 17 00:00:00 2001 From: Carsten Behring Date: Mon, 1 Sep 2025 09:45:33 +0200 Subject: [PATCH 9/9] Update uv.clj use "--managed-python" --- src/libpython_clj2/python/uv.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libpython_clj2/python/uv.clj b/src/libpython_clj2/python/uv.clj index c500507..ce5a0ad 100644 --- a/src/libpython_clj2/python/uv.clj +++ b/src/libpython_clj2/python/uv.clj @@ -66,7 +66,7 @@ (slurp "python.edn") edn/read-string)] (write-pyproject-toml! deps-edn) - (start-and-print! ["uv" "sync" "--python" (-> deps-edn :python-version)]))) + (start-and-print! ["uv" "sync" "--managed-python" "--python" (-> deps-edn :python-version)])))