diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..e5089c0 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,25 @@ +FROM clojure:temurin-21-tools-deps-1.11.3.1456-jammy + +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + # + # [Optional] Add sudo support. Omit if you don't need to install software after connecting. + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +# ******************************************************** +# * Anything else you want to do like clean up goes here * +# ******************************************************** + +# [Optional] Set the default user. Omit if you want to keep the default as root. +USER $USERNAME +SHELL ["/bin/bash", "-ec"] +ENTRYPOINT ["bash"] + diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..c5be6aa --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +{ + "build": { + "dockerfile": "Dockerfile" + }, + + "features": { + "ghcr.io/devcontainers/features/desktop-lite:1": {}, + "ghcr.io/devcontainers-contrib/features/apt-get-packages:1": { + "packages": "r-base-dev,rlwrap,expect" + }, + "ghcr.io/devcontainers/features/python:1": {}, + "ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": {}, + "ghcr.io/rocker-org/devcontainer-features/r-apt:0": {}, + "ghcr.io/rocker-org/devcontainer-features/r-packages:1": {}, + "ghcr.io/wxw-matt/devcontainer-features/command_runner:latest": { + "command1": "bash < <(curl -s https://raw.githubusercontent.com/clojure-lsp/clojure-lsp/master/install)", + "command2": "bash < <(curl -s https://raw.githubusercontent.com/babashka/babashka/master/install)", + "command3": "bash -c 'wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein -O /usr/local/bin/lein && chmod +x /usr/local/bin/lein'" + }, + "ghcr.io/wxw-matt/devcontainer-features/command_runner:0": {}, + "ghcr.io/va-h/devcontainers-features/uv:1": {} + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/rocker-org/devcontainer-features/r-apt", + "ghcr.io/devcontainers-contrib/features/apt-get-packages", + "ghcr.io/rocker-org/devcontainer-features/r-packages" + ], + + "customizations": { + "vscode": { + "extensions": [ + "betterthantomorrow.calva" + ] + } + } + + +} diff --git a/.gitignore b/.gitignore index 6bd4f64..a875db5 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,8 @@ a.out *.iml .lsp .clj-kondo -pregen-ffi-test \ No newline at end of file +pregen-ffi-test +.calva +.lock +pyproject.toml +uv.lock diff --git a/deps.edn b/deps.edn index 5aefc5f..9796990 100644 --- a/deps.edn +++ b/deps.edn @@ -3,11 +3,15 @@ cnuernber/dtype-next {:mvn/version "10.124"} net.java.dev.jna/jna {:mvn/version "5.12.1"} org.clojure/data.json {:mvn/version "1.0.0"} + cheshire/cheshire {:mvn/version "5.13.0"} + scicloj/clojisr {:mvn/version "1.0.0"} ;;Replace me with caffeine... com.google.guava/guava {:mvn/version "31.1-jre"}} + + :aliases {:dev - {:extra-deps {criterium/criterium {:mvn/version"0.4.5"} + {:extra-deps {criterium/criterium {:mvn/version "0.4.5"} ch.qos.logback/logback-classic {:mvn/version "1.1.3"}}} :fastcall {:jvm-opts ["-Dlibpython_clj.manual_gil=true"]} @@ -71,4 +75,6 @@ :exec-args {:installer :local :artifact "target/libpython-clj.jar"}} - }} + } + + } diff --git a/python.edn.example b/python.edn.example new file mode 100644 index 0000000..926344d --- /dev/null +++ b/python.edn.example @@ -0,0 +1,8 @@ +{ + ;:python-version "3.10.16" + ;:python-deps ["openai==1.58.1" + ; "langextract"] + :python-executable ".venv/Scripts/python" + ;:pre-initialize-fn libpython-clj2.python.uv/sync-python-setup! + +} \ No newline at end of file diff --git a/src/libpython_clj2/python.clj b/src/libpython_clj2/python.clj index 1ec9027..21e6666 100644 --- a/src/libpython_clj2/python.clj +++ b/src/libpython_clj2/python.clj @@ -113,6 +113,7 @@ user> (py/py. np linspace 2 3 :num 10) (let [python-edn-opts (-> (try (slurp "python.edn") (catch java.io.FileNotFoundException _ "{}")) clojure.edn/read-string) + _ (log/debugf "Pre-initialize-fn %s" (some-> python-edn-opts :pre-initialize-fn)) _ (some-> python-edn-opts :pre-initialize-fn requiring-resolve (apply [])) options (merge python-edn-opts options) info (py-info/detect-startup-info options) @@ -819,3 +820,5 @@ user> c [symbols input] `(let [~symbols ~input] ~@(for [s symbols] `(def ~s ~s)))) + + diff --git a/src/libpython_clj2/python/uv.clj b/src/libpython_clj2/python/uv.clj new file mode 100644 index 0000000..ce5a0ad --- /dev/null +++ b/src/libpython_clj2/python/uv.clj @@ -0,0 +1,72 @@ +(ns libpython-clj2.python.uv + (:require + [clojure.edn :as edn] + [clojure.java.process :as process] + [clojure.string :as str] + [cheshire.core :as json] + [clojure.java.process :as proc])) + +(defn- write-pyproject-toml! [deps-edn] + + (let [python-deps + (:python-deps deps-edn) + + python-version (:python-version deps-edn) + + py-project-header-lines ["[project]" + "name = \"temp\"" + "version = \"0.0\"" + (format "requires-python = \"==%s\"" python-version) + ] + python-deps-lines + (map + (fn [dep] + (format "\"%s\"," dep)) + python-deps) + + py-project-lines + (concat + py-project-header-lines + ["dependencies = ["] + python-deps-lines + "]\n")] + + (spit "pyproject.toml" + (str/join "\n" py-project-lines)))) + +(defn- char-seq + [^java.io.Reader rdr] + (let [chr (.read rdr)] + (when (>= chr 0) + (do + ;(def chr chr) + (print (char (Integer. chr))) + (flush) + (cons chr (lazy-seq (char-seq rdr))))))) + + +(defn- start-and-print! [process-args] + (let [p(apply process/start process-args)] + + (with-open [in-rdr (java.io.InputStreamReader. (.getInputStream p)) + err-rdr (java.io.InputStreamReader. (.getErrorStream p))] + + + + (dorun (char-seq in-rdr)) + (dorun (char-seq err-rdr))))) + + +(defn sync-python-setup! + "Synchronize python venv at .venv with 'uv sync'." + [] + (println "Synchronize python venv at .venv with 'uv sync'. This might take a few minutes") + (let [deps-edn + (-> + (slurp "python.edn") + edn/read-string)] + (write-pyproject-toml! deps-edn) + (start-and-print! ["uv" "sync" "--managed-python" "--python" (-> deps-edn :python-version)]))) + + + diff --git a/topics/environments.md b/topics/environments.md index 4a5693b..9da6e50 100644 --- a/topics/environments.md +++ b/topics/environments.md @@ -14,3 +14,59 @@ Conda requires that we set the LD_LIBRARY_PATH to the conda install. * [example conda repl launcher](https://github.com/clj-python/libpython-clj/blob/master/scripts/conda-repl) * [libpython-clj issue for Conda](https://github.com/clj-python/libpython-clj/issues/18) + + + +## uv + +When you are using the (awesome !) python package manager [uv](https://docs.astral.sh/uv/) we provide a nice integration, which allows to auto-mange +declarative python environments. + +Assuming that you have 'uv' installed (it exists for Linux, Windows , Mac) you can specify and auto-setup a local python venv incl. python version by adding the following to `python.edn` +(Linux example) + +``` +:python-version "3.10.16" +:python-deps ["openai==1.58.1" + "langextract"] +:python-executable ".venv/bin/python" +:pre-initialize-fn libpython-clj2.python.uv/sync-python-setup! +``` + +The versions specification takes the same values as in uv, so would allow ranges a swell, for examples. We suggest to use precise versions, if possible + +Having this, on a call to `(py/initialize!)` a python venv will be created/updated to match the python version and the specified packages. This calls behind the scenes `uv sync` so the spec and the venv are "brought in sync". + +Re-syncing can be as well called manually (while the Clojure repl runs), invoking directly `(libpython-clj2.python.uv/sync-python-setup!)` + +### ux on Windows + +On Windows we need to use: +`:python-executable ".venv/Scripts/python"` + +as the python executable. + +### Performance and progress logging + +uv is very fast, usualy package installations and even python installations run in under one minute. + +It was not possible to print progress logging output from the 'uv sync' run, so please be patient if nothings happens for a while, after changing python version or adding (larger) python packages. + +If you don't want to wait, you can just "kill the clojure process" and run `uv sync` by hand (which prints the logging). +The integration creates an updated "pyproject.toml" file on disk, which `uv sync` will process. + +### Caveat + +We have noticed that under Windows for some python versions `libpython-clj` does not setup the python library path correctly, resulting in python libraries not found using for example: `(py/import-module "xxx")` + +This is visible by inspecting python `sys.path`, which should contain `.venv/` via `(py/run-simple-string "import sys; print(sys.path)")`, + +`sys.path` should contain something like +`c:\\Users\\behrica\\Repos\\tryLangExtract\\.venv` + +This can be fixed as by running after `(py/initialize!)` the following: + +Windows: `(py/run-simple-string "import sys; sys.path.append('.venv/Lib/site-packages')")` +Linux: `(py/run-simple-string "import sys; sys.path.append('/.venv/lib//site-packages')")` + +Not sure, if the precise paths can change across python versions. They can be discovered by looking into `.venv` directory and see where precisely the "site-packages" directory is located.