From 114822934173b2386d729a012fef90714cfa125e Mon Sep 17 00:00:00 2001 From: Keming Date: Fri, 7 Oct 2022 18:03:06 +0800 Subject: [PATCH 1/6] proposal(ir): state based implementation Signed-off-by: Keming --- docs/proposals/20221007-ir-state.md | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/proposals/20221007-ir-state.md diff --git a/docs/proposals/20221007-ir-state.md b/docs/proposals/20221007-ir-state.md new file mode 100644 index 000000000..f2c8b46eb --- /dev/null +++ b/docs/proposals/20221007-ir-state.md @@ -0,0 +1,53 @@ +# Refactor the IR to use state + +Author: +- [kemingy](https://github.com/kemingy/) + + +## Summary + +This proposal is designed to refactor the IR, as we discussed in [envd/#91](https://github.com/tensorchord/envd/issues/91#issuecomment-1239405359). + +## Motivation + +* Currently, we build the image with hard-coded order and parallelism. It's not flexible enough for some use cases. +* The design of buildkit LLB state is suitable for method chaining implementation. Users can define the dependencies and parallelism easily. + +## Goals + +* Able to define the order and parallelism. +* Provide low level APIs to build the image from scratch. + +## API + +```python +def conda_env(root): + conda = root.env("MAMBA_ROOT_PREFIX", "/opt/conda").run( + [ + "curl micro.mamba.pm/install.sh | sh -s -- -y", + "/opt/conda/bin/micromamba create -n envd python={}".format("3.10"), + ] + ) + return conda + + +def build(): + root = ( + Image("ubuntu:22.04") + .apt_packages( + ["curl", "wget"], update=True, clean=True, without_recommends=True + ) + .run(["curl https://starship.rs/install.sh | sh -s -- -y"], mount=None) + .copy( + Image("tensorchord/envd-sshd-from-scratch:latest"), + source_path="/usr/bin/envd-sshd", + dest_path="/var/envd/bin/envd-sshd", + ) + ) + return conda_env(root) +``` + +## Risk + +* Implementation in Starlark +* VSCode language server From 5207862b5eb8369ae1e0436eccd7384b9e63c74c Mon Sep 17 00:00:00 2001 From: Keming Date: Mon, 10 Oct 2022 18:16:52 +0800 Subject: [PATCH 2/6] update example Signed-off-by: Keming --- docs/proposals/20221007-ir-state.md | 38 +++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/proposals/20221007-ir-state.md b/docs/proposals/20221007-ir-state.md index f2c8b46eb..911c70061 100644 --- a/docs/proposals/20221007-ir-state.md +++ b/docs/proposals/20221007-ir-state.md @@ -21,30 +21,58 @@ This proposal is designed to refactor the IR, as we discussed in [envd/#91](http ## API ```python -def conda_env(root): +def conda_env(root, version="3.10"): conda = root.env("MAMBA_ROOT_PREFIX", "/opt/conda").run( [ "curl micro.mamba.pm/install.sh | sh -s -- -y", - "/opt/conda/bin/micromamba create -n envd python={}".format("3.10"), + "/opt/conda/bin/micromamba create -n envd python={}".format(version), ] ) return conda +def install_shell(root, sh="bash"): + if sh == "bash": + return root + + +def install_vscode_ext(ext=()): + root = State.merge(Source.http("openvsx/{}".format(x), filename=x) for x in ext) + return root + + +def install_key(): + root = ( + Source.scratch() + .mkdir("/var/envd", permission=755) + .file("/var/envd/authorized_keys", data="xxx envd", permission=644) + ) + return root + + +def parallel_build(root): + """demo for parallel + diff + merge""" + # use `root.state()` to create a copy explicitly + conda = conda_env(install_shell(root.state())) + vscode = install_vscode_ext() + key = config_key() + return State.merge([root, key, vscode, Diff(root, conda)]) + + def build(): root = ( - Image("ubuntu:22.04") + Source.image("ubuntu:22.04") .apt_packages( ["curl", "wget"], update=True, clean=True, without_recommends=True ) .run(["curl https://starship.rs/install.sh | sh -s -- -y"], mount=None) .copy( - Image("tensorchord/envd-sshd-from-scratch:latest"), + Source.image("tensorchord/envd-sshd-from-scratch:latest"), source_path="/usr/bin/envd-sshd", dest_path="/var/envd/bin/envd-sshd", ) ) - return conda_env(root) + return parallel_build(root) ``` ## Risk From 4e09b6ef37906e3392c4336af5f82b023051fab6 Mon Sep 17 00:00:00 2001 From: Keming Date: Tue, 11 Oct 2022 16:18:32 +0800 Subject: [PATCH 3/6] move merge to Source Signed-off-by: Keming --- docs/proposals/20221007-ir-state.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposals/20221007-ir-state.md b/docs/proposals/20221007-ir-state.md index 911c70061..fa98ac7db 100644 --- a/docs/proposals/20221007-ir-state.md +++ b/docs/proposals/20221007-ir-state.md @@ -37,7 +37,7 @@ def install_shell(root, sh="bash"): def install_vscode_ext(ext=()): - root = State.merge(Source.http("openvsx/{}".format(x), filename=x) for x in ext) + root = Source.merge(Source.http("openvsx/{}".format(x), filename=x) for x in ext) return root @@ -56,7 +56,7 @@ def parallel_build(root): conda = conda_env(install_shell(root.state())) vscode = install_vscode_ext() key = config_key() - return State.merge([root, key, vscode, Diff(root, conda)]) + return Source.merge([root, key, vscode, Diff(root, conda)]) def build(): From d43bc3e8c1696cb5baae5093a77a3262a57dfa46 Mon Sep 17 00:00:00 2001 From: Keming Date: Wed, 12 Oct 2022 13:13:36 +0800 Subject: [PATCH 4/6] rm Source Signed-off-by: Keming --- docs/proposals/20221007-ir-state.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/proposals/20221007-ir-state.md b/docs/proposals/20221007-ir-state.md index fa98ac7db..890bc2dc3 100644 --- a/docs/proposals/20221007-ir-state.md +++ b/docs/proposals/20221007-ir-state.md @@ -37,13 +37,13 @@ def install_shell(root, sh="bash"): def install_vscode_ext(ext=()): - root = Source.merge(Source.http("openvsx/{}".format(x), filename=x) for x in ext) + root = merge(http("openvsx/{}".format(x), filename=x) for x in ext) return root def install_key(): root = ( - Source.scratch() + scratch() .mkdir("/var/envd", permission=755) .file("/var/envd/authorized_keys", data="xxx envd", permission=644) ) @@ -56,18 +56,18 @@ def parallel_build(root): conda = conda_env(install_shell(root.state())) vscode = install_vscode_ext() key = config_key() - return Source.merge([root, key, vscode, Diff(root, conda)]) + return merge([root, key, vscode, Diff(root, conda)]) def build(): root = ( - Source.image("ubuntu:22.04") + image("ubuntu:22.04") .apt_packages( ["curl", "wget"], update=True, clean=True, without_recommends=True ) .run(["curl https://starship.rs/install.sh | sh -s -- -y"], mount=None) .copy( - Source.image("tensorchord/envd-sshd-from-scratch:latest"), + image("tensorchord/envd-sshd-from-scratch:latest"), source_path="/usr/bin/envd-sshd", dest_path="/var/envd/bin/envd-sshd", ) From b17c54a562b4d15c0a114d6211285bbc37ec0986 Mon Sep 17 00:00:00 2001 From: Keming Date: Wed, 12 Oct 2022 13:25:01 +0800 Subject: [PATCH 5/6] add example Signed-off-by: Keming --- docs/proposals/20221007-ir-state.md | 47 +++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/proposals/20221007-ir-state.md b/docs/proposals/20221007-ir-state.md index 890bc2dc3..388da9362 100644 --- a/docs/proposals/20221007-ir-state.md +++ b/docs/proposals/20221007-ir-state.md @@ -20,6 +20,14 @@ This proposal is designed to refactor the IR, as we discussed in [envd/#91](http ## API +*highlight* +* `build()` function need to return a state +* a state can started from + * `scratch()` + * `merge()` + * the returned state from the function call +* envd should provide the default build like `envd_python(version=3.10, dev=True)` + ```python def conda_env(root, version="3.10"): conda = root.env("MAMBA_ROOT_PREFIX", "/opt/conda").run( @@ -68,13 +76,48 @@ def build(): .run(["curl https://starship.rs/install.sh | sh -s -- -y"], mount=None) .copy( image("tensorchord/envd-sshd-from-scratch:latest"), - source_path="/usr/bin/envd-sshd", - dest_path="/var/envd/bin/envd-sshd", + host_path="/usr/bin/envd-sshd", + envd_path="/var/envd/bin/envd-sshd", ) ) return parallel_build(root) ``` +### Changes + +Before: + +```python +def build(): + base(os="ubuntu20.04", language="python") + install.python_packages( + [ + "via", + ] + ) + io.copy(host_path="./build.envd", envd_path="/") + runtime.command( + commands={ + "test": "ls /", + } + ) + runtime.environ(env={"ENVD_MODE": "DEV"}) +``` + +After: + +```python +def build(): + root = ( + envd_python() + .install_pip(["via"]) + .copy(localhost(), host_path="./build.envd", envd_path="/") + .env({"ENVD": "DEV"}) + .runtime_cmd({"test": "ls /"}) + ) + return root +``` + ## Risk * Implementation in Starlark From 09edfbee222a248cb092eb5b8cbadc4aa2636736 Mon Sep 17 00:00:00 2001 From: Keming Date: Wed, 12 Oct 2022 18:57:06 +0800 Subject: [PATCH 6/6] split base and python env Signed-off-by: Keming --- docs/proposals/20221007-ir-state.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/proposals/20221007-ir-state.md b/docs/proposals/20221007-ir-state.md index 388da9362..facbc8f90 100644 --- a/docs/proposals/20221007-ir-state.md +++ b/docs/proposals/20221007-ir-state.md @@ -64,7 +64,8 @@ def parallel_build(root): conda = conda_env(install_shell(root.state())) vscode = install_vscode_ext() key = config_key() - return merge([root, key, vscode, Diff(root, conda)]) + # use `diff` when it's *not* built from scratch + return merge([root, key, vscode, diff(root, conda)]) def build(): @@ -109,8 +110,9 @@ After: ```python def build(): root = ( - envd_python() - .install_pip(["via"]) + image("ubuntu:20.04") + .setup_envd_python(version="3.10") + .install_python_packages(["via"]) .copy(localhost(), host_path="./build.envd", envd_path="/") .env({"ENVD": "DEV"}) .runtime_cmd({"test": "ls /"})