diff --git a/.envrc b/.envrc index a3b3f8770..8ffdbe4ab 100644 --- a/.envrc +++ b/.envrc @@ -1,7 +1,6 @@ -watch_file devenv.nix -watch_file devenv.yaml -watch_file devenv.lock -watch_file result/bin/devenv -watch_dir src -nix-build -eval "$(./result/bin/devenv print-dev-env)" +source_url "https://raw.githubusercontent.com/cachix/devenv/5811f4817ba24da923506d134fff2610b8f95ff2/direnvrc" "sha256-IN2rc7pbaBxxjcdIpYOe9lkpiyjSr2V2AwF6KwlnWYQ=" + +# use our own last built devenv/nix in CLI +PATH_add result/bin + +use devenv \ No newline at end of file diff --git a/.github/workflows/buildtest.yml b/.github/workflows/buildtest.yml index a711fdbfb..64f8573a3 100644 --- a/.github/workflows/buildtest.yml +++ b/.github/workflows/buildtest.yml @@ -16,7 +16,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} @@ -52,7 +52,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} @@ -82,7 +82,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable - uses: cachix/cachix-action@v12 diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index adc9597e9..4f2f61444 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v20 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index eea0ab2ac..bf4ff7602 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ result /include /lib pyvenv.cfg +/.direnv diff --git a/devenv.lock b/devenv.lock index 3f840f05f..f8856149b 100644 --- a/devenv.lock +++ b/devenv.lock @@ -104,11 +104,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1676549890, - "narHash": "sha256-sq/WcOEAl7gWrrfGkWdnyYazRyTf+enEim/o6LOQzI8=", + "lastModified": 1677534593, + "narHash": "sha256-PuZSAHeq4/9pP/uYH1FcagQ3nLm/DrDrvKi/xC9glvw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8c66bd1b68f4708c90dcc97c6f7052a5a7b33257", + "rev": "3ad64d9e2d5bf80c877286102355b1625891ae9a", "type": "github" }, "original": { @@ -161,11 +161,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1676513100, - "narHash": "sha256-MK39nQV86L2ag4TmcK5/+r1ULpzRLPbbfvWbPvIoYJE=", + "lastModified": 1677160285, + "narHash": "sha256-tBzpCjMP+P3Y3nKLYvdBkXBg3KvTMo3gvi8tLQaqXVY=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "5f0cba88ac4d6dd8cad5c6f6f1540b3d6a21a798", + "rev": "2bd861ab81469428d9c823ef72c4bb08372dd2c4", "type": "github" }, "original": { diff --git a/devenv.nix b/devenv.nix index 42db4e888..4dc3b6669 100644 --- a/devenv.nix +++ b/devenv.nix @@ -9,19 +9,20 @@ ]; languages.python.enable = true; + languages.python.venv.enable = true; devcontainer.enable = true; difftastic.enable = true; # bin/mkdocs serve --config-file mkdocs.insiders.yml - processes.docs.exec = "bin/mkdocs serve"; + processes.docs.exec = "mkdocs serve"; processes.build.exec = "${pkgs.watchexec}/bin/watchexec -e nix nix build"; enterShell = '' echo "To Install:" echo - echo "virtualenv ." - echo "bin/pip install -r requirements.txt" + echo " $ pip install -r requirements.txt" + echo ''; scripts.devenv-bump-version.exec = '' diff --git a/devenv.yaml b/devenv.yaml index a392628e6..d759fba2c 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -12,4 +12,4 @@ inputs: url: github:cachix/pre-commit-hooks.nix inputs: nixpkgs: - follows: "nixpkgs" + follows: "nixpkgs" \ No newline at end of file diff --git a/direnvrc b/direnvrc new file mode 100644 index 000000000..9ee72175f --- /dev/null +++ b/direnvrc @@ -0,0 +1,154 @@ +# shellcheck shell=bash +# adapted from https://github.com/nix-community/nix-direnv/blob/master/direnvrc + +REQUIRED_DIRENV_VERSION="2.21.3" + +_nix_direnv_preflight () { + if [[ -z "$direnv" ]]; then + printf '%s\n' "\$direnv environment variable was not defined. Was this script run inside direnv?" + exit 1 + fi + + if [[ -z ${DEVENV_BIN:-} ]]; then + DEVENV_BIN=$(command -v devenv) + if [[ -z "${DEVENV_BIN}" ]]; then + log_status "command not found: devenv, see https://devenv.sh/getting-started/" + exit 1 + fi + fi + + if ! has direnv_version || ! direnv_version "$REQUIRED_DIRENV_VERSION" 2>/dev/null; then + log_status "base direnv version is older than the required v$REQUIRED_DIRENV_VERSION." + exit 1 + fi + + local layout_dir + layout_dir=$(direnv_layout_dir) + + if [[ ! -d "$layout_dir" ]]; then + mkdir -p "$layout_dir" + fi +} + +_nix_export_or_unset() { + local key=$1 value=$2 + if [[ "$value" == __UNSET__ ]]; then + unset "$key" + else + export "$key=$value" + fi +} + +_nix_import_env() { + local profile_rc=$1 + + local old_nix_build_top=${NIX_BUILD_TOP:-__UNSET__} + local old_tmp=${TMP:-__UNSET__} + local old_tmpdir=${TMPDIR:-__UNSET__} + local old_temp=${TEMP:-__UNSET__} + local old_tempdir=${TEMPDIR:-__UNSET__} + local old_xdg_data_dirs=${XDG_DATA_DIRS:-} + eval "$(< "$profile_rc")" + # `nix print-dev-env` will create a temporary directory and use it as TMPDIR + # We cannot rely on this directory being availble at all times, + # as it may be garbage collected. + # Instead - just remove it immediately. + # Use recursive & force as it may not be empty. + if [[ -n "${NIX_BUILD_TOP+x}" && "$NIX_BUILD_TOP" == */nix-shell.* && -d "$NIX_BUILD_TOP" ]]; then + rm -rf "$NIX_BUILD_TOP" + fi + + _nix_export_or_unset NIX_BUILD_TOP "$old_nix_build_top" + _nix_export_or_unset TMP "$old_tmp" + _nix_export_or_unset TMPDIR "$old_tmpdir" + _nix_export_or_unset TEMP "$old_temp" + _nix_export_or_unset TEMPDIR "$old_tempdir" + local new_xdg_data_dirs=${XDG_DATA_DIRS:-} + export XDG_DATA_DIRS= + local IFS=: + for dir in $new_xdg_data_dirs${old_xdg_data_dirs:+:}$old_xdg_data_dirs; do + dir="${dir%/}" # remove trailing slashes + if [[ :$XDG_DATA_DIRS: = *:$dir:* ]]; then + continue # already present, skip + fi + XDG_DATA_DIRS="$XDG_DATA_DIRS${XDG_DATA_DIRS:+:}$dir" + done +} + +_nix_argsum_suffix() { + local out checksum content + + content=$(cat "$@" 2>/dev/null) + + if has sha1sum; then + out=$(sha1sum <<< "$content") + elif has shasum; then + out=$(shasum <<< "$content") + else + log_status "not hashing your cache, please install sha1sum" + # degrate gracefully both tools are not present + return + fi + read -r checksum _ <<< "$out" + echo "-$checksum" +} + +nix_direnv_watch_file() { + watch_file "$@" + nix_watches+=("$@") +} + +use_devenv() { + _nix_direnv_preflight + + flake_expr="${1:-.}" + flake_dir="${flake_expr%#*}" + + local files_to_watch + files_to_watch=(".envrc" "$HOME/.direnvrc" "$HOME/.config/direnv/direnvrc") + + if [[ -d "$flake_dir" ]]; then + files_to_watch+=("$flake_dir/devenv.nix" "$flake_dir/devenv.lock" "$flake_dir/devenv.yaml" "$flake_dir/devenv.local.nix") + if [[ -f "$flake_dir/devenv.yaml" ]]; then + if ! devenv assemble; then + log_status "$(devenv version) failed to parse devenv.yaml, make sure to use version 0.6 or newer and fix the errors above." + exit 1 + fi + + if [[ -f "$flake_dir/.devenv/imports.txt" ]]; then + for file in $(cat "$flake_dir/.devenv/imports.txt"); do + files_to_watch+=("$file") + done + fi + fi + fi + + nix_direnv_watch_file "${files_to_watch[@]}" + + local layout_dir profile_rc + layout_dir=$(direnv_layout_dir) + profile_rc="${layout_dir}/devenv-profile$(_nix_argsum_suffix "${files_to_watch[@]}").rc" + + local need_update=0 + local file= + for file in "${nix_watches[@]}"; do + if [[ "$file" -nt "$profile_rc" ]]; then + need_update=1 + log_status "$file changed, reloading" + break + fi + done + + if [[ ! -e "$profile_rc" || "$need_update" == "1" ]]; + then + # We need to update our cache + local tmp_profile_rc + tmp_profile_rc=$("${DEVENV_BIN}" print-dev-env) + echo "$tmp_profile_rc" > "$profile_rc" + log_status "updated devenv shell cache" + else + log_status "using cached devenv shell" + fi + + _nix_import_env "$profile_rc" +} \ No newline at end of file diff --git a/docs/blog/posts/devenv-v0.6-generating-containers-and-instant-shell-activation.md b/docs/blog/posts/devenv-v0.6-generating-containers-and-instant-shell-activation.md new file mode 100644 index 000000000..2db17f166 --- /dev/null +++ b/docs/blog/posts/devenv-v0.6-generating-containers-and-instant-shell-activation.md @@ -0,0 +1,172 @@ +--- +draft: false +date: 2023-03-02 +authors: + - domenkozar +--- + +# devenv 0.6: Generating containers and instant shell activation + +After about two months of active development, I'm happy to announce [devenv 0.6](/getting-started/) is ready. + +This release comes with the most notable improvements based on the feedback from existing users: + +- Adding the ability to [generate containers](#generating-containers). +- [Instant shell activation](#instant-shell-activation) of the developer environment. +- [Hosts and ceritifcates](#hosts-and-certificates) provisioning. +- [New devenv.yaml options](#allowunfree-and-overlays): `allowUnfree` and `overlays`. + +## Generating containers + +While `devenv shell` provides a [simple native developer environment](/basics/) experience, +`devenv container ` allows you to generate and copy [OCI container](https://opencontainers.org/) into a registry. + +Containers are a great way to distribute ready-made applications, leveraging platforms like [fly.io](https://github.com/cachix/devenv/tree/main/examples/fly.io) to deploy them into production. + +An example for Ruby: + +```nix title="devenv.nix" +{ + name = "simple-ruby-app"; + + languages.ruby.enable = true; + languages.ruby.version = "3.2.1"; +} +``` + +We can generate a container called `shell` that enters the environment, copy it to the local Docker daemon and run it: + + +``` +$ devenv container shell --docker-run +... +(devenv) bash-5.2# ruby --version +ruby 3.2.1 (2023-02-08 revision 31819e82c8) [x86_64-linux] +``` + +You can read more in the new [Containers](/containers/) section of the documentation, specifically: + +- [How to generate a container for shell](/containers/#entering-the-development-environment) +- [How to generate a container to start all processes](/containers/#running-processes) +- [How to generate a container to start a single process](/containers/#running-a-single-process) +- [How to generate a container to start a custom built binary](/containers/#running-artifacts) +- [How to copy the containers to a registry](/containers/#copying-container-to-a-registry) +- [How to conditionalize environment based on native/container target](/containers/#changing-environment-based-on-the-build-type) + +## Instant shell activation + +Especially **monorepo** developer environments can sometimes be even **a few gigabytes** of size, taking **a few seconds** for the environment to be activated. + +A developer **environment should only be built +when something changes** and if not, the environment +can be used **instantly using a cached snapshot**. + +With the latest [direnv.net integration](/automatic-shell-activation/), +we've **finally reached that goal** by making caching work properly (it will even watch each of your imports for changes!). + +!!! note "Migrating from an older devenv" + Make sure to use the latest `.envrc` from `devenv init` and for everyone on the team to [upgrade to devenv 0.6](/getting-started/). + +In the near future we'll experiment to improve [devenv shell](https://github.com/cachix/devenv/issues/240) experience. + +## Hosts and certificates + +Hosts and certificates can now be specified declaratively: + +```nix +{ pkgs, config, ... }: + +{ + certificates = [ + "example.com" + ]; + + hosts."example.com" = "127.0.0.1"; + + services.caddy.enable = true; + services.caddy.virtualHosts."example.com" = { + extraConfig = '' + tls ${config.env.DEVENV_STATE}/mkcert/example.com.pem ${config.env.DEVENV_STATE}/mkcert/example.com-key.pem + respond "Hello, world!" + ''; + }; +} +``` + +And when you run `devenv up` to start [the processes](/processes/), these hosts and certificates will be provisioned locally. + +## `allowUnfree` and `overlays` + +For example in `devenv.yaml`: + +```yaml +allowUnfree: true +inputs: + nixpkgs: + url: github:NixOS/nixpkgs/nixpkgs-unstable + rust-overlay: + url: github:oxalica/rust-overlay + overlays: + - default +``` + +Will allow building unfree software and wire up `default` overlay into `pkgs` from [rust-overlay](https://github.com/oxalica/rust-overlay). + + +!!! note "Migrating from an older devenv" + Make sure *everyone* on the team upgrades [to devenv 0.6](/getting-started/). + + +## Languages changelog + +- **Python:** Added support for virtualenv creation and poetry by [bobvanderlinden](https://github.com/bobvanderlinden/). +- **Ruby:** First-class support for setting `version` or `versionFile` by [bobvanderlinden](https://github.com/bobvanderlinden/). +- **Go:** Received significant improvements by [shyim](https://github.com/shyim). +- **PHP:** Added first-class support for setting version to make it easier to set extensions by [shyim](https://github.com/shyim). +- **Scala:** Now allows changing the package and offers scala-cli as an option if the JDK is too old by [domenkozar](https://github.com/domenkozar). +- **R:** Added an option to specify the package by [adfaure](https://github.com/adfaure). +- **Rust:** Can now find headers for darwin frameworks by [domenkozar](https://github.com/domenkozar). +- **OCaml:** Allowed using a different version of OCaml by [ankhers](https://github.com/ankhers). +- **Tex Live:** Added support by [BurNiinTRee](https://github.com/BurNiinTRee). +- **Swift:** Added support by [domenkozar](https://github.com/domenkozar). +- **Raku:** Added support by [0pointerexception](https://github.com/0pointerexception). +- **Gawk:** Added support by [0pointerexception](https://github.com/0pointerexception). +- **Racket:** Added support by [totoroot](https://github.com/totoroot). +- **Dart:** Added support by [domenkozar](https://github.com/domenkozar). +- **Julia:** Added support by [domenkozar](https://github.com/domenkozar). +- **Crystal:** Added support by [bcardiff](https://github.com/bcardiff). +- **Unison:** Added support by [ereslibre](https://github.com/ereslibre). +- **Zig:** Added support by [ereslibre](https://github.com/ereslibre). +- **Deno:** Added support by [janathandion](https://github.com/janathandion). + +## Services changelog + +- **Cassandra:** Added by [ankers](https://github.com/ankers). + +- **CouchDB**: Added by [MSBarbieri](https://github.com/MSBarbieri). + +- **MariaDB:** Corrected user and database handling by [jochenmanz](https://github.com/jochenmanz). + +- **MinIO:** Now allows specifying what buckets to provision by [shyim](https://github.com/shyim). + +## Fixed issues and other improvements + +- process-compose: Faster shutdown, restart on failure by default, escape env variables properly by [thenonameguy](https://github.com/thenonameguy). + +- Support assertions in modules by [bobvanderlinden](https://github.com/bobvanderlinden). + +- Fix overmind root by [domenkozar](https://github.com/domenkozar). + +- Make `devenv info` output pluggable from devenv modules by [domenkozar](https://github.com/domenkozar). + +- Expand the flake guide by [sandydoo](https://github.com/sandydoo). + +- Set `LOCALE_ARCHIVE` when missing by [sandydoo](https://github.com/sandydoo). + +- Numerous option documentation fixes by [sandydoo](https://github.com/sandydoo). + +- Fix starship integration with a custom config by [domenkozar](https://github.com/domenkozar). + +- Test direnv integration with strict bash mode by [stephank](https://github.com/stephank). + +- Add a shim `devenv` for flakes integration by [rgonzalez](https://github.com/rgonzalez). \ No newline at end of file diff --git a/docs/containers.md b/docs/containers.md new file mode 100644 index 000000000..bf0b4908b --- /dev/null +++ b/docs/containers.md @@ -0,0 +1,165 @@ +!!! info "New in version 0.6." + +!!! warning + + Creating containers is [only possible on Linux](https://github.com/cachix/devenv/issues/430) at the moment. + + +`devenv container ` can generate an [OCI container](https://opencontainers.org/) from your development environment. + +By default `shell` and `processes` containers are predefined and you can [craft your own](#running-artifacts)! + +Examples of what `devenv container` can do: + +- `devenv container shell`: Generate a container [starting the environment](#entering-the-development-environment), equivalent of using `devenv shell`. +- `devenv container processes`: Generate a container [starting processes](#running-processes), equivalent of using `devenv up`. +- `devenv container --registry docker://ghcr.io/ --copy`: [Copy the container](#copying-container-to-a-registry) `` into the **GitHub package registry**. +- `devenv container --docker-run`: Run the container `` using **Docker**. + +## Entering the development environment + +Given a simple environment using Python: + +```nix title="devenv.nix" +{ + name = "simple-python-app"; + + languages.python.enable = true; +} +``` + +Generate a container specification that enters the environment: + +```shell-session +$ devenv container shell +/nix/store/...-image-devenv.json +``` + +Let's test it locally using Docker: + +```shell-session +$ devenv container shell --docker-run +... +(devenv) bash-5.2# python +Python 3.10.9 (main, Dec 6 2022, 18:44:57) [GCC 12.2.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +## Running processes + +It's common deployment strategy to run all [processes](./processes.md) as the entrypoint to the container. + +```nix title="devenv.nix" +{ + name = "myapp"; + + packages = [ pkgs.procps ]; + + processes.hello-docker.exec = "while true; do echo 'Hello Docker!' && sleep 1; done"; + processes.hello-nix.exec = "while true; do echo 'Hello Nix!' && sleep 1; done"; + + # Exclude the source repo to make the container smaller. + containers."processes".copyToRoot = null; +} +``` + +You can now copy the newly created image and start the container: + +```shell-session +$ devenv container processes --docker-run +... +06:30:06 system | hello-docker.1 started (pid=15) +06:30:06 hello-docker.1 | Hello Docker! +06:30:06 system | hello-nix.1 started (pid=16) +06:30:06 hello-nix.1 | Hello Nix! +06:30:07 hello-nix.1 | Hello Nix! +06:30:07 hello-docker.1 | Hello Docker! +06:30:08 hello-nix.1 | Hello Nix! +06:30:08 hello-docker.1 | Hello Docker! +``` + +## Running a single process + + +You can specify the command to run when the container starts +(instead of entering the default development environment): + +```nix title="devenv.nix" +{ + processes.serve.exec = "python -m http.server"; + + containers."serve".name = "myapp"; + containers."serve".startupCommand = config.processes.serve.exec; +} +``` + +``` +$ devenv container serve --docker-run +``` + +## Running artifacts + +If you're building binaries as part of the development environment, +you can choose to only include those in the final image: + +```nix title="devenv.nix" +{ + # watch local changes and build the project to ./dist + processes.build.exec = "${pkgs.watchexec}/bin/watchexec my-build-tool"; + + containers."prod".copyToRoot = ./dist; + containers."prod".startupCommand = "/mybinary serve"; +} +``` + +```shell-session +$ devenv container prod --docker-run +... +``` + + + +## Copying container to a registry + +To copy a container into a registry, pass it a `--copy` operation: + +```shell-session +$ devenv container processes --copy --registry docker:// +``` + +Another common example is deploying to [fly.io](https://fly.io). +Any arguments passed to `--copy-args` are forwarded to [skopeo copy](https://github.com/containers/skopeo/blob/main/docs/skopeo-copy.1.md#options): + + +```shell-session +$ devenv container processes --copy --registry docker://registry.fly.io/ --copy-args="--dest-creds x:$(flyctl auth token)" +``` + +You can also specify these options declaratively: + +```nix title="devenv.nix" +{ + containers."processes".registry = "docker://registry.fly.io/"; + containers."processes".defaultCopyArgs = [ + "--dest-creds" + "x:\"$(${pkgs.flyctl}/bin/flyctl auth token)\"" + ]; +} +``` + +## Changing environment based on the build type + +If we wanted to provide `openssl` package to native and container environments, +but `git` only for native environments: + +```nix title="devenv.nix" +{ pkgs, config, lib, ... }: + +{ + packages = [ pkgs.openssl ] + ++ lib.optionals (!config.container.isBuilding) [ pkgs.git ]; +} +``` + +You can also conditionalize based on the particular container that is being built, e.g. `config.containers."processes".isBuilding`. \ No newline at end of file diff --git a/docs/integrations/github-actions.md b/docs/integrations/github-actions.md index f71a1961d..bf660034a 100644 --- a/docs/integrations/github-actions.md +++ b/docs/integrations/github-actions.md @@ -20,7 +20,7 @@ jobs: with: name: devenv - name: Install devenv.sh - run: nix profile install github:cachix/devenv/v0.5 + run: nix profile install github:cachix/devenv/v0.5.1 shell: sh - run: devenv ci - run: devenv shell echo ok diff --git a/docs/overrides/home.html b/docs/overrides/home.html index a41e0462a..255eb5d1d 100644 --- a/docs/overrides/home.html +++ b/docs/overrides/home.html @@ -311,7 +311,8 @@

Fast, Declarative, Reproducible, and Composable Developer Environments

-

Build, share, and run your local development environments with a single command. Without containers.

+

Build, share, and run your local development environments with a single command.

+

Develop natively, deploy with containers.

Fast, Declarative, Reproducible, and Composable Developer Environments
- - -
+
@@ -538,6 +529,31 @@

Scripts and pre-commit hooks

+ + + +