From 2e54be55f6172b21acff9587d97fb7705d0d8fa0 Mon Sep 17 00:00:00 2001 From: Casey Link Date: Thu, 31 Oct 2024 18:08:11 +0100 Subject: [PATCH] Add option to build a custom, slimmer, JRE for datomic --- CHANGELOG.md | 4 +- flake.lock | 112 ++++++++++++++++++++++++++- flake.nix | 11 ++- nixos-modules/datomic-console.nix | 2 - pkgs/datomic-generate-properties.nix | 72 +++++++++-------- pkgs/datomic-pro-container-image.nix | 30 ++++--- pkgs/datomic-pro-peer.nix | 4 - pkgs/datomic-pro.nix | 68 +++++++++++----- pkgs/generate-properties.clj | 2 +- 9 files changed, 227 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44d98c3..2d8df7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,11 +23,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - nix pkg: Exposed more packages: `datomic-shell`, `datomic-run`, `datomic-repl`, `datomic-peer-server` - nixos module: You can now configure: logging, extra classpath entries, and extra java options. - nix pkg: Added datomic-pro-peer package which is the datomic peer library with all of its dependencies +- nix pkg: Added option to build slimmed down JRE for datomic-pro ### Changed - nix pkg: Updated datomic-pro to version 1.0.7260 -- nix pkg: Switched to Nix's JDK 21 headless package (now that it is sufficiently headless +- nix pkg: Switched to Nix's JDK 21 headless package (which is supported by Datomic) +- oci image: Use the slimmed down JRE and a custom babashka build to reduce size of the image ### Fixed diff --git a/flake.lock b/flake.lock index 787f20a..c2f645e 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,64 @@ { "nodes": { + "clj-nix": { + "inputs": { + "devshell": "devshell", + "nix-fetcher-data": "nix-fetcher-data", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1729713058, + "narHash": "sha256-n64jNi0oVJXr+VJElTl3MvStbZmgbBLszXan3fvQj1Y=", + "owner": "jlesquembre", + "repo": "clj-nix", + "rev": "3064cbd2b949967a826414c91a53cabec307f4c8", + "type": "github" + }, + "original": { + "owner": "jlesquembre", + "repo": "clj-nix", + "type": "github" + } + }, + "devshell": { + "inputs": { + "nixpkgs": [ + "clj-nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1728330715, + "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=", + "owner": "numtide", + "repo": "devshell", + "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1719745305, + "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -18,7 +77,57 @@ "type": "github" } }, + "nix-fetcher-data": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": [ + "clj-nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1728229178, + "narHash": "sha256-p5Fx880uBYstIsbaDYN7sECJT11oHxZQKtHgMAVblWA=", + "owner": "jlesquembre", + "repo": "nix-fetcher-data", + "rev": "f3a73c34d28db49ef90fd7872a142bfe93120e55", + "type": "github" + }, + "original": { + "owner": "jlesquembre", + "repo": "nix-fetcher-data", + "type": "github" + } + }, "nixpkgs": { + "locked": { + "lastModified": 1728492678, + "narHash": "sha256-9UTxR8eukdg+XZeHgxW5hQA9fIKHsKCdOIUycTryeVw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5633bcff0c6162b9e4b5f1264264611e950c8ec7", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1717284937, + "narHash": "sha256-lIbdfCsf8LMFloheeE6N31+BMIeixqyQWbSr2vk79EQ=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz" + } + }, + "nixpkgs_2": { "locked": { "lastModified": 1730272153, "narHash": "sha256-B5WRZYsRlJgwVHIV6DvidFN7VX7Fg9uuwkRW9Ha8z+w=", @@ -36,8 +145,9 @@ }, "root": { "inputs": { + "clj-nix": "clj-nix", "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs_2" } }, "systems": { diff --git a/flake.nix b/flake.nix index 4607f91..925adaf 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; - #clj-nix.url = "github:jlesquembre/clj-nix"; + clj-nix.url = "github:jlesquembre/clj-nix"; }; outputs = @@ -10,16 +10,18 @@ self, nixpkgs, flake-utils, + clj-nix, }: flake-utils.lib.eachDefaultSystem ( system: let pkgs = import nixpkgs { inherit system; - overlays = [ self.overlays."${system}" ]; + overlays = [ + clj-nix.overlays.default + self.overlays."${system}" + ]; }; - - jdk-minimal = pkgs.jdk21_headless; in { overlays = final: prev: { @@ -70,6 +72,7 @@ buildInputs = [ pkgs.skopeo pkgs.babashka + pkgs.dive pkgs.gnumake ]; }; diff --git a/nixos-modules/datomic-console.nix b/nixos-modules/datomic-console.nix index 0f7daf9..7b8140c 100644 --- a/nixos-modules/datomic-console.nix +++ b/nixos-modules/datomic-console.nix @@ -18,7 +18,6 @@ in services.datomic-console = { enable = lib.mkEnableOption "Datomic Console"; package = lib.mkPackageOption pkgs "datomic-pro" { }; - javaPackage = lib.mkPackageOption pkgs "jdk21_headless" { }; port = lib.mkOption { type = lib.types.port; description = "The port the console will bind to"; @@ -151,7 +150,6 @@ in db_uri="$(<"$CREDENTIALS_DIRECTORY/datomic-console-db-uri")" ${cfg.package}/bin/datomic-console -p ${toString cfg.port} "${cfg.alias}" "$db_uri" ''; - path = [ cfg.javaPackage ]; environment = { DATOMIC_JAVA_OPTS = toString extraJavaOptions; diff --git a/pkgs/datomic-generate-properties.nix b/pkgs/datomic-generate-properties.nix index e1a6475..e54c790 100644 --- a/pkgs/datomic-generate-properties.nix +++ b/pkgs/datomic-generate-properties.nix @@ -1,39 +1,49 @@ { + hostname, lib, + mkBabashka, stdenv, - fetchzip, - babashka, - dockerTools, - datomic-pro, - hostname, + substituteAll, ... }: -stdenv.mkDerivation (finalAttrs: { - pname = "datomic-generate-properties"; - version = "0.1"; +stdenv.mkDerivation ( + finalAttrs: + let + bb = mkBabashka { + bbLean = true; + wrap = false; + withFeatures = [ "transit" ]; + }; + in + { + pname = "datomic-generate-properties"; + version = "0.1"; - src = ./generate-properties.clj; + src = substituteAll { + babashkaBin = "${bb}/bin/bb"; + src = ./generate-properties.clj; + }; - dontUnpack = true; - propagatedBuildInputs = [ - babashka - hostname - ]; - installPhase = '' - runHook preInstall - ls -al $src - mkdir -p $out/bin - cp -r $src $out/bin/datomic-generate-properties - chmod +x $out/bin/datomic-generate-properties - find $out - runHook postInstall - ''; - meta = { - description = "Generates a java properties file for Datomic Pro based off of environment variables and sane defaults."; - homepage = ""; - changelog = ""; - license = lib.licenses.asl20; - mainProgram = "datomic-generate-properties"; - }; -}) + dontUnpack = true; + propagatedBuildInputs = [ + hostname + ]; + installPhase = '' + runHook preInstall + ls -al $src + mkdir -p $out/bin + cp -r $src $out/bin/datomic-generate-properties + chmod +x $out/bin/datomic-generate-properties + find $out + runHook postInstall + ''; + meta = { + description = "Generates a java properties file for Datomic Pro based off of environment variables and sane defaults."; + homepage = ""; + changelog = ""; + license = lib.licenses.asl20; + mainProgram = "datomic-generate-properties"; + }; + } +) diff --git a/pkgs/datomic-pro-container-image.nix b/pkgs/datomic-pro-container-image.nix index d7eda5d..a6a23d2 100644 --- a/pkgs/datomic-pro-container-image.nix +++ b/pkgs/datomic-pro-container-image.nix @@ -1,14 +1,11 @@ { - babashka, cacert, coreutils, datomic-generate-properties, datomic-pro, dockerTools, - fetchzip, hostname, imageTag, - jre_minimal, lib, mysql_jdbc, runCommand, @@ -20,17 +17,21 @@ }: let - jre = jre_minimal; datomicBuild = datomic-pro.override { + withCustomJre = true; extraJavaPkgs = [ sqlite-jdbc mysql_jdbc ]; }; + entrypointSmall = writeShellScriptBin "datomic-entrypoint" '' + #! ${runtimeShell} + ${datomicBuild}/bin/datomic-transactor "$DATOMIC_TRANSACTOR_PROPERTIES_PATH" + ''; entrypoint = writeShellScriptBin "datomic-entrypoint" '' #! ${runtimeShell} set -e - export PATH=${hostname}/bin:${jre}/bin:${coreutils}/bin:${babashka}:$PATH + export PATH=${hostname}/bin:${coreutils}/bin:$PATH if [ "$(id -u)" = "0" ]; then echo "WARNING: Running Datomic as root is not recommended. Please run as a non-root user." echo " This can be ignored if you are using rootless mode." @@ -71,16 +72,13 @@ dockerTools.buildLayeredImage { contents = [ dockerTools.usrBinEnv dockerTools.binSh - babashka - cacert - coreutils - datomic-generate-properties - datomicBuild - entrypoint - env-shim - jre - mysql_jdbc - sqlite-jdbc + #cacert + #datomic-generate-properties + #datomicBuild + #entrypoint + #env-shim + #mysql_jdbc + #sqlite-jdbc ]; extraCommands = '' mkdir -p tmp @@ -88,7 +86,7 @@ dockerTools.buildLayeredImage { ''; config = { - Entrypoint = [ "${entrypoint}/bin/datomic-entrypoint" ]; + Entrypoint = [ "${entrypointSmall}/bin/datomic-entrypoint" ]; Env = [ "DATOMIC_TRANSACTOR_PROPERTIES_PATH=/config/transactor.properties" "LC_ALL=C.UTF-8" diff --git a/pkgs/datomic-pro-peer.nix b/pkgs/datomic-pro-peer.nix index cc41009..fb0fd57 100644 --- a/pkgs/datomic-pro-peer.nix +++ b/pkgs/datomic-pro-peer.nix @@ -3,8 +3,6 @@ maven, fetchzip, stdenv, - jre, - jdk, ... }: @@ -29,8 +27,6 @@ stdenv.mkDerivation ( maven ]; - JAVA_HOME = jdk; - buildPhase = '' runHook preBuild mvn de.qaware.maven:go-offline-maven-plugin:1.2.8:resolve-dependencies -Dmaven.repo.local=$out/.m2 diff --git a/pkgs/datomic-pro.nix b/pkgs/datomic-pro.nix index 5764740..36718b1 100644 --- a/pkgs/datomic-pro.nix +++ b/pkgs/datomic-pro.nix @@ -1,23 +1,24 @@ { - makeWrapper, - lib, - stdenv, + babashka, fetchzip, jdk21_headless, - sqlite, - sqlite-jdbc, - extraLibs ? [ ], - extraJavaPkgs ? [ ], + jre, + lib, + makeWrapper, + stdenv, + extraLibs ? [ ], # native libs will be added to datomic's library path + extraJavaPkgs ? [ ], # jars contained in share/java will be added to datomic's class path + withCustomJre ? false, # set to true to built a custom jre with only the required modules + overrideJre ? null, # provide your own jre/jdk package that datomic-pro will use ... }: -let - jdk = jdk21_headless; -in - stdenv.mkDerivation ( finalAttrs: let + jdk = jdk21_headless; + actualJre = + if withCustomJre then "$out/jre" else (if overrideJre != null then "${overrideJre}" else "${jre}"); classpath = "$out/share/datomic-pro/resources:$out/lib/datomic-transactor-pro-${finalAttrs.version}.jar:$out/share/datomic-pro/lib/*:$out/share/datomic-pro/samples/clj:$out/share/datomic-pro/bin"; libpath = lib.makeLibraryPath extraLibs; in @@ -29,15 +30,46 @@ stdenv.mkDerivation ( url = "https://datomic-pro-downloads.s3.amazonaws.com/${finalAttrs.version}/datomic-pro-${finalAttrs.version}.zip"; hash = "sha256-J3uGNOcA2JsHGecQbnS2w57XCfiF3H0FNcBJ+vB/OYE="; }; - propagatedBuildInputs = [ jdk ]; - nativeBuildInputs = [ makeWrapper ]; + nativeBuildInputs = + [ + makeWrapper + ] + ++ lib.optionals withCustomJre [ + jdk + babashka + ]; dontConfigure = true; - dontBuild = true; + dontBuild = !withCustomJre; + stripDebugFlags = lib.optionals withCustomJre [ "--strip-unneeded" ]; + + buildPhase = lib.optionalString withCustomJre '' + runHook preBuild + # Create a temporary file to store the jdeps output + mkdir -p tmp + + # Scan all jar files and analyze their dependencies + find lib -name "*.jar" -exec bash -c "${jdk}/bin/jdeps --multi-release base --ignore-missing-deps --print-module-deps {} >> tmp/modules.txt" \; + + ${jdk}/bin/jdeps --ignore-missing-deps --print-module-deps datomic-transactor*.jar >> tmp/modules.txt + # Combine all unique modules into a comma delimited string + modules=$(${babashka}/bin/bb -e '(print (str/join "," (->> (slurp *in*) str/split-lines (mapcat #(str/split % #",")) distinct (remove str/blank?))))' < tmp/modules.txt) + + rm -rf tmp + # Build the minimal JRE with the detected modules + jlink --module-path ${jdk}/lib/openjdk/jmods \ + --no-man-pages \ + --no-header-files \ + --add-modules "$modules" \ + --output $out/jre + + runHook postBuild + ''; installPhase = '' runHook preInstall mkdir -p $out/{bin,lib,share} cp *transactor*.jar $out/lib/ + rm -rf peer*.jar presto-server mkdir -p $out/share/datomic-pro cp -R * $out/share/datomic-pro/ mv $out/share/datomic-pro/bin/logback.xml $out/share/datomic-pro/logback-sample.xml @@ -51,7 +83,7 @@ stdenv.mkDerivation ( } ${lib.concatMapStringsSep "\n" (pl: "install_jars ${lib.escapeShellArg pl}") extraJavaPkgs} - makeWrapper ${jdk}/bin/java $out/bin/datomic-transactor \ + makeWrapper ${actualJre}/bin/java $out/bin/datomic-transactor \ --set-default "JAVA_OPTS" "-XX:+UseG1GC -XX:MaxGCPauseMillis=50 -Djava.library.path=\$LD_LIBRARY_PATH" \ --set-default "DATOMIC_JAVA_OPTS" "" \ --prefix LD_LIBRARY_PATH : "${libpath}" \ @@ -63,7 +95,7 @@ stdenv.mkDerivation ( --add-flags "--main" \ --add-flags "datomic.launcher" - makeWrapper ${jdk}/bin/java $out/bin/datomic-console \ + makeWrapper ${actualJre}/bin/java $out/bin/datomic-console \ --chdir "$out/share/datomic-pro" \ --prefix CLASSPATH : "$out/share/datomic-pro/lib/console/*:${classpath}" \ --add-flags "-server" \ @@ -77,7 +109,7 @@ stdenv.mkDerivation ( --add-flags "--main" \ --add-flags "datomic.console" - makeWrapper ${jdk}/bin/java $out/bin/datomic-shell \ + makeWrapper ${actualJre}/bin/java $out/bin/datomic-shell \ --chdir "$out/share/datomic-pro" \ --prefix CLASSPATH : "${classpath}" \ --add-flags "-server" \ @@ -89,7 +121,7 @@ stdenv.mkDerivation ( --add-flags clojure.main \ --add-flags "$out/share/datomic-pro/bin/shell.clj" - makeWrapper ${jdk}/bin/java $out/bin/datomic-run \ + makeWrapper ${actualJre}/bin/java $out/bin/datomic-run \ --chdir "$out/share/datomic-pro" \ --prefix CLASSPATH : "${classpath}" \ --add-flags "-server" \ diff --git a/pkgs/generate-properties.clj b/pkgs/generate-properties.clj index 9092b8e..f4f2251 100755 --- a/pkgs/generate-properties.clj +++ b/pkgs/generate-properties.clj @@ -1,4 +1,4 @@ -#!/usr/bin/env bb +#!@babashkaBin@ (require '[babashka.fs :as fs]) (require '[babashka.process :as p])