Make the docker image extensible with extra jdbc drivers

Ramblurr · Oct 30, 2024 · e95b876 · e95b876
1 parent 1471dc2
commit e95b876
Show file tree

Hide file tree

Showing 8 changed files with 282 additions and 91 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,28 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
-### Added
+## v0.2.0 (2024-10-30)
 
-- Include sqlite, postgresql, and mysql JDBC drivers by default
-- Added `unstable` container image tag that follows the `main` branch.
-- Added coreutils and bash to the OCI container image, to make administration easier
+### Breaking
 
-### Fixed
-
-- The changelog formatting
-
-## v0.2.0 (2024-10-30)
+- `transactor` bin renamed to `datomic-transactor`
+- `console` bin renamed to `datomic-console`
 
 ### Added
 
-- Added Docker container image
+- oci image: Added Docker container image with lots of customizable features
+   - Includes sqlite, postgresql, and mysql JDBC drivers by default
+   - Ability to customize the CLASSPATH and LD_LIBRARY_PATH
+   - `unstable` container image tag that follows the `main` branch
+- nix pkg: Added ability to override the build and add extra native libs or java libs
+- nix pkg: Exposed more packages: `datomic-shell`, `datomic-run`, `datomic-repl`, `datomic-peer-server`
 
 ### Changed
 
-- Updated datomic-pro to version 1.0.7260
-- Switched to Nix's JDK 21 headless package (now that it is sufficiently headless
+- nix pkg: Updated datomic-pro to version 1.0.7260
+- nix pkg: Switched to Nix's JDK 21 headless package (now that it is sufficiently headless
 
-### Added
+### Fixed
+
+- This changelog formatting
 
 ## v0.1.0 (2024-06-12)
 

diff --git a/Makefile b/Makefile
@@ -4,9 +4,17 @@ check:
 	nix flake check
 test: check
 
+
 datomic-pro:
 	nix build .#datomic-pro -o result --show-trace
 
+test-pkg: datomic-pro
+	./result/bin/datomic-transactor testsql.properties
+	#./result/bin/datomic-transactor ./result/share/datomic-pro/config/samples/dev-transactor-template.properties
+
+test-pkg-console: datomic-pro
+	./result/bin/datomic-console -p 8080 app datomic:dev://localhost:4334/
+
 datomic-pro-container:
 	nix build .#datomic-pro-container -o datomic-pro-container --show-trace
 
@@ -15,6 +23,7 @@ datomic-pro-container-unstable:
 
 clean:
 	rm -f result container datomic-pro-container-unstable datomic-pro-container
+	rm -rf data/
 
 load: datomic-pro-container-unstable
 	$(DOCKER) load < ./datomic-pro-container-unstable
diff --git a/README.md b/README.md
@@ -12,6 +12,9 @@
 
 All of the above are tested automatically with a virtual machine!
 
+
+**Project status:** Experimental but ready for testing. Breaking changes may occur until version 1.0. The 1.0 release will be considered production-ready.
+
 <!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc -->
 **Table of Contents**
 
@@ -182,6 +185,11 @@ The following environment vars configure the properties, refer to the Datomic do
 * `DATOMIC_VALCACHE_PATH` - `valcache-path`
 * `DATOMIC_WRITE_CONCURRENCY` - `write-concurrency`
 
+If you want to provide your own `transactor.properties`, you can opt out of all of the above by:
+
+1. Placing your properties file into a location such that `/config/transactor.properties` will exist when the container runs.
+2. Set the env variable `DOCKER_DATOMIC_GENERATE_PROPERTIES_SKIP` to anything except an empty string.
+
 ### Console Mode
 
 Runs the Datomic Console.
@@ -198,7 +206,7 @@ Runs the Datomic Console.
 
 #### Datomic Pro with Local Storage
 
-Please note the tag below may not be up to date.
+Be sure to `mkdir data/ config/` before running this.
 
 ``` yaml
 ---
@@ -224,13 +232,79 @@ services:
     #user: 1000:1000 # if using rootful containers uncomment this
 ```
 
+#### Datomic Pro with sqlite storage
+
+Be sure to `mkdir data/ config/` before running this.
+
+``` yaml
+---
+services:
+  datomic-transactor:
+    image: ghcr.io/ramblurr/datomic-pro:unstable
+    environment:
+      DATOMIC_PROTOCOL: sql
+      DATOMIC_SQL_URL: jdbc:sqlite:/data/datomic-sqlite.db
+      DATOMIC_SQL_DRIVER_CLASS: org.sqlite.JDBC
+      DATOMIC_JAVA_OPTS: -Dlogback.configurationFile=/config/logback.xml
+      # this one is for the containers
+      DATOMIC_HOST: datomic-transactor
+      # this one is for the host machine
+      DATOMIC_ALT_HOST: "127.0.0.1"
+    volumes:
+      - "./data:/data:z"
+      - "./config:/config:z"
+    ports:
+      - 127.0.0.1:4334:4334
+    depends_on:
+      datomic-storage-migrator:
+        condition: service_completed_successfully
+
+  datomic-console:
+    image: ghcr.io/ramblurr/datomic-pro:unstable
+    command: console
+    environment:
+      # you don’t specify the db name in the uri (because console can access all dbs)
+      DB_URI: "datomic:sql://?jdbc:sqlite:/data/datomic-sqlite.db"
+    volumes:
+      - "./data:/data:z"
+    ports:
+      - 127.0.0.1:8081:8080
+    depends_on:
+      datomic-storage-migrator:
+        condition: service_completed_successfully
+
+  datomic-storage-migrator:
+    image: ghcr.io/ramblurr/datomic-pro:unstable
+    volumes:
+      - "./data:/data:z"
+    entrypoint: /bin/sh
+    command: |
+      -c '
+      echo "Creating SQLite database and schema..."
+      sqlite3 /data/datomic-sqlite.db "
+      CREATE TABLE IF NOT EXISTS datomic_kvs (
+        id TEXT NOT NULL PRIMARY KEY,
+        rev INTEGER,
+        map TEXT,
+        val BLOB
+      );"
+      echo "Database initialization complete."
+      '
+
+      export DATOMIC_PROTOCOL=sql
+      export DATOMIC_SQL_URL=jdbc:sqlite:/tmp/db.db
+      export DATOMIC_SQL_DRIVER_CLASS=org.sqlite.JDBC
+      export DATOMIC_ALT_HOST=127.0.0.1
+      export DATOMIC_JAVA_OPTS=-Dlogback.configurationFile=/config/logback.xml
+```
+
 #### Datomic Pro with Postgres Storage and memcached
 
 This compose file is near-production ready. But you shouldn't manage the
 lifecycle of the postgres schema this way. How you do it depends on your
 environment.
 
-Please note the tag below may not be up to date.
+Be sure to `mkdir data/ config/` before running this.
 
 ``` yaml
 ---

diff --git a/nixos-modules/datomic-console.nix b/nixos-modules/datomic-console.nix
@@ -54,7 +54,7 @@ in
       wantedBy = [ "multi-user.target" ];
       script = ''
         db_uri="$(<"$CREDENTIALS_DIRECTORY/datomic-console-db-uri")"
-        ${cfg.package}/bin/console -p ${toString cfg.port} "${cfg.alias}" "$db_uri"
+        ${cfg.package}/bin/datomic-console -p ${toString cfg.port} "${cfg.alias}" "$db_uri"
       '';
       path = [ cfg.javaPackage ];
       serviceConfig = {

diff --git a/nixos-modules/datomic-pro.nix b/nixos-modules/datomic-pro.nix
@@ -90,8 +90,11 @@ in
         mkdir -p ${stateDir}/data ${stateDir}/log
       '';
       script = ''
-        ${cfg.package}/bin/transactor ${runtimePropertiesPath}
+        ${cfg.package}/bin/datomic-transactor ${runtimePropertiesPath}
       '';
+      environment = {
+        DATOMIC_JAVA_OPTS = "-Dlogback.configurationFile ${cfg.package}/share/datomic-pro/logback-sample.xml";
+      };
       serviceConfig = {
         Type = "simple";
         DynamicUser = true;

diff --git a/pkgs/datomic-pro-container-image.nix b/pkgs/datomic-pro-container-image.nix
@@ -1,24 +1,40 @@
 {
+  babashka,
+  bash,
+  bashInteractive,
+  cacert,
+  coreutils,
+  datomic-generate-properties,
+  datomic-pro,
+  dockerTools,
+  fetchzip,
+  hostname,
   imageTag,
+  jdk21_headless,
   lib,
+  mysql_jdbc,
+  runCommand,
+  sqlite-jdbc,
   stdenv,
-  fetchzip,
-  dockerTools,
-  datomic-pro,
-  datomic-generate-properties,
   writeShellScriptBin,
-  cacert,
-  bashInteractive,
-  coreutils,
-  jdk21_headless,
-  babashka,
-  hostname,
-  bash,
+  ## TODO: remove these, they are only for testing
+  sqlite-interactive,
+  gnugrep,
+  which,
+  findutils,
+  vim,
   ...
 }:
 
 let
   jdk = jdk21_headless;
+
+  datomicBuild = datomic-pro.override {
+    extraJavaPkgs = [
+      sqlite-jdbc
+      mysql_jdbc
+    ];
+  };
   entrypoint = writeShellScriptBin "datomic-entrypoint" ''
     set -e
     export PATH=${bash}/bin:${hostname}/bin:${jdk}/bin:${coreutils}/bin:${babashka}:$PATH
@@ -35,33 +51,53 @@ let
         echo "DB_URI is not set. Please set DB_URI environment variable or provide a file path with DB_URI_FILE."
         exit 1
       fi
-      ${datomic-pro}/bin/console -p 8080 dev "$DB_URI"
+      ${datomicBuild}/bin/datomic-console -p 8080 dev "$DB_URI"
     else
-      echo "Generating Datomic Properties"
-      ${datomic-generate-properties}/bin/datomic-generate-properties
+      if [ -z "$DOCKER_DATOMIC_GENERATE_PROPERTIES_SKIP" ]; then
+        echo "Generating Datomic Properties"
+        ${datomic-generate-properties}/bin/datomic-generate-properties
+      else
+        echo "Skipping Datomic Properties Generation"
+      fi
       echo "Starting Datomic Transactor..."
-      ${datomic-pro}/bin/transactor "$DATOMIC_TRANSACTOR_PROPERTIES_PATH"
+      ${datomicBuild}/bin/datomic-transactor "$DATOMIC_TRANSACTOR_PROPERTIES_PATH"
     fi
   '';
+  env-shim = runCommand "env-shim" { } ''
+    mkdir -p $out/usr/bin
+    ln -s ${coreutils}/bin/env $out/usr/bin/env
+  '';
 in
 dockerTools.buildLayeredImage {
   name = "ghcr.io/ramblurr/datomic-pro";
   tag = imageTag;
   fromImage = null;
   contents = [
-    datomic-pro
-    entrypoint
-    datomic-generate-properties
+    babashka
     bashInteractive
+    cacert
     coreutils
-    babashka
+    datomic-generate-properties
+    datomicBuild
+    entrypoint
+    env-shim
     jdk
-    cacert
+    mysql_jdbc
+    sqlite-jdbc
+    ## TODO: remove these, they are only for testing
+    sqlite-interactive
+    gnugrep
+    which
+    findutils
+    vim
   ];
+  extraCommands = ''
+    mkdir -p tmp
+    chmod 1777 tmp
+  '';
+
   config = {
-    WorkingDir = datomic-pro;
     Entrypoint = [ "${entrypoint}/bin/datomic-entrypoint" ];
-
     Env = [
       "DATOMIC_TRANSACTOR_PROPERTIES_PATH=/config/transactor.properties"
       "LC_ALL=C.UTF-8"
@@ -74,7 +110,7 @@ dockerTools.buildLayeredImage {
       "org.opencontainers.image.url" = "https://github.com/Ramblurr/datomic-pro-flake/tree/main/pkgs/datomic-pro-container-image.nix";
       "org.opencontainers.image.source" = "https://github.com/Ramblurr/datomic-pro-flake";
       "org.opencontainers.image.description" = "Datomic Pro";
-      "org.opencontainers.image.version" = datomic-pro.version;
+      "org.opencontainers.image.version" = datomicBuild.version;
       "org.opencontainers.image.licenses" = "Apache-2.0";
     };
     Volumes = {