Skip to content

Commit

Permalink
Add more nixos module features and improve tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ramblurr
Ramblurr committed Oct 31, 2024
1 parent e95b876 commit 8951eb6
Show file tree
Hide file tree
Showing 11 changed files with 537 additions and 70 deletions.
8 changes: 4 additions & 4 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [UNRELEASED]

## v0.2.0 (2024-10-30)

### Breaking

- `transactor` bin renamed to `datomic-transactor`
- `console` bin renamed to `datomic-console`
- nix pkg: `transactor` bin renamed to `datomic-transactor`
- nix pkg: `console` bin renamed to `datomic-console`
- nixos module: removed the default settings that leaned towards dev/h2 storage by default

### Added

Expand All @@ -22,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `unstable` container image tag that follows the `main` branch
- nix pkg: Added ability to override the build and add extra native libs or java libs
- nix pkg: Exposed more packages: `datomic-shell`, `datomic-run`, `datomic-repl`, `datomic-peer-server`
- nixos module: You can now configure: logging, extra classpath entries, and extra java options.

### Changed

Expand Down
26 changes: 19 additions & 7 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,31 @@
DOCKER ?= docker

check:
nix flake check
test: check
nix --print-build-logs flake check

test/nixos:
nix --print-build-logs run '.#checks.x86_64-linux.moduleTest.driver'
test/container:
nix --print-build-logs run '.#checks.x86_64-linux.containerImageTest.driver'

test: test/nixos test/container

datomic-pro:
nix build .#datomic-pro -o result --show-trace

test-pkg: datomic-pro
./result/bin/datomic-transactor testsql.properties
#./result/bin/datomic-transactor ./result/share/datomic-pro/config/samples/dev-transactor-template.properties
test/pkg-dev: datomic-pro
mkdir -p data
./result/bin/datomic-transactor tests/fixtures/testdev.properties

test/pkg-sql: datomic-pro
mkdir -p data
./result/bin/datomic-transactor tests/fixtures/testsql.properties

test/pkg-console-dev: datomic-pro
./result/bin/datomic-console -p 8080 app 'datomic:dev://localhost:4334/?password=datpass'

test-pkg-console: datomic-pro
./result/bin/datomic-console -p 8080 app datomic:dev://localhost:4334/
test/pkg-console-sql: datomic-pro
./result/bin/datomic-console -p 8080 app 'datomic:sql://?jdbc:sqlite:data/db-sqlite.db'

datomic-pro-container:
nix build .#datomic-pro-container -o datomic-pro-container --show-trace
Expand Down
106 changes: 104 additions & 2 deletions nixos-modules/datomic-console.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,16 @@

let
cfg = config.services.datomic-console;
logbackConfigFile = pkgs.writeText "logback.xml" cfg.logbackConfig;
extraJavaOptions =
cfg.extraJavaOptions
++ lib.optional (cfg.logbackConfig != "") "-Dlogback.configurationFile=${logbackConfigFile}";
extraClasspath = lib.concatStringsSep ":" cfg.extraClasspathEntries;
in
{
options = {
services.datomic-console = {
enable = lib.mkEnableOption "Datomic Pro Console";
enable = lib.mkEnableOption "Datomic Console";
package = lib.mkPackageOption pkgs "datomic-pro" { };
javaPackage = lib.mkPackageOption pkgs "jdk21_headless" { };
port = lib.mkOption {
Expand All @@ -37,6 +42,96 @@ in
default = "datomic-console";
description = "The name of the directory under /var/lib that will be used as the state directory for datomic.";
};

extraJavaOptions = lib.mkOption {
description = "Extra command line options for Java.";
default = [ ];
type = lib.types.listOf lib.types.str;
example = [
"-Dfoo=bar"
"-Xbaz=bar"
];
};
extraClasspathEntries = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = ''
Extra entries added to the Java classpath when running Datomic Console
'';
example = [
"/path/to/my.jer"
"/path/to/folder/of/jars/*"
];
};
logbackConfig = lib.mkOption {
type = lib.types.lines;
default = ''
<configuration>
<!-- prevent per-message overhead for jul logging calls, e.g. Hornet -->
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %-10contextName %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger name="datomic.cast2slf4j" level="DEBUG"/>
<!-- uncomment to log storage access -->
<!-- <logger name="datomic.kv-cluster" level="DEBUG"/> -->
<!-- uncomment to log transactor heartbeat -->
<!-- <logger name="datomic.lifecycle" level="DEBUG"/> -->
<!-- uncomment to log transactions (transactor side) -->
<!-- <logger name="datomic.transaction" level="DEBUG"/> -->
<!-- uncomment to log transactions (peer side) -->
<!-- <logger name="datomic.peer" level="DEBUG"/> -->
<!-- uncomment to log the transactor log -->
<!-- <logger name="datomic.log" level="DEBUG"/> -->
<!-- uncomment to log peer connection to transactor -->
<!-- <logger name="datomic.connector" level="DEBUG"/> -->
<!-- uncomment to log storage gc -->
<!-- <logger name="datomic.garbage" level="DEBUG"/> -->
<!-- uncomment to log indexing jobs -->
<!-- <logger name="datomic.index" level="DEBUG"/> -->
<!-- these namespsaces create a ton of log noise -->
<logger name="org.apache.activemq.audit" level="WARN"/>
<logger name="httpclient" level="INFO"/>
<logger name="org.apache.commons.httpclient" level="INFO"/>
<logger name="org.apache.http" level="INFO"/>
<logger name="org.jets3t" level="INFO"/>
<logger name="com.amazonaws" level="INFO"/>
<logger name="com.amazonaws.request" level="WARN"/>
<logger name="sun.rmi" level="INFO"/>
<logger name="datomic.spy.memcached" level="INFO"/>
<logger name="com.couchbase.client" level="INFO"/>
<logger name="com.ning.http.client.providers.netty" level="INFO"/>
<logger name="org.eclipse.jetty" level="INFO"/>
<logger name="org.hornetq.core.client.impl" level="INFO"/>
<logger name="org.apache.tomcat.jdbc.pool" level="INFO"/>
<logger name="datomic.cast2slf4j" level="DEBUG"/>
<root level="info">
<appender-ref ref="STDOUT"/>
</root>
</configuration>
'';
description = ''
XML logback configuration for the datomic-pro transactor.
'';
};

};
};
config = lib.mkIf cfg.enable {
Expand All @@ -50,13 +145,20 @@ in
}
];
systemd.services.datomic-console = {
description = "Datomic Pro Console";
description = "Datomic Console";
wantedBy = [ "multi-user.target" ];
script = ''
db_uri="$(<"$CREDENTIALS_DIRECTORY/datomic-console-db-uri")"
${cfg.package}/bin/datomic-console -p ${toString cfg.port} "${cfg.alias}" "$db_uri"
'';
path = [ cfg.javaPackage ];
environment =
{
DATOMIC_JAVA_OPTS = toString extraJavaOptions;
}
// lib.optionalAttrs (cfg.extraClasspathEntries != [ ]) {
CLASSPATH = extraClasspath;
};
serviceConfig = {
Type = "simple";
LoadCredential = [ "datomic-console-db-uri:${cfg.dbUriFile}" ];
Expand Down
141 changes: 118 additions & 23 deletions nixos-modules/datomic-pro.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,23 @@ let
settingsFormat = pkgs.formats.javaProperties { };
stateDir = "/var/lib/${cfg.stateDirectoryName}";
runtimePropertiesPath = "${stateDir}/transactor.properties";
# default settings that will be used unless overriden by the user
settingsDefault = {
host = "localhost";
memory-index-max = "256m";
memory-index-threshold = "32m";
object-cache-max = "128m";
host = "127.0.0.1";
port = 4334;
protocol = "dev";
data-dir = "${stateDir}/data";
log-dir = "${stateDir}/log";
};
propertiesFile = settingsFormat.generate "transactor.properties" (settingsDefault // cfg.settings);
logbackConfigFile = pkgs.writeText "logback.xml" cfg.logbackConfig;
extraJavaOptions =
cfg.extraJavaOptions
++ lib.optional (cfg.logbackConfig != "") "-Dlogback.configurationFile=${logbackConfigFile}";
extraClasspath = lib.concatStringsSep ":" cfg.extraClasspathEntries;
in
{
options = {
services.datomic-pro = {
enable = lib.mkEnableOption "Datomic Pro";
package = lib.mkPackageOption pkgs "datomic-pro" { };
javaPackage = lib.mkPackageOption pkgs "jdk21_headless" { };
secretsFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
Expand All @@ -38,6 +36,94 @@ in
Should be owned by root and have 0600 permissions.
'';
};
extraJavaOptions = lib.mkOption {
description = "Extra command line options for Java.";
default = [ ];
type = lib.types.listOf lib.types.str;
example = [
"-Dfoo=bar"
"-Xbaz=bar"
];
};
extraClasspathEntries = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = ''
Extra entries added to the Java classpath when running Datomic Pro.
'';
example = [
"/path/to/my.jer"
"/path/to/folder/of/jars/*"
];
};
logbackConfig = lib.mkOption {
type = lib.types.lines;
default = ''
<configuration>
<!-- prevent per-message overhead for jul logging calls, e.g. Hornet -->
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %-10contextName %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<logger name="datomic.cast2slf4j" level="DEBUG"/>
<!-- uncomment to log storage access -->
<!-- <logger name="datomic.kv-cluster" level="DEBUG"/> -->
<!-- uncomment to log transactor heartbeat -->
<!-- <logger name="datomic.lifecycle" level="DEBUG"/> -->
<!-- uncomment to log transactions (transactor side) -->
<!-- <logger name="datomic.transaction" level="DEBUG"/> -->
<!-- uncomment to log transactions (peer side) -->
<!-- <logger name="datomic.peer" level="DEBUG"/> -->
<!-- uncomment to log the transactor log -->
<!-- <logger name="datomic.log" level="DEBUG"/> -->
<!-- uncomment to log peer connection to transactor -->
<!-- <logger name="datomic.connector" level="DEBUG"/> -->
<!-- uncomment to log storage gc -->
<!-- <logger name="datomic.garbage" level="DEBUG"/> -->
<!-- uncomment to log indexing jobs -->
<!-- <logger name="datomic.index" level="DEBUG"/> -->
<!-- these namespsaces create a ton of log noise -->
<logger name="org.apache.activemq.audit" level="WARN"/>
<logger name="httpclient" level="INFO"/>
<logger name="org.apache.commons.httpclient" level="INFO"/>
<logger name="org.apache.http" level="INFO"/>
<logger name="org.jets3t" level="INFO"/>
<logger name="com.amazonaws" level="INFO"/>
<logger name="com.amazonaws.request" level="WARN"/>
<logger name="sun.rmi" level="INFO"/>
<logger name="datomic.spy.memcached" level="INFO"/>
<logger name="com.couchbase.client" level="INFO"/>
<logger name="com.ning.http.client.providers.netty" level="INFO"/>
<logger name="org.eclipse.jetty" level="INFO"/>
<logger name="org.hornetq.core.client.impl" level="INFO"/>
<logger name="org.apache.tomcat.jdbc.pool" level="INFO"/>
<logger name="datomic.cast2slf4j" level="DEBUG"/>
<root level="info">
<appender-ref ref="STDOUT"/>
</root>
</configuration>
'';
description = ''
XML logback configuration for the datomic-pro transactor.
'';
};

stateDirectoryName = lib.mkOption {
type = lib.types.str;
Expand All @@ -47,16 +133,7 @@ in

settings = lib.mkOption {
type = lib.types.submodule { freeformType = settingsFormat.type; };
default = {
host = "localhost";
memory-index-max = "256m";
memory-index-threshold = "32m";
object-cache-max = "128m";
port = 4334;
protocol = "dev";
data-dir = "${stateDir}/data";
log-dir = "${stateDir}/log";
};
default = settingsDefault;
description = ''
Configuration written to `transactor.properties`.
Expand All @@ -70,17 +147,31 @@ in
};
config = lib.mkIf cfg.enable {
assertions = [
{
assertion = lib.attrsets.hasAttr "protocol" cfg.settings;
message = ''
You must define your storage procotol with the `protocol` key in <option>services.datomic-pro.settings</option> , refer to the Datomic Pro documentation.
Some possible values are `"dev"`, `"sql"`, etc. Each protocol will have additional required settings that are not validated by this NixOS module.
'';
}
{
assertion = lib.strings.hasInfix "/" cfg.stateDirectoryName == false;
message = ''
<option>services.datomic-pro.stateDirectoryName> must be a single directory name, not a path with /.
<option>services.datomic-pro.stateDirectoryName</option> must be a single directory name, not a path with /.
'';
}

{
assertion = !(lib.attrsets.hasAttr "log-dir" cfg.settings);
message = ''<option>services.datomic-pro.settings</option> must not contain the `log-dir` key, use <option>services.datomic-pro.logbackConfig</option> instead.'';
# Ok intrepid spelunker, why can we not use log-dir? Because as of 2024-10, the log-dir is specially handled by datomic code and hardcodes a lookup for
# logback.xml in the `bin/` dir of the datomic tarball. This obviously doesn't work on NixOS.
# The solution is to NOT define log-dir, but instead just define your own logback configuration, we include a variation of the default that logs to stdout and ends up in systemd's journal.
}
];
systemd.services.datomic-pro = {
description = "Datomic Pro";
wantedBy = [ "multi-user.target" ];
path = [ cfg.javaPackage ];
preStart = ''
cat ${propertiesFile} > ${runtimePropertiesPath}
chmod 0600 ${runtimePropertiesPath}
Expand All @@ -92,9 +183,13 @@ in
script = ''
${cfg.package}/bin/datomic-transactor ${runtimePropertiesPath}
'';
environment = {
DATOMIC_JAVA_OPTS = "-Dlogback.configurationFile ${cfg.package}/share/datomic-pro/logback-sample.xml";
};
environment =
{
DATOMIC_JAVA_OPTS = toString extraJavaOptions;
}
// lib.optionalAttrs (cfg.extraClasspathEntries != [ ]) {
CLASSPATH = extraClasspath;
};
serviceConfig = {
Type = "simple";
DynamicUser = true;
Expand Down
Loading

0 comments on commit 8951eb6

Please sign in to comment.