From 0ef6df07b3bc9bb08dec0c77d37cfef5c01c7f49 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 20:22:23 +0100 Subject: [PATCH 01/13] add template loki config --- .../tgsatan/modules/monitoring/default.nix | 1 + .../tgsatan/modules/monitoring/loki.nix | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/default.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/default.nix index 77d0bbb4..ea901b5e 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/default.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/default.nix @@ -1,5 +1,6 @@ {...}: { imports = [ ./prometheus.nix + ./loki.nix ]; } diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix new file mode 100644 index 00000000..a49340ca --- /dev/null +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -0,0 +1,46 @@ +{ ... }: { + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + server = { + http_listen_port = 3100; + grpc_listen_port = 9096; + }; + common = { + instance_addr = 127.0.0.1; + path_prefix = "/tmp/loki"; + storage.filesystem = { + chunks_directory = "/tmp/loki/chunks"; + rules_directory = "/tmp/loki/rules"; + }; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + query_range.results.cache.embedded_cache = { + enabled = true; + max_size_mb = 100; + }; + limits_config.metric_aggregation_enabled = true; + schema_config.configs = [ + { + from = "2020-10-24"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + pattern_ingester = { + enabled = true; + metric_aggregation.local_address = "localhost:3100"; + }; + ruler.alertmanager_url = "http://localhost:9093"; + frontend.encoding = "protobuf"; + querier.engine.enable_multi_variant_queries = true; + } + } +} From 22a813b9fbb08ba8d74d87d6e79ca2d97b10c863 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 20:22:48 +0100 Subject: [PATCH 02/13] oops --- .../game-servers/systems/tgsatan/modules/monitoring/loki.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix index a49340ca..57474daa 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -8,7 +8,7 @@ grpc_listen_port = 9096; }; common = { - instance_addr = 127.0.0.1; + instance_addr = "127.0.0.1"; path_prefix = "/tmp/loki"; storage.filesystem = { chunks_directory = "/tmp/loki/chunks"; From d94f303022f804892d49db815b916ae2bd031c98 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 21:59:05 +0100 Subject: [PATCH 03/13] add alloy --- modules/alloy.nix | 32 +++++++++++++++++++ modules/base.nix | 1 + .../tgsatan/modules/monitoring/loki.nix | 7 ++-- 3 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 modules/alloy.nix diff --git a/modules/alloy.nix b/modules/alloy.nix new file mode 100644 index 00000000..37aa96c7 --- /dev/null +++ b/modules/alloy.nix @@ -0,0 +1,32 @@ +{ pkgs }: let + loki_http_port = 3100; +in { + services.loki.configuration.server = { + http_listen_port = loki_http_port; + }; + services.alloy = { + enable = true; + configPath = pkgs.writeText "config.alloy" '' + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + } + + loki.source.journal "read" { + forward_to = [loki.write.endpoint.receiver] + relabel_rules = loki.relabel.journal.rules + labels = {component = "loki.source.journal"} + } + + loki.write "endpoint" { + endpoint { + url = "tgsatan.tg.lan:${loki_http_port}/api/v1/push" + } + } + ''; + }; +} diff --git a/modules/base.nix b/modules/base.nix index 67630c04..ba8e4163 100755 --- a/modules/base.nix +++ b/modules/base.nix @@ -7,6 +7,7 @@ imports = [ ./node-exporter.nix ./systemd-exporter.nix + ./alloy.nix ]; environment.variables."FLAKE" = "${self}"; diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix index 57474daa..add04546 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -3,10 +3,7 @@ enable = true; configuration = { auth_enabled = false; - server = { - http_listen_port = 3100; - grpc_listen_port = 9096; - }; + # ports defined in ./modules/alloy.nix common = { instance_addr = "127.0.0.1"; path_prefix = "/tmp/loki"; @@ -42,5 +39,5 @@ frontend.encoding = "protobuf"; querier.engine.enable_multi_variant_queries = true; } - } + }; } From 61bbf7faff0d1980ba3aa34626ab2fd215af470f Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 21:59:42 +0100 Subject: [PATCH 04/13] add loki to prometheus --- .../systems/tgsatan/modules/monitoring/prometheus.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/prometheus.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/prometheus.nix index b77d6557..9b183d98 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/prometheus.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/prometheus.nix @@ -23,6 +23,11 @@ in { job_name = "caddy"; static_configs = [{targets = ["tgsatan.tg.lan:2019"];}]; } + { + job_name = "loki"; + metrics_path = "/metrics"; + static_configs = [{targets = ["tgsatan.tg.lan:3100"];}]; + } # { # job_name = "tgsatan_forgejo"; # static_configs [ { targets= [ "tgsatan.tg.lan:9001" ]; } ]; From 083945b49bf3cbae1af06c9b864c8e07c9c87f79 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 22:11:46 +0100 Subject: [PATCH 05/13] add loki datasource --- .../systems/tgsatan/modules/grafana/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/systems/game-servers/systems/tgsatan/modules/grafana/default.nix b/systems/game-servers/systems/tgsatan/modules/grafana/default.nix index c7e9e400..72a26dc4 100644 --- a/systems/game-servers/systems/tgsatan/modules/grafana/default.nix +++ b/systems/game-servers/systems/tgsatan/modules/grafana/default.nix @@ -71,7 +71,13 @@ type = "prometheus"; access = "proxy"; url = "http://tgsatan.tg.lan:9090"; # Change when prometheus is moved - isDefault = true; + isDefault = true; # dead var? + } + { + name = "loki"; + type = "loki"; + proxy = "proxy"; + url = "http://tgsatan.tg.lan:3100"; } ]; From 6daf5640c32a82bd82018d081c233fa43d37ba57 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 22:30:30 +0100 Subject: [PATCH 06/13] comment --- .../game-servers/systems/tgsatan/modules/grafana/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/game-servers/systems/tgsatan/modules/grafana/default.nix b/systems/game-servers/systems/tgsatan/modules/grafana/default.nix index 72a26dc4..1689539b 100644 --- a/systems/game-servers/systems/tgsatan/modules/grafana/default.nix +++ b/systems/game-servers/systems/tgsatan/modules/grafana/default.nix @@ -77,7 +77,7 @@ name = "loki"; type = "loki"; proxy = "proxy"; - url = "http://tgsatan.tg.lan:3100"; + url = "http://tgsatan.tg.lan:3100"; # Change when Loki is moved } ]; From d914d2e7dbac257b5064b4385dbcef5c64dd8baf Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 22:33:02 +0100 Subject: [PATCH 07/13] fix build error --- modules/alloy.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/alloy.nix b/modules/alloy.nix index 37aa96c7..4281b6a0 100644 --- a/modules/alloy.nix +++ b/modules/alloy.nix @@ -1,4 +1,5 @@ -{ pkgs }: let +{ pkgs, ... }: +let loki_http_port = 3100; in { services.loki.configuration.server = { From a71398b8beaa788374268a34899e6b48bab01dff Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 22:34:41 +0100 Subject: [PATCH 08/13] another one ;_; --- .../game-servers/systems/tgsatan/modules/monitoring/loki.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix index add04546..2e9ad428 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -38,6 +38,6 @@ ruler.alertmanager_url = "http://localhost:9093"; frontend.encoding = "protobuf"; querier.engine.enable_multi_variant_queries = true; - } + }; }; } From a786f3096620a71c39009fd8909092a28611f0b8 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 22:37:42 +0100 Subject: [PATCH 09/13] another oneneee qwq --- modules/alloy.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alloy.nix b/modules/alloy.nix index 4281b6a0..963525a0 100644 --- a/modules/alloy.nix +++ b/modules/alloy.nix @@ -25,7 +25,7 @@ in { loki.write "endpoint" { endpoint { - url = "tgsatan.tg.lan:${loki_http_port}/api/v1/push" + url = "tgsatan.tg.lan:${toString loki_http_port}/api/v1/push" } } ''; From 90800f269467ca97b31831018afb83beffd7a01c Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Sun, 23 Mar 2025 23:02:59 +0100 Subject: [PATCH 10/13] this sucks im removing it --- modules/alloy.nix | 10 ++-------- .../systems/tgsatan/modules/monitoring/loki.nix | 4 +++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/modules/alloy.nix b/modules/alloy.nix index 963525a0..60a5c4e9 100644 --- a/modules/alloy.nix +++ b/modules/alloy.nix @@ -1,10 +1,4 @@ -{ pkgs, ... }: -let - loki_http_port = 3100; -in { - services.loki.configuration.server = { - http_listen_port = loki_http_port; - }; +{ pkgs, ... }: { services.alloy = { enable = true; configPath = pkgs.writeText "config.alloy" '' @@ -25,7 +19,7 @@ in { loki.write "endpoint" { endpoint { - url = "tgsatan.tg.lan:${toString loki_http_port}/api/v1/push" + url = "tgsatan.tg.lan:3100/api/v1/push" } } ''; diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix index 2e9ad428..6a6d6545 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -3,7 +3,9 @@ enable = true; configuration = { auth_enabled = false; - # ports defined in ./modules/alloy.nix + server = { + http_listen_port = 3100; + }; common = { instance_addr = "127.0.0.1"; path_prefix = "/tmp/loki"; From ccf96fb0fdc7a6aebaf0fc9fa92ea6ae755d0c91 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Mon, 24 Mar 2025 10:40:15 +0100 Subject: [PATCH 11/13] use newest example loki config --- .../systems/tgsatan/modules/monitoring/loki.nix | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix index 6a6d6545..d96b0bd3 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -20,7 +20,6 @@ enabled = true; max_size_mb = 100; }; - limits_config.metric_aggregation_enabled = true; schema_config.configs = [ { from = "2020-10-24"; @@ -33,13 +32,9 @@ }; } ]; - pattern_ingester = { - enabled = true; - metric_aggregation.local_address = "localhost:3100"; - }; - ruler.alertmanager_url = "http://localhost:9093"; + # Enable this if we ever use services.prometheus.alertManager.enable = true; + # ruler.alertmanager_url = "http://localhost:9093"; frontend.encoding = "protobuf"; - querier.engine.enable_multi_variant_queries = true; }; }; } From 9523039524a829bca28f9bb39573ab03d95bc8f1 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Mon, 24 Mar 2025 11:34:50 +0100 Subject: [PATCH 12/13] misread that --- .../game-servers/systems/tgsatan/modules/monitoring/loki.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix index d96b0bd3..d3c09dff 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -16,7 +16,7 @@ replication_factor = 1; ring.kvstore.store = "inmemory"; }; - query_range.results.cache.embedded_cache = { + query_range.results_cache.cache.embedded_cache = { enabled = true; max_size_mb = 100; }; From d24d43f40134eb52b29964752bbbb01f27829140 Mon Sep 17 00:00:00 2001 From: distributivgesetz Date: Mon, 24 Mar 2025 14:04:27 +0100 Subject: [PATCH 13/13] add loki data dir --- systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix index d3c09dff..a7fdcf7f 100644 --- a/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix +++ b/systems/game-servers/systems/tgsatan/modules/monitoring/loki.nix @@ -1,6 +1,7 @@ { ... }: { services.loki = { enable = true; + dataDir = "/persist/loki"; configuration = { auth_enabled = false; server = {