From 44a029da0a03eaca54b6ddb7c6de72981ba74d08 Mon Sep 17 00:00:00 2001 From: mcorbin Date: Tue, 24 Dec 2024 14:59:48 +0100 Subject: [PATCH 1/3] New release doc --- dev/resources/dsl/stream.clj | 46 +- dev/resources/streams/stream.clj | 57 +- dev/resources/tests/test1.edn | 40 +- site/mirabelle/content/_index.md | 58 +- site/mirabelle/content/howto/build/_index.md | 2 +- site/mirabelle/content/howto/pubsub/_index.md | 21 +- .../content/howto/stream-index/_index.md | 74 -- site/mirabelle/content/howto/stream/_index.md | 65 +- site/mirabelle/content/howto/tests/_index.md | 14 +- site/mirabelle/public/404.html | 14 +- site/mirabelle/public/api/css/default.css | 551 ---------- site/mirabelle/public/api/css/highlight.css | 97 -- site/mirabelle/public/api/index.html | 90 +- site/mirabelle/public/api/js/highlight.min.js | 2 - site/mirabelle/public/api/js/jquery.min.js | 4 - site/mirabelle/public/api/js/page_effects.js | 112 -- .../api/mirabelle.action.condition.html | 3 - .../public/api/mirabelle.action.html | 313 ------ site/mirabelle/public/api/mirabelle.b64.html | 3 - .../public/api/mirabelle.config.html | 3 - site/mirabelle/public/api/mirabelle.core.html | 3 - .../public/api/mirabelle.db.memtable.html | 5 - .../public/api/mirabelle.db.queue.html | 3 - .../mirabelle/public/api/mirabelle.event.html | 3 - .../public/api/mirabelle.graphviz.html | 3 - .../public/api/mirabelle.handler.html | 3 - site/mirabelle/public/api/mirabelle.http.html | 3 - .../mirabelle/public/api/mirabelle.index.html | 3 - .../api/mirabelle.interceptor.route.html | 3 - .../api/mirabelle.io.elasticsearch.html | 3 - .../public/api/mirabelle.io.file.html | 3 - site/mirabelle/public/api/mirabelle.io.html | 3 - .../public/api/mirabelle.io.influxdb.html | 3 - .../public/api/mirabelle.io.pagerduty.html | 3 - site/mirabelle/public/api/mirabelle.math.html | 5 - site/mirabelle/public/api/mirabelle.path.html | 3 - site/mirabelle/public/api/mirabelle.pool.html | 9 - .../public/api/mirabelle.pubsub.html | 3 - site/mirabelle/public/api/mirabelle.spec.html | 3 - .../public/api/mirabelle.stream.html | 4 - site/mirabelle/public/api/mirabelle.test.html | 3 - site/mirabelle/public/api/mirabelle.time.html | 3 - .../public/api/mirabelle.transport.html | 8 - .../public/api/mirabelle.transport.tcp.html | 4 - .../api/mirabelle.transport.websocket.html | 3 - site/mirabelle/public/api/user.html | 3 - site/mirabelle/public/categories/index.html | 90 +- site/mirabelle/public/changelog/index.html | 90 +- site/mirabelle/public/extension/index.html | 90 +- .../mirabelle/public/generated-doc/index.html | 3 +- .../mirabelle.action.condition.html | 6 +- .../generated-doc/mirabelle.action.html | 278 ++--- .../public/generated-doc/mirabelle.b64.html | 2 +- .../generated-doc/mirabelle.config.html | 4 +- .../public/generated-doc/mirabelle.core.html | 10 +- .../public/generated-doc/mirabelle.event.html | 16 +- .../generated-doc/mirabelle.graphviz.html | 4 +- .../generated-doc/mirabelle.handler.html | 6 +- .../public/generated-doc/mirabelle.http.html | 2 +- .../public/generated-doc/mirabelle.index.html | 6 +- .../public/generated-doc/mirabelle.io.html | 4 +- .../public/generated-doc/mirabelle.math.html | 22 +- .../generated-doc/mirabelle.otel.traces.html | 2 +- .../generated-doc/mirabelle.output.batch.html | 3 + .../mirabelle.output.elasticsearch.html | 2 +- .../generated-doc/mirabelle.output.file.html | 2 +- .../generated-doc/mirabelle.output.http.html | 3 + .../mirabelle.output.influxdb.html | 8 +- .../mirabelle.output.pagerduty.html | 10 +- .../mirabelle.output.prometheus.html | 3 + .../public/generated-doc/mirabelle.path.html | 4 +- .../public/generated-doc/mirabelle.pool.html | 8 +- .../generated-doc/mirabelle.prometheus.html | 6 +- .../generated-doc/mirabelle.pubsub.html | 2 +- .../public/generated-doc/mirabelle.spec.html | 2 +- .../public/generated-doc/mirabelle.store.html | 2 +- .../generated-doc/mirabelle.stream.html | 14 +- .../public/generated-doc/mirabelle.test.html | 4 +- .../public/generated-doc/mirabelle.time.html | 4 +- .../mirabelle.transport.codec.html | 7 + .../generated-doc/mirabelle.transport.html | 32 +- .../mirabelle.transport.tcp.html | 10 +- .../mirabelle.transport.websocket.html | 8 +- .../public/howto/action-io-ref/index.html | 92 +- site/mirabelle/public/howto/build/index.html | 92 +- .../public/howto/configuration/index.html | 90 +- site/mirabelle/public/howto/index.html | 90 +- site/mirabelle/public/howto/index/index.html | 983 ------------------ site/mirabelle/public/howto/index/index.xml | 10 - .../public/howto/on-disk-queue/index.html | 962 ----------------- .../public/howto/on-disk-queue/index.xml | 10 - site/mirabelle/public/howto/pubsub/index.html | 114 +- .../public/howto/stream-index/index.html | 941 ----------------- .../public/howto/stream-index/index.xml | 11 - site/mirabelle/public/howto/stream/index.html | 142 +-- site/mirabelle/public/howto/tests/index.html | 103 +- site/mirabelle/public/index.html | 174 ++-- site/mirabelle/public/integration/index.html | 90 +- site/mirabelle/public/production/index.html | 90 +- site/mirabelle/public/riemann-diff/index.html | 90 +- site/mirabelle/public/sitemap.xml | 2 - site/mirabelle/public/support/index.html | 90 +- site/mirabelle/public/tags/index.html | 90 +- .../mirabelle/static/generated-doc/index.html | 2 +- .../generated-doc/mirabelle.action.html | 14 +- .../generated-doc/mirabelle.handler.html | 4 +- .../static/generated-doc/mirabelle.math.html | 20 +- .../static/generated-doc/mirabelle.time.html | 4 +- src/clojure/mirabelle/action.clj | 12 +- test/mirabelle/action_test.clj | 3 +- test/mirabelle/stream_test.clj | 35 + 111 files changed, 978 insertions(+), 5815 deletions(-) delete mode 100644 site/mirabelle/content/howto/stream-index/_index.md delete mode 100644 site/mirabelle/public/api/css/default.css delete mode 100644 site/mirabelle/public/api/css/highlight.css delete mode 100644 site/mirabelle/public/api/js/highlight.min.js delete mode 100644 site/mirabelle/public/api/js/jquery.min.js delete mode 100644 site/mirabelle/public/api/js/page_effects.js delete mode 100644 site/mirabelle/public/api/mirabelle.action.condition.html delete mode 100644 site/mirabelle/public/api/mirabelle.action.html delete mode 100644 site/mirabelle/public/api/mirabelle.b64.html delete mode 100644 site/mirabelle/public/api/mirabelle.config.html delete mode 100644 site/mirabelle/public/api/mirabelle.core.html delete mode 100644 site/mirabelle/public/api/mirabelle.db.memtable.html delete mode 100644 site/mirabelle/public/api/mirabelle.db.queue.html delete mode 100644 site/mirabelle/public/api/mirabelle.event.html delete mode 100644 site/mirabelle/public/api/mirabelle.graphviz.html delete mode 100644 site/mirabelle/public/api/mirabelle.handler.html delete mode 100644 site/mirabelle/public/api/mirabelle.http.html delete mode 100644 site/mirabelle/public/api/mirabelle.index.html delete mode 100644 site/mirabelle/public/api/mirabelle.interceptor.route.html delete mode 100644 site/mirabelle/public/api/mirabelle.io.elasticsearch.html delete mode 100644 site/mirabelle/public/api/mirabelle.io.file.html delete mode 100644 site/mirabelle/public/api/mirabelle.io.html delete mode 100644 site/mirabelle/public/api/mirabelle.io.influxdb.html delete mode 100644 site/mirabelle/public/api/mirabelle.io.pagerduty.html delete mode 100644 site/mirabelle/public/api/mirabelle.math.html delete mode 100644 site/mirabelle/public/api/mirabelle.path.html delete mode 100644 site/mirabelle/public/api/mirabelle.pool.html delete mode 100644 site/mirabelle/public/api/mirabelle.pubsub.html delete mode 100644 site/mirabelle/public/api/mirabelle.spec.html delete mode 100644 site/mirabelle/public/api/mirabelle.stream.html delete mode 100644 site/mirabelle/public/api/mirabelle.test.html delete mode 100644 site/mirabelle/public/api/mirabelle.time.html delete mode 100644 site/mirabelle/public/api/mirabelle.transport.html delete mode 100644 site/mirabelle/public/api/mirabelle.transport.tcp.html delete mode 100644 site/mirabelle/public/api/mirabelle.transport.websocket.html delete mode 100644 site/mirabelle/public/api/user.html create mode 100644 site/mirabelle/public/generated-doc/mirabelle.output.batch.html create mode 100644 site/mirabelle/public/generated-doc/mirabelle.output.http.html create mode 100644 site/mirabelle/public/generated-doc/mirabelle.output.prometheus.html create mode 100644 site/mirabelle/public/generated-doc/mirabelle.transport.codec.html delete mode 100644 site/mirabelle/public/howto/index/index.html delete mode 100644 site/mirabelle/public/howto/index/index.xml delete mode 100644 site/mirabelle/public/howto/on-disk-queue/index.html delete mode 100644 site/mirabelle/public/howto/on-disk-queue/index.xml delete mode 100644 site/mirabelle/public/howto/stream-index/index.html delete mode 100644 site/mirabelle/public/howto/stream-index/index.xml diff --git a/dev/resources/dsl/stream.clj b/dev/resources/dsl/stream.clj index 2739a44a..a0338c7c 100644 --- a/dev/resources/dsl/stream.clj +++ b/dev/resources/dsl/stream.clj @@ -1,31 +1,17 @@ -;; (streams -;; ;; (stream {:name :traces :default true} -;; ;; (where [:= :kind "client"] -;; ;; (rate {:duration 10} -;; ;; (with {:name "RATE"} -;; ;; (info))))) -;; ;; (stream {:name :test :default false} -;; ;; (where [:= :kind "client"] -;; ;; (rate {:duration 10} -;; ;; (with {:name "RATE"} -;; ;; (tap :result) -;; ;; (info))))) - -;; (stream {:name :percentiles }) -;; ) - (streams - (stream {:name :percentiles :default true} - (where [:= :name "http_request_duration_seconds"] - - (by {:fields [[:attributes :application] - [:attributes :environment]]} - (percentiles {:percentiles [0.5 0.75 0.99] - :duration 20 - :nb-significant-digits 3} - (info) - (where [:and [:= :quantile "0.99"] - [:> :metric 1]] - (with {:state "error"} - (error) - (tap :alert)))))))) + (stream {:name :multiple-branches} + (where [:= :name "http_request_duration_seconds"] + (with :ttl 60 + ;; push everything into influxdb + (output! :influxdb) + ;; by will generate a branch for each :host value. Like that, downstream + ;; computations will be per host and will not conflict between each other + (by {:fields [:host]} + ;; if the metric is greater than 1 for more than 60 seconds + ;; Pass events downstream + (above-dt {:duration 60 :threshold 1} + ;; pass the state to critical + (with :state "critical" + ;; one alert only every 60 sec to avoid flooding pagerduty + (throttle {:duration 60 :count 1} + (output! :pagerduty))))))))) diff --git a/dev/resources/streams/stream.clj b/dev/resources/streams/stream.clj index 1e35d49c..19d2de1a 100644 --- a/dev/resources/streams/stream.clj +++ b/dev/resources/streams/stream.clj @@ -1,6 +1,5 @@ -{:percentiles - {:default true, - :actions +{:multiple-branches + {:actions {:action :sdo, :description {:message "Forward events to children"}, :children @@ -10,41 +9,35 @@ :params "[:= :name \"http_request_duration_seconds\"]"}, :params [[:= :name "http_request_duration_seconds"]], :children - ({:action :by, - :description - {:message - "Split streams by field(s) [[:attributes :application] [:attributes :environment]]"}, - :params - [{:fields - [[:attributes :application] [:attributes :environment]]}], + ({:action :with, + :description {:message "Set the field :ttl to 60"}, :children - ({:action :percentiles, + ({:action :output!, :description - {:message "Computes the quantiles [0.5 0.75 0.99]"}, - :params - [{:percentiles [0.5 0.75 0.99], - :duration 20000000000, - :nb-significant-digits 3}], + {:message "Forward events to the output :influxdb"}, + :params [:influxdb]} + {:action :by, + :description {:message "Split streams by field(s) [:host]"}, + :params [{:fields [:host]}], :children - ({:action :info, + ({:action :above-dt, :description - {:message "Print the event in the logs as info"}} - {:action :where, - :description - {:message "Filter events based on the provided condition", - :params "[:and [:= :quantile \"0.99\"] [:> :metric 1]]"}, - :params [[:and [:= :quantile "0.99"] [:> :metric 1]]], + {:message + "Keep events if :metric is greater than 1 during 60 seconds"}, + :params [[:> :metric 1] 60000000000], :children ({:action :with, :description - {:message "Merge the events with the provided fields", - :params "{:state \"error\"}"}, + {:message "Set the field :state to critical"}, :children - ({:action :error, - :description - {:message "Print the event in the logs as error"}} - {:action :tap, + ({:action :throttle, :description - {:message "Save events into the tap :alert"}, - :params [:alert]}), - :params [{:state "error"}]})})})})})}}} + {:message "Let 1 events pass at most every 60 seconds"}, + :params [{:duration 60000000000, :count 1}], + :children + ({:action :output!, + :description + {:message "Forward events to the output :pagerduty"}, + :params [:pagerduty]})}), + :params [{:state "critical"}]})})}), + :params [{:ttl 60}]})})}}} diff --git a/dev/resources/tests/test1.edn b/dev/resources/tests/test1.edn index f022b3bc..735b9d1d 100644 --- a/dev/resources/tests/test1.edn +++ b/dev/resources/tests/test1.edn @@ -1,32 +1,20 @@ {:percentiles {:input [{:name "http_request_duration_seconds" - :metric 1.1 - :time 1e9 - :attributes {:application "app1" - :environment "production"}} + :metric 0.1 + :time 1e9} {:name "http_request_duration_seconds" - :metric 4.2 - :time 10e9 - :attributes {:application "app1" - :environment "production"}} + :metric 1.2 + :time 30e9} {:name "http_request_duration_seconds" - :metric 2 - :time 14e9 - :attributes {:application "app1" - :environment "production"}} + :metric 10 + :time 40e9} {:name "http_request_duration_seconds" - :metric 5 - :time 18e9 - :attributes {:application "app1" - :environment "production"}} + :metric 8 + :time 50e9} {:name "http_request_duration_seconds" - :metric 0.2 - :time 22e9 - :attributes {:application "app1" - :environment "production"}}] + :metric 3 + :time 70e9}] :taps {:alert [{:name "http_request_duration_seconds" - :metric 5 - :time 22e9 - :quantile "0.99" - :attributes {:application "app1" - :environment "production"} - :state "error"}]}}} + :metric 10 + :time 70e9 + :state "critical" + :attributes {:quantile "0.99"}}]}}} diff --git a/site/mirabelle/content/_index.md b/site/mirabelle/content/_index.md index 25cde782..e943428b 100644 --- a/site/mirabelle/content/_index.md +++ b/site/mirabelle/content/_index.md @@ -40,7 +40,7 @@ Let's say a web application is pushing the duration (in seconds) of the HTTP req ```clojure {:name "http_request_duration_seconds" - :time 1619731016,145 + :time 1619731016145000000 :tags ["web"] :metric 0.5 :attributes {:application "my-api" @@ -52,38 +52,45 @@ You could write a Mirabelle stream which will compute on the fly the quantiles f ```clojure (streams (stream {:name :percentiles :default true} - (where [:= :service "http_request_duration_seconds"] - (fixed-time-window {:duration 60} - (coll-percentiles [0.5 0.75 0.99] - (where [:and [:= :quantile "0.99"] - [:> :metric 1]] - (with :state "critical" - (tap :alert) - (output! :pagerduty)))))))) + (where [:= :name "http_request_duration_seconds"] + (percentiles {:duration 60 + :percentiles [0.5 0.75 0.99 1]} + (where [:and [:= [:attributes :quantile] "0.99"] + [:> :metric 1]] + (with :state "critical" + (tap :alert) + (output! :pagerduty))))))) ``` The `tap` action is an action which will be only enabled in test mode, and which will save in a tap named `:alert` events passing by it. Indeed, everything can be unit tested easily in Mirabelle. -A test for this stream would be: +A test for this stream would be (remember that the time is in nanoseconds): ```clojure -{:percentiles {:input [{:service "http_request_duration_seconds" +{:percentiles {:input [{:name "http_request_duration_seconds" :metric 0.1 - :time 1} - {:service "http_request_duration_seconds" + :time 1e9} + {:name "http_request_duration_seconds" :metric 1.2 - :time 30} - {:service "http_request_duration_seconds" - :metric 0.2 - :time 70}] - :tap-results {:alert [{:service "http_request_duration_seconds" - :metric 1.2 - :time 30 - :quantile "0.99" - :state "critical"}]}}} + :time 30e9} + {:name "http_request_duration_seconds" + :metric 10 + :time 40e9} + {:name "http_request_duration_seconds" + :metric 8 + :time 50e9} + {:name "http_request_duration_seconds" + :metric 3 + :time 70e9}] + :taps {:alert [{:name "http_request_duration_seconds" + :metric 10 + :time 70e9 + :state "critical" + :attributes {:quantile "0.99"}}]}}} + ``` -In this test, we inject into the `:percentiles` stream three events, and we verify that the tap named `:alert` contains the expected alert (the `0.99` quantile is greater than 1) generated for these events. +In this test, we inject into the `:percentiles` stream some events, and we verify that the tap named `:alert` contains the expected alert (the `0.99` quantile is greater than 1) generated for these events. Thanks to Clojure datastructures, there is *no side effects between streams and actions*. It's OK to modify events in parallel (in multiple threads) and to have multiple branches per stream. You can even pass events between streams (like described [here](todo)). You are free to organize streams and how they communicate between each other exactly how you want to; the tool and its DSL will not limit you. @@ -92,12 +99,10 @@ Here is a more complete and commented example, with multiple actions performed i ```clojure (streams (stream {:name :multiple-branches} - (where [:= :service "http_request_duration_seconds"] + (where [:= :name "http_request_duration_seconds"] (with :ttl 60 ;; push everything into influxdb (output! :influxdb) - ;; index events in memory by host and service - (index [:host :service]) ;; by will generate a branch for each :host value. Like that, downstream ;; computations will be per host and will not conflict between each other (by {:fields [:host]} @@ -111,3 +116,4 @@ Here is a more complete and commented example, with multiple actions performed i (output! :pagerduty))))))))) ``` +Raw events or computation results could also be forwarded to external systems (TSDB for example) for long-term storage. diff --git a/site/mirabelle/content/howto/build/_index.md b/site/mirabelle/content/howto/build/_index.md index 72a78020..2a6a5443 100644 --- a/site/mirabelle/content/howto/build/_index.md +++ b/site/mirabelle/content/howto/build/_index.md @@ -11,7 +11,7 @@ In order to build Mirabelle, you need to install: - [Leiningen](https://leiningen.org/), the Clojure build tool. - Java. Mirabelle is tested under Java 17 (LTS). -Then, clone the Mirabelle [Git repository](https://github.com/mcorbin/mirabelle). You can now build the project with `lein uberjar`. +Then, clone the Mirabelle [Git repository](https://github.com/appclacks/mirabelle). You can now build the project with `lein uberjar`. The resulting jar will be in `target/uberjar/mirabelle--standalone.jar` diff --git a/site/mirabelle/content/howto/pubsub/_index.md b/site/mirabelle/content/howto/pubsub/_index.md index 39f93074..8804a48e 100644 --- a/site/mirabelle/content/howto/pubsub/_index.md +++ b/site/mirabelle/content/howto/pubsub/_index.md @@ -4,36 +4,25 @@ weight: 14 disableToc: false --- -The Mirabelle Websocket server allows user to subscribe to the streams [index](/howto/stream-index/) or to channels. +The Mirabelle Websocket server allows user to subscribe to channels. ## An example ```clojure (streams - (stream - {:name :bar :default true} + (stream {:name :bar :default true} (where [:= :service "bar"] - (index [:host]) (publish! :my-channel)))) ``` -In this stream, we filter all events with `:service` "bar". Then, we index them, and we also call `publish!` to publish the event to a channel named `:my-channel`. +In this stream, we filter all events with `:service` "bar", then we publish them to a channel named `:my-channel`. -A channel is created by default for each index (the channel name being `-index`). -You can also subscribe for events indexed into indexes belonging to streams tagged `:default` by subscribing to the `default-index` channel. - -Users can also subscribe to other channels, not related to indexes. It's what `publish!` does. In this example, all users subscribing to the `:my-channel` channel will receive the events. +Users then subscripe to those channels. In this example, all users subscribing to the `:my-channel` channel will receive the events. You can test that yourself by running the `websocket.py` script available [here](https://github.com/mcorbin/mirabelle/tree/master/pubsub). You will need Python 3, and to install the dependencies listed in `requirements.txt` using `pip install -r requirements.txt`. When you subscribe to a channel, you should provide a valid [where clause](/howto/stream/#filtering-events) in base64. For example, the query `[:always-true]` which matches everything would be `WzphbHdheXMtdHJ1ZV0=`. -You can now run the script with `./websocket.py`. By default, it subscribes to the default index. You can specify a channel with `--channel` (like `--channel my-channel` here), or a query with `--query` (the query will be automatically converted to base64 by the script). +You can now run the script with `./websocket.py`. You can specify a channel with `--channel` (like `--channel my-channel` here), or a query with `--query` (the query will be automatically converted to base64 by the script). You can also write your own scripts in various languages. You need to subscribe to `ws://:/channel/?query=`. - -### Riemann compatibility - -In order to be compatible with Riemann clients, you can also subscribe on `/index` (to subscribe to the default index), or `/index?stream=` to subscribe to an index for a dynamic stream. - - diff --git a/site/mirabelle/content/howto/stream-index/_index.md b/site/mirabelle/content/howto/stream-index/_index.md deleted file mode 100644 index 7c61ab59..00000000 --- a/site/mirabelle/content/howto/stream-index/_index.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -title: The index -weight: 12 -disableToc: false ---- - -The Mirabelle Index is an in memory map which can be used to store events. - -Each Mirabelle stream (generated from the configuration file or from the API) has its own, dedicated index instance. - -## How to push events into the index - -```clojure -(stream - {:name :bar :default true} - (where [:= :service "bar"] - (index [:host :service]))) -``` - -In this example, all events with `:service` "bar" will be added to the index. They will be indexed by their `:host` and `:service` key. - -For example, if these two events `{:host "foo" :service "bar" :time 1 :ttl 60}` and `{:host "mirabelle" :service "bar" :time 1 :ttl 60}` arrive in the system, the index would contain two keys: - -`{:host "foo" :service "bar"}` -> `{:host "foo" :service "bar" :time 1 :ttl 60}` -`{:host "mirabelle" :service "bar"}` -> `{:host "mirabelle" :service "bar" :time 1 :ttl 60}` - -If a new event arrives for an already existing key, the old key is overrided. - -You can have multiple calls to `index` in Mirabelle, and index events based on the fields you want. - -## Expiration - -In Mirabelle, events have a `:time` (when the event was generated) and optionally a TTL. The default TTL for events in the index is `120` (in seconds). You could also add another default TTL using the `with` stream. - -For example, an event with `:time` 1 and `:ttl` 120 would be expired at time 122. - -Expiration is important. First, a lot of streams which work on windows of events will remove expired events automatically. Indeed, if you use for example [coalesce](/howto/stream/#coalesce-and-project), you don't want to keep forever the latest event for a host which does not emit anymore. When the event is expired, it's removed from the action. - -Expiration can also be used to detect services which stopped emitting. - -Indeed, if an event stored in the index is expired, it will be reinjected into the stream with `:state "expired"`. You could then catch it with something like: - -```clojure -(expired - (error)) -``` - -By forwarding expired events into dedicated actions, you can for example trigger alerts based on that. - -In order to trigger event expiration from events stored into the index, we should use the `reaper` action. All streams in Riemann are using the events clocks to trigger side effects, and the reaper action will update the index clock if needed, trigger events expiration, and then reinject the events into streams: - -```clojure -(reaper 5) -(reaper 5 :custom-stream) -``` - -The `reaper` action takes as first parameter the interval (in seconds) on which the reaper stream will expire events. Don't forget you should feed the reaper action with events in order to move its clock. - -The `custom-stream` parameter is optional. By default, events are reinjected into the current stream. You can also pass a stream name to the reaper action in order to reinject events into another stream. - -I recommand you to not call forward all events to the reaper (it will impact performances). Instead, identify a couple of events arriving regularly, and forward them to the reaper in order to update the index clock. - -## Queries - -You can query the events stored into the index. The query should be a valid [where clause](/howto/stream/#filtering-events) in base64. For example `[:and [:> :metric 10] [:= :service "bar"]]` would be `WzphbmQgWzo+IDptZXRyaWMgMTBdIFs6PSA6c2VydmljZSAiYmFyIl1d`. - -You can send queries using the Mirabelle (or Riemann) TCP clients, or using the [HTTP API](http://localhost:1313/api/#query-the-index). - -## Pub Sub - -Users can subscribe to the index using Websocket, in order to see in real time which events are indexed. Check the [pubsub documentation](howto/pubsub/) for more information. - - - diff --git a/site/mirabelle/content/howto/stream/_index.md b/site/mirabelle/content/howto/stream/_index.md index f9c0ba05..26e087a2 100644 --- a/site/mirabelle/content/howto/stream/_index.md +++ b/site/mirabelle/content/howto/stream/_index.md @@ -39,17 +39,19 @@ Mirabelle ships with a complete, extensible DSL to define streams. The DSL is he ### Events -Events are represented as an immutable map. An event has standard fields. All fields are optional. +Events are represented as an immutable map. An event has standard fields. All fields are optional and you can also add arbitrary ones, the schema is fully free. - `:host`: the event source. It can be an hostname for example. -- `:service`: What is measured. `http_requests_duration_seconds` for example. +- `:service`: the name of the service emitting the event. +- `:name`: What is measured. `http_requests_duration_seconds` for example. - `:state`: A string representing the event state. By convention, `ok`, `warning`, `critical` or `expired` are often used. - `:metric`: A number associated to the event (the value of what is measured). -- `:time`: The event time in second, as a timestamp (`1619988803` for example). It could also be a float (`1619988803,173413` for example), the Mirabelle/Riemann protocol supports microsecond resolution. +- `:time`: The event time in nanoseconds, as a timestamp (`1735047872000000000` for example). - `:description`: The event description. -- `:ttl`: The duration that the event is considered valid. See the [Index](/index) documentation for more information about the index and events expiration. - `:tags`: A list of tags associated to the event (like `["foo" "bar"]` for example). -- Extra fields can also be added if you want to. One important extra field is `:stream`. It can be used to specify on which stream the event should be send. By default, events are sent to all streags with `:default` in their configurations. +- `:attributes`: a list of arbitrary ke-value attributes. + +Extra fields can also be added if you want to. One important extra field is `:stream`. It can be used to specify on which stream the event should be send. By default, events are sent to all streams with `:default` set to `true` in their configurations. ### Streams @@ -73,7 +75,7 @@ Let's now define another stream: (stream {:name :log :default true} (info)) (stream {:name :http_requests_duration} - (where [:= :service "http_requests_duration_seconds"] + (where [:= :name "http_requests_duration_seconds"] (info) (over 1.5 (with :state "critical" @@ -82,7 +84,7 @@ Let's now define another stream: In this second example, we still have our first stream named `:log`. We also have another stream, a bit more complex, named `:http_requests_duration`. -This second stream will first keep only events with services equal to "http_requests_duration_seconds" using the `where` action. +This second stream will first keep only events with `:name` equal to "http_requests_duration_seconds" using the `where` action. Then, it will log (using `info`) the events. In another branch, `over` is used tp keep only events with `:metric` greater than `1.5` (we can imagine that @@ -102,7 +104,7 @@ The Mirabelle DSL should first be compiled to an EDN datastructure before being ```clojure (streams (stream {:name :http_requests_duration} - (where [:= :service "http_requests_duration_seconds"] + (where [:= :name "http_requests_duration_seconds"] (info) (over 1.5 (with :state "critical" @@ -115,7 +117,7 @@ You then need to compile this file using this command: java -jar mirabelle.jar compile ``` -For example, let's say you have put the previous stream in a file named `stream.clj` in the `/tmp/streams` directory. +For example, let's say you have put the previous stream in a file named `stream.clj` in the `/tmp/streams` directory. If ou launch `java -jar mirabelle.jar compile /tmp/streams /tmp/compiled`, your file will be compiled and a new `stream.clj` file will be created in the destination directory (which is `/tmp/compiled` here). Let's do that. @@ -132,7 +134,7 @@ The resulting file in `/tmp/compiled/stream.clj` should be: {:action :sdo, :children ({:action :where, - :params [[:= :service "http_requests_duration_seconds"]], + :params [[:= :name "http_requests_duration_seconds"]], :children ({:action :info} {:action :over, @@ -153,14 +155,16 @@ How to launch Mirabelle is explained in [this section](/howto/build/). Once Mirabelle started, you can send events to it. For that, you can check the [integration](/integration/) documentation section for the available clients (Riemann clients are fully compatible with Mirabelle). In this example, I will use the [Riemann C client](https://github.com/algernon/riemann-c-client) which provides a CLI and is available in many Linux package managers. +TODO: replace with Appclacks CLI in the whole example + ``` -riemann-client send --metric-f 1 --service "http_requests_duration_seconds" --host=my-host +riemann-client send --metric-f 1 --name "http_requests_duration_seconds" --host=my-host ``` If I send the previous event, I should see in Mirabelle logs: ```json -{"@timestamp":"2021-05-01T22:48:58.786+02:00","@version":"1","message":"#riemann.codec.Event{:host \"my-host\", :service \"http_requests_duration_seconds\", :state nil, :description nil, :metric 1.0, :tags nil, :time 1.619902138786E9, :ttl nil, :x-client \"riemann-c-client\"}","logger_name":"mirabelle.action","thread_name":"defaultEventExecutorGroup-2-8","level":"INFO","level_value":20000} +{"@timestamp":"2021-05-01T22:48:58.786+02:00","@version":"1","message":"#riemann.codec.Event{:host \"my-host\", :name \"http_requests_duration_seconds\", :state nil, :description nil, :metric 1.0, :tags nil, :time 1.619902138786E9, :ttl nil, :x-client \"riemann-c-client\"}","logger_name":"mirabelle.action","thread_name":"defaultEventExecutorGroup-2-8","level":"INFO","level_value":20000} ``` My event was indeed logging by the `info` action in my stream. Let's send an event with the metric greater than our threshold: @@ -197,7 +201,7 @@ You can set the `PROFILE` environment variable in order to use Aero [profiles](h (streams (stream {:name :foo :default true} (where [:and - [:= :service "disk-used"] + [:= :name "disk-used"] [:> :metric #profile {:preprod 70 :prod 60 :default 90}]] @@ -212,10 +216,10 @@ You can also use other Aero build-in readers described in the Aero [readme](http **Include** -It's possible to include a configuration file in another one. Let's take this file named for example `log-service.clj`: +It's possible to include a configuration file in another one. Let's take this file named for example `log-name.clj`: ```clojure -(where [:= :service #mirabelle/var :my-service] +(where [:= :name #mirabelle/var :my-name] (info)) ``` @@ -224,16 +228,16 @@ You can then use this file using `include` in a Mirabelle stream: ```clojure (streams (stream {:name :foo :default true} - (include "log-service.clj" {:variables {:my-service "disk-used"}}) - (include "log-service.clj" {:variables {:my-service "ram-used"}}))) + (include "log-name.clj" {:variables {:my-name "disk-used"}}) + (include "log-name.clj" {:variables {:my-name "ram-used"}}))) ``` -The `#mirabelle/var` reader allows you to read a variable passed to the `include` action (here, the variable is named `:my-service`). +The `#mirabelle/var` reader allows you to read a variable passed to the `include` action (here, the variable is named `:my-name`). You can also override the default Mirabelle profile (passed as an environment variable) by passing the `;profile` key to the `include` options: ```clojure -(include "log-service.clj" {:variables {:my-service "disk-used"} +(include "log-name.clj" {:variables {:my-name "disk-used"} :profile :dev}) ``` @@ -266,6 +270,8 @@ You can now use this output named `:pagerduty-client` in a stream by using the ` (output! :pagerduty-client))) ``` +TODO review example + If this event is set to Mirabelle: ```shell @@ -368,7 +374,7 @@ For example, the `above-dt` stream will only let events pass if all events recei In this example, `above-dt` will let events pass (to log them as error) only if it receives events with `:metric` greater than 1 during more than 6O seconds. -The streams `below-dt`, `between-dt`, `outside-dt`, `critical-dt` also work that way. They are useful to avoid alerting on spikes for examples. +The streams `below-dt`, `between-dt`, `outside-dt`, `cond-dt` also work that way. They are useful to avoid alerting on spikes for examples. The `tagged-all` stream is also available to keep only events containing one tag or a set of tags: `(tagged-all "foo")` or `(tagged-all ["foo" "bar"])`. @@ -404,12 +410,12 @@ You can use `rename-keys` to rename some events keys: :environment :env}) ``` -In this example, the `:host` key will be renamed `:service` and the `:environment` key is renamed `:env`. Existing values will be overrided. +In this example, the `:host` key will be renamed `:service` and the `:environment` key is renamed `:env`. Existing values will be overrided. Nested keys are also supported, by passing a list of fields as keys or attributes (for example `[:attributes :foo]`. If you want to keep only some keys from an event (and so remove all the others), you can use `keep-keys`: ```clojure -(keep-keys [:host :service :time :metric :description :environment]) +(keep-keys [:host :service :time :metric :description [:nested :field]]) ``` Some actions can modify the `:metric` field. `increment` and `decrement` will add +1 or -1 to it, and you can use `scale` to multiply it with a value: `(scale 1000)` for example. @@ -430,11 +436,10 @@ The `percentiles` action allows you to compute percentiles (quantiles) on events ```clojure (percentiles {:percentiles [0.5 0.75 0.99] - :duration 10 - :nb-significant-digits 3} + :duration 10} ``` -This example will compute the 0.5, 0.75 and 0.99 quantiles every 10 seconds. It also supports the `:delay` parameter (to tolerate events arriving late), `:highest-trackable-value` and `:lowest-discernible-value` to bound results. +This example will compute the 0.5, 0.75 and 0.99 quantiles every 10 seconds. It also supports the `:delay` parameter (to tolerate events arriving late), `:highest-trackable-value` and `:lowest-discernible-value` to bound results, and `:nb-significant-digits` for precision (default to 3). The implementation uses the [HdrHistogram](https://github.com/HdrHistogram/HdrHistogram) library. @@ -448,7 +453,6 @@ You can compute the rate of incoming events (by counting them) using the `rate` This action will send the rate of events downstream every 20 seconds. You can also add a `:delay` parameter to tolerate late events. - #### Sum events for a time period The `:sum` stream can be used to sum events `:metrics` field for a period of time: @@ -688,6 +692,15 @@ We already saw in the [Action on lists of events: max, min, count, percentiles, `smax` and `smin` will send *for each event they receive* the maximum or minimum event. The value is never resetted. +#### Convert a value to string + +The `to-string` stream can be used to convert values associated to some keys to string: + +```clojure +(to-string [:service :state [:nested :attributes] + (info)) +``` + #### Format a string based on values If you need to create a value from others keys values, use `sformat`: diff --git a/site/mirabelle/content/howto/tests/_index.md b/site/mirabelle/content/howto/tests/_index.md index 3c524978..64b30b18 100644 --- a/site/mirabelle/content/howto/tests/_index.md +++ b/site/mirabelle/content/howto/tests/_index.md @@ -40,10 +40,10 @@ Let's write a test file for these streams. The tests file should be referenced i {:metric 101 :service "bar"}] - :tap-results {:foo [{:metric 10 - :service "foo"}] + :taps {:foo [{:metric 10 + :service "foo"}] :bar [{:metric 101 - :service "bar"}]}}} + :service "bar"}]}}} ``` We defined here a test named `:test1`. You could have add more tests to the map (or in another files), all tests are run in isolation. @@ -51,7 +51,7 @@ We defined here a test named `:test1`. You could have add more tests to the map This `:test1` key contains a map with two keys: - `:input`: the events which will be injected into the streams -- `:tap-results`: the expected contents of the tap. +- `:taps`: the expected contents of the tap. Here, we see that the tap `:foo` should contain the first event (with `:service` "foo"), and the tap `:bar` the second (with `:metric` greater than 100). @@ -82,14 +82,12 @@ Tests can also take a `:target` configuration, for example: {:metric 101 :service "bar"}] :target :foo - :tap-results {:foo [{:metric 10 - :service "foo"}]}}} + :taps {:foo [{:metric 10 + :service "foo"}]}}} ``` In this example, events in `:input` will only be injected into the `:foo` stream. -One dedicated [index](/howto/stream-index/) is created for each test. - ## Things excluded from tests In test mode, some streams behave differently than in the regular mode: diff --git a/site/mirabelle/public/404.html b/site/mirabelle/public/404.html index cd0c3057..d6a61405 100644 --- a/site/mirabelle/public/404.html +++ b/site/mirabelle/public/404.html @@ -9,13 +9,13 @@ 404 Page not found - - - - - - - + + + + + + + - - - - - - - - -
-
-
- -
-
- - - - -
-
- -
-
- - -
-
- -
- -
- -
- -

- - The index -

- - - - - - -

The Mirabelle Index is an in memory map which can be used to store events.

-

Each Mirabelle stream (generated from the configuration file or from the API) has its own, dedicated index instance.

-

How to push events into the index

-
(stream
-  {:name :bar :default true}
-  (where [:= :service "bar"]
-    (index [:host :service])))
-

In this example, all events with :service “bar” will be added to the index. They will be indexed by their :host and :service key.

-

For example, if these two events {:host "foo" :service "bar" :time 1 :ttl 60} and {:host "mirabelle" :service "bar" :time 1 :ttl 60} arrive in the system, the index would contain two keys:

-

{:host "foo" :service "bar"} -> {:host "foo" :service "bar" :time 1 :ttl 60} -{:host "mirabelle" :service "bar"} -> {:host "mirabelle" :service "bar" :time 1 :ttl 60}

-

If a new event arrives for an already existing key, the old key is overrided.

-

You can have multiple calls to index in Mirabelle, and index events based on the fields you want.

-

Expiration

-

In Mirabelle, events have a :time (when the event was generated) and optionally a TTL. The default TTL for events in the index is 120 (in seconds). You could also add another default TTL using the with stream.

-

For example, an event with :time 1 and :ttl 120 would be expired at time 122.

-

Expiration is important. First, a lot of streams which work on windows of events will remove expired events automatically. Indeed, if you use for example coalesce, you don’t want to keep forever the latest event for a host which does not emit anymore. When the event is expired, it’s removed from the action.

-

Expiration can also be used to detect services which stopped emitting.

-

Indeed, if an event stored in the index is expired, it will be reinjected into the stream with :state "expired". You could then catch it with something like:

-
(expired
-  (error))
-

By forwarding expired events into dedicated actions, you can for example trigger alerts based on that.

-

In order to trigger event expiration from events stored into the index, we should use the reaper action. All streams in Riemann are using the events clocks to trigger side effects, and the reaper action will update the index clock if needed, trigger events expiration, and then reinject the events into streams:

-
(reaper)
-(reaper 5 :custom-stream)
-

The reaper action takes as first parameter the interval (in seconds) on which the reaper stream will expire events. Don’t forget you should feed the reaper action with events in order to move its clock.

-

The custom-stream parameter is optional. By default, events are reinjected into the current stream. You can also pass a stream name to the reaper action in order to reinject events into another stream.

-

I recommand you to not call forward all events to the reaper (it will impact performances). Instead, identify a couple of events arriving regularly, and forward them to the reaper in order to update the index clock.

-

Queries

-

You can query the events stored into the index. The query should be a valid where clause in base64. For example [:and [:> :metric 10] [:= :service "bar"]] would be WzphbmQgWzo+IDptZXRyaWMgMTBdIFs6PSA6c2VydmljZSAiYmFyIl1d.

-

You can send queries using the Mirabelle (or Riemann) TCP clients, or using the HTTP API.

-

Pub Sub

-

Users can subscribe to the index using Websocket, in order to see in real time which events are indexed. Check the pubsub documentation for more information.

- - - - - -
- -
- - -
- - -
- - - -
- -
-
-
- - - - - - - - - - - - - - - - - - - - diff --git a/site/mirabelle/public/howto/index/index.xml b/site/mirabelle/public/howto/index/index.xml deleted file mode 100644 index 45dd9d8d..00000000 --- a/site/mirabelle/public/howto/index/index.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - The index on Mirabelle - https://www.mirabelle.mcorbin.fr/howto/index/ - Recent content in The index on Mirabelle - Hugo -- gohugo.io - en-us - - diff --git a/site/mirabelle/public/howto/on-disk-queue/index.html b/site/mirabelle/public/howto/on-disk-queue/index.html deleted file mode 100644 index c1caf741..00000000 --- a/site/mirabelle/public/howto/on-disk-queue/index.html +++ /dev/null @@ -1,962 +0,0 @@ - - - - - - - - - - - - On Disk queue :: Mirabelle - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- -
-
- - - - -
-
- -
-
- - -
-
- -
- -
- -
- -

- - On Disk queue -

- - - - - - -

Warning: I’m not totally satisfied with this feature and with the current implementation. It will change in the future. Please provide me feedbacks ;)

-

Mirabelle allows you in its configuration file to optionally persist some events into a queue written on disk. The queue implementation is a Chronicle Queue.

-

You can configure the queue roll cycle in the Mirabelle configuration. The default is :half-hourly. More information about roll cycles can be found here.

-

Note: Mirabelle will not rotate/delete old files for you. You need to do it yourself.

-
(where [:= :service "foo"]
-  (disk-queue!))
-

In this example, Mirabelle will write all events with :service “foo” on disk.

-

Why is it useful ? You can use the queue for:

-
    -
  • Archive events. You could have a cronjob taking all events of the day and putting them into S3 for example. You could then reuse these events for example to reinject them later into Mirabelle.
    • -
  • Rebuilding states. Mirabelle streams will not keep their states if Mirabelle crashes. But when Mirabelle starts, all events from the ondisk queue are reinjected in the order they are in the queue into Mirabelle.
    • -
-

The reinjected events will be tagged “discard”. All stateful actions (I/O, pubsub, logger) will not be executed for discarded events. It means the events will rebuild all streams internal states but without producing side effects.

-

It’s nice, but it’s not perfect. The two big limitations today are:

-
    -
  • Events will only be reinjected into the :default streams. So be careful about that, the queue will not work with other streams. I will work on that.
    • -
  • It can be slow. What if you have 10 millions events in the queue ? Even if you process them at 100 000 events per seconds, it will take 100 seconds before Mirabelle would be fully started. Do you really want to wait minutes for your service to start ?
    • -
-

This feature will probably change in the future, i’m thinking hard about how to rebuild streams states quickly and without sacrificing streams performances. Ideas are welcomed.

- - - - - -
- -
- - -
- - -
- - - -
- -
-
-
- - - - - - - - - - - - - - - - - - - - diff --git a/site/mirabelle/public/howto/on-disk-queue/index.xml b/site/mirabelle/public/howto/on-disk-queue/index.xml deleted file mode 100644 index 61bf6267..00000000 --- a/site/mirabelle/public/howto/on-disk-queue/index.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - On Disk queue on Mirabelle - https://www.mirabelle.mcorbin.fr/howto/on-disk-queue/ - Recent content in On Disk queue on Mirabelle - Hugo -- gohugo.io - en-us - - diff --git a/site/mirabelle/public/howto/pubsub/index.html b/site/mirabelle/public/howto/pubsub/index.html index 934bb1c3..d38646a1 100644 --- a/site/mirabelle/public/howto/pubsub/index.html +++ b/site/mirabelle/public/howto/pubsub/index.html @@ -12,20 +12,20 @@ Publish Subscribe :: Mirabelle - - - - - - - - - - + + + + + + + + + + - + - - - - - - - - -
-
-
- -
-
- - - - -
-
- -
-
- - -
-
- -
- -
- -
- -

- - The index -

- - - - - - -

The Mirabelle Index is an in memory map which can be used to store events.

-

Each Mirabelle stream (generated from the configuration file or from the API) has its own, dedicated index instance.

-

How to push events into the index

-
(stream
-  {:name :bar :default true}
-  (where [:= :service "bar"]
-    (index [:host :service])))
-

In this example, all events with :service “bar” will be added to the index. They will be indexed by their :host and :service key.

-

For example, if these two events {:host "foo" :service "bar" :time 1 :ttl 60} and {:host "mirabelle" :service "bar" :time 1 :ttl 60} arrive in the system, the index would contain two keys:

-

{:host "foo" :service "bar"} -> {:host "foo" :service "bar" :time 1 :ttl 60} -{:host "mirabelle" :service "bar"} -> {:host "mirabelle" :service "bar" :time 1 :ttl 60}

-

If a new event arrives for an already existing key, the old key is overrided.

-

You can have multiple calls to index in Mirabelle, and index events based on the fields you want.

-

Expiration

-

In Mirabelle, events have a :time (when the event was generated) and optionally a TTL. The default TTL for events in the index is 120 (in seconds). You could also add another default TTL using the with stream.

-

For example, an event with :time 1 and :ttl 120 would be expired at time 122.

-

Expiration is important. First, a lot of streams which work on windows of events will remove expired events automatically. Indeed, if you use for example coalesce, you don’t want to keep forever the latest event for a host which does not emit anymore. When the event is expired, it’s removed from the action.

-

Expiration can also be used to detect services which stopped emitting.

-

Indeed, if an event stored in the index is expired, it will be reinjected into the stream with :state "expired". You could then catch it with something like:

-
(expired
-  (error))
-

By forwarding expired events into dedicated actions, you can for example trigger alerts based on that.

-

In order to trigger event expiration from events stored into the index, we should use the reaper action. All streams in Riemann are using the events clocks to trigger side effects, and the reaper action will update the index clock if needed, trigger events expiration, and then reinject the events into streams:

-
(reaper 5)
-(reaper 5 :custom-stream)
-

The reaper action takes as first parameter the interval (in seconds) on which the reaper stream will expire events. Don’t forget you should feed the reaper action with events in order to move its clock.

-

The custom-stream parameter is optional. By default, events are reinjected into the current stream. You can also pass a stream name to the reaper action in order to reinject events into another stream.

-

I recommand you to not call forward all events to the reaper (it will impact performances). Instead, identify a couple of events arriving regularly, and forward them to the reaper in order to update the index clock.

-

Queries

-

You can query the events stored into the index. The query should be a valid where clause in base64. For example [:and [:> :metric 10] [:= :service "bar"]] would be WzphbmQgWzo+IDptZXRyaWMgMTBdIFs6PSA6c2VydmljZSAiYmFyIl1d.

-

You can send queries using the Mirabelle (or Riemann) TCP clients, or using the HTTP API.

-

Pub Sub

-

Users can subscribe to the index using Websocket, in order to see in real time which events are indexed. Check the pubsub documentation for more information.

- - - - - -
- -
- - -
- - -
- - - -
- -
-
-
- - - - - - - - - - - - - - - - - - - - diff --git a/site/mirabelle/public/howto/stream-index/index.xml b/site/mirabelle/public/howto/stream-index/index.xml deleted file mode 100644 index 3dd93985..00000000 --- a/site/mirabelle/public/howto/stream-index/index.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - The index on Mirabelle - https://www.mirabelle.mcorbin.fr/howto/stream-index/ - Recent content in The index on Mirabelle - Hugo -- gohugo.io - en-us - - - diff --git a/site/mirabelle/public/howto/stream/index.html b/site/mirabelle/public/howto/stream/index.html index b61f80c4..a0f02c90 100644 --- a/site/mirabelle/public/howto/stream/index.html +++ b/site/mirabelle/public/howto/stream/index.html @@ -12,20 +12,20 @@ Writing streams :: Mirabelle - - - - - - - - - - + + + + + + + + + + - +