diff --git a/dev/resources/dsl/stream.clj b/dev/resources/dsl/stream.clj index 2739a44a..095408e9 100644 --- a/dev/resources/dsl/stream.clj +++ b/dev/resources/dsl/stream.clj @@ -1,31 +1,4 @@ -;; (streams -;; ;; (stream {:name :traces :default true} -;; ;; (where [:= :kind "client"] -;; ;; (rate {:duration 10} -;; ;; (with {:name "RATE"} -;; ;; (info))))) -;; ;; (stream {:name :test :default false} -;; ;; (where [:= :kind "client"] -;; ;; (rate {:duration 10} -;; ;; (with {:name "RATE"} -;; ;; (tap :result) -;; ;; (info))))) - -;; (stream {:name :percentiles }) -;; ) - (streams - (stream {:name :percentiles :default true} - (where [:= :name "http_request_duration_seconds"] - - (by {:fields [[:attributes :application] - [:attributes :environment]]} - (percentiles {:percentiles [0.5 0.75 0.99] - :duration 20 - :nb-significant-digits 3} - (info) - (where [:and [:= :quantile "0.99"] - [:> :metric 1]] - (with {:state "error"} - (error) - (tap :alert)))))))) + (stream {:name :bar :default true} + (where [:= :service "bar"] + (publish! :my-channel)))) diff --git a/dev/resources/streams/stream.clj b/dev/resources/streams/stream.clj index 1e35d49c..757f598e 100644 --- a/dev/resources/streams/stream.clj +++ b/dev/resources/streams/stream.clj @@ -1,4 +1,4 @@ -{:percentiles +{:bar {:default true, :actions {:action :sdo, @@ -7,44 +7,11 @@ ({:action :where, :description {:message "Filter events based on the provided condition", - :params "[:= :name \"http_request_duration_seconds\"]"}, - :params [[:= :name "http_request_duration_seconds"]], + :params "[:= :service \"bar\"]"}, + :params [[:= :service "bar"]], :children - ({:action :by, + ({:action :publish!, :description - {:message - "Split streams by field(s) [[:attributes :application] [:attributes :environment]]"}, - :params - [{:fields - [[:attributes :application] [:attributes :environment]]}], - :children - ({:action :percentiles, - :description - {:message "Computes the quantiles [0.5 0.75 0.99]"}, - :params - [{:percentiles [0.5 0.75 0.99], - :duration 20000000000, - :nb-significant-digits 3}], - :children - ({:action :info, - :description - {:message "Print the event in the logs as info"}} - {:action :where, - :description - {:message "Filter events based on the provided condition", - :params "[:and [:= :quantile \"0.99\"] [:> :metric 1]]"}, - :params [[:and [:= :quantile "0.99"] [:> :metric 1]]], - :children - ({:action :with, - :description - {:message "Merge the events with the provided fields", - :params "{:state \"error\"}"}, - :children - ({:action :error, - :description - {:message "Print the event in the logs as error"}} - {:action :tap, - :description - {:message "Save events into the tap :alert"}, - :params [:alert]}), - :params [{:state "error"}]})})})})})}}} + {:message "Publish events into the channel :my-channel"}, + :params [:my-channel], + :children []})})}}} diff --git a/dev/resources/tests/test1.edn b/dev/resources/tests/test1.edn index f022b3bc..735b9d1d 100644 --- a/dev/resources/tests/test1.edn +++ b/dev/resources/tests/test1.edn @@ -1,32 +1,20 @@ {:percentiles {:input [{:name "http_request_duration_seconds" - :metric 1.1 - :time 1e9 - :attributes {:application "app1" - :environment "production"}} + :metric 0.1 + :time 1e9} {:name "http_request_duration_seconds" - :metric 4.2 - :time 10e9 - :attributes {:application "app1" - :environment "production"}} + :metric 1.2 + :time 30e9} {:name "http_request_duration_seconds" - :metric 2 - :time 14e9 - :attributes {:application "app1" - :environment "production"}} + :metric 10 + :time 40e9} {:name "http_request_duration_seconds" - :metric 5 - :time 18e9 - :attributes {:application "app1" - :environment "production"}} + :metric 8 + :time 50e9} {:name "http_request_duration_seconds" - :metric 0.2 - :time 22e9 - :attributes {:application "app1" - :environment "production"}}] + :metric 3 + :time 70e9}] :taps {:alert [{:name "http_request_duration_seconds" - :metric 5 - :time 22e9 - :quantile "0.99" - :attributes {:application "app1" - :environment "production"} - :state "error"}]}}} + :metric 10 + :time 70e9 + :state "critical" + :attributes {:quantile "0.99"}}]}}} diff --git a/project.clj b/project.clj index b4ac0789..98d0772f 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject fr.mcorbin/mirabelle "0.13.0" +(defproject fr.mcorbin/mirabelle "0.14.0" :description "A stream processing engine inspired by Riemann" :url "https://github.com/mcorbin/mirabelle" :license {:name "EPL-1.0" diff --git a/site/mirabelle/content/_index.md b/site/mirabelle/content/_index.md index 25cde782..8f1ee667 100644 --- a/site/mirabelle/content/_index.md +++ b/site/mirabelle/content/_index.md @@ -16,9 +16,7 @@ Mirabelle isinspired by [Riemann](https://riemann.io/). I would like to thank al Mirabelle supports the same protocol than Riemann. It means all Riemann tooling and integrations should work seamlessly with Mirabelle (which also contains a lot of new features). -Mirabelle also provides an HTTP API and natively supports receiving metrics in [Prometheus remote write](https://prometheus.io/docs/operating/integrations/) format. See the [API documentation](/api/#prometheus-remote-write) for more information about Prometheus integration. - -It also supports [Opentelemetry traces](/api/#opentelemetry-traces-input) as input. +Mirabelle also provides an HTTP API and natively supports receiving metrics in [Prometheus remote write](https://prometheus.io/docs/operating/integrations/). It also supports [Opentelemetry traces](/api/#opentelemetry-traces-input) as input. ## Streams clocks, real time, continuous queries @@ -40,7 +38,7 @@ Let's say a web application is pushing the duration (in seconds) of the HTTP req ```clojure {:name "http_request_duration_seconds" - :time 1619731016,145 + :time 1619731016145000000 :tags ["web"] :metric 0.5 :attributes {:application "my-api" @@ -52,38 +50,45 @@ You could write a Mirabelle stream which will compute on the fly the quantiles f ```clojure (streams (stream {:name :percentiles :default true} - (where [:= :service "http_request_duration_seconds"] - (fixed-time-window {:duration 60} - (coll-percentiles [0.5 0.75 0.99] - (where [:and [:= :quantile "0.99"] - [:> :metric 1]] - (with :state "critical" - (tap :alert) - (output! :pagerduty)))))))) + (where [:= :name "http_request_duration_seconds"] + (percentiles {:duration 60 + :percentiles [0.5 0.75 0.99 1]} + (where [:and [:= [:attributes :quantile] "0.99"] + [:> :metric 1]] + (with :state "critical" + (tap :alert) + (output! :pagerduty))))))) ``` The `tap` action is an action which will be only enabled in test mode, and which will save in a tap named `:alert` events passing by it. Indeed, everything can be unit tested easily in Mirabelle. -A test for this stream would be: +A test for this stream would be (remember that the time is in nanoseconds): ```clojure -{:percentiles {:input [{:service "http_request_duration_seconds" +{:percentiles {:input [{:name "http_request_duration_seconds" :metric 0.1 - :time 1} - {:service "http_request_duration_seconds" + :time 1e9} + {:name "http_request_duration_seconds" :metric 1.2 - :time 30} - {:service "http_request_duration_seconds" - :metric 0.2 - :time 70}] - :tap-results {:alert [{:service "http_request_duration_seconds" - :metric 1.2 - :time 30 - :quantile "0.99" - :state "critical"}]}}} + :time 30e9} + {:name "http_request_duration_seconds" + :metric 10 + :time 40e9} + {:name "http_request_duration_seconds" + :metric 8 + :time 50e9} + {:name "http_request_duration_seconds" + :metric 3 + :time 70e9}] + :taps {:alert [{:name "http_request_duration_seconds" + :metric 10 + :time 70e9 + :state "critical" + :attributes {:quantile "0.99"}}]}}} + ``` -In this test, we inject into the `:percentiles` stream three events, and we verify that the tap named `:alert` contains the expected alert (the `0.99` quantile is greater than 1) generated for these events. +In this test, we inject into the `:percentiles` stream some events, and we verify that the tap named `:alert` contains the expected alert (the `0.99` quantile is greater than 1) generated for these events. Thanks to Clojure datastructures, there is *no side effects between streams and actions*. It's OK to modify events in parallel (in multiple threads) and to have multiple branches per stream. You can even pass events between streams (like described [here](todo)). You are free to organize streams and how they communicate between each other exactly how you want to; the tool and its DSL will not limit you. @@ -92,12 +97,10 @@ Here is a more complete and commented example, with multiple actions performed i ```clojure (streams (stream {:name :multiple-branches} - (where [:= :service "http_request_duration_seconds"] + (where [:= :name "http_request_duration_seconds"] (with :ttl 60 - ;; push everything into influxdb - (output! :influxdb) - ;; index events in memory by host and service - (index [:host :service]) + ;; push everything into Prometheus + (output! :prometheus) ;; by will generate a branch for each :host value. Like that, downstream ;; computations will be per host and will not conflict between each other (by {:fields [:host]} @@ -111,3 +114,4 @@ Here is a more complete and commented example, with multiple actions performed i (output! :pagerduty))))))))) ``` +Raw events or computation results could also be forwarded to external systems (TSDB for example) for long-term storage. diff --git a/site/mirabelle/content/api/_index.md b/site/mirabelle/content/api/_index.md index 44af0d81..038951e0 100644 --- a/site/mirabelle/content/api/_index.md +++ b/site/mirabelle/content/api/_index.md @@ -92,12 +92,12 @@ curl -X DELETE localhost:5558/api/v1/stream/trololo - **PUT** `/api/v1/stream/` -Push an event to the specified stream. +Push an event (or a batch of events) to the specified stream. --- ``` -curl -H "Content-Type: application/json" -X PUT --data '{"event": {"service": "foo", "metric": 10}}' 127.0.0.1:5558/api/v1/stream/foo +curl -H "Content-Type: application/json" -X PUT --data '{"events": [{"service": "foo", "metric": 10}]}' 127.0.0.1:5558/api/v1/stream/foo {"message":"ok"} ``` diff --git a/site/mirabelle/content/howto/action-io-ref/_index.md b/site/mirabelle/content/howto/action-io-ref/_index.md index ff0b40e7..bbd8a228 100644 --- a/site/mirabelle/content/howto/action-io-ref/_index.md +++ b/site/mirabelle/content/howto/action-io-ref/_index.md @@ -12,68 +12,36 @@ Outputs can be [referenced in streams](/howto/stream/#outputs-and-async-queues) ### File -This I/O write all events into a file, as edn. +This I/O writes all events into a file, as edn. ```clojure {:my-io-file {:config {:path "/tmp/events?edn"} :type :file}} ``` -### Pagerduty +### Prometheus -This I/O forwards events to [Pagerduty](https://pagerduty.com). +This I/O forwards events to [Prometheus](https://prometheus.io/). + +The `:name` value is used for the metric name. The `:state` and `service` keys will be used as labels, as well as all keys stored in the `:attributes` map. ```clojure -{:pagerduty-client {:config {:service-key #secret "pagerduty-service-key" - :source-key :service - :summary-keys [:host :service :state] - :dedup-keys [:host :service] - :http-options {:socket-timeout 5000 - :connection-timeout 5000}} - :type :pagerduty}} +{:name "http_requests_total" + :service "my-app" + :state "critical" + :metric 10 + :timestamp 1735047872000000000 + :attributes {:environment "production"}} ``` -- The `:service-key` parameter is your Pagerduty service (integration) key. -- `:source-key` is the event key which will be used for the alert source in the Pagerduty payload. -- `:summary-keys` is a list of keys which will be used to build the event summary. In this example, the summary would be `--`. -- `:dedup-keys` is a list of keys used to build the Pagerduty dedup key in the alert payload. - `:http-options` is an optional map for extra HTTP options (see [clj-http](https://github.com/dakrone/clj-http) for more information). +This event will produce the metric `http_requests_total{"service": "my-app", "state": "critical", "environment": "production"} 10` for the given timestamp. -The raw event will also be sent to Pagerduty in the `custom_details` field. The alert timestamp will be the event time, or the current time if the event has no time. - -By default, the event `:state` is used to deduce the right Pagerduty action: - -- "critical": trigger -- "ok": resolve - -You can also set a `:pagerduty/action` key to your event in order to set the action (with the `with` action for example: `(with :pagerduty/action :trigger ...)` - -### InfluxDB - -Forward events to the [InfluxDB](https://www.influxdata.com/) timserie database. This I/O forwards events to InfluxDB asynchronously. ```clojure -{:influxdb {:config {:connection-string "http://127.0.0.1:8086" - :bucket "mirabelle" - :org "mirabelle" - :measurement :service - ;; either use username/password - :username "mirabelle" - :password #secret "mirabelle" - ;; or token authenticate - :token #secret "my-token" - :default-tags {"project" "mirabelle"} - :tags [:service] - :fields [:environment]} - :type :influxdb}} +{:prometheus {:type :prometheus + :config {:url "http://localhost:9090/api/v1/write"}}} ``` -The `:measurement`, `:username`, `:password`, `:token`, `:default-tags`, `:tags` and `fields` parameters are optional. The `:measurement` parameter is the event key which will be used for the influxdb measurement - -Default tags will be added to all events. The `:tags` option contains the list of keys to convert to influxdb tags, and the `:fields` option for fields. - -You can also add the `:influxdb/measurement`, `:influxdb/fields` and `:influxdb/tags` to your events (using the `with` action for example) in order to override per event the default configuration for these options. - ### Elasticsearch Forward events to [ElasticSearch](https://www.elastic.co/fr/). This I/O forwards events to Elasticsearch asynchronously. @@ -111,6 +79,35 @@ Forward events to [ElasticSearch](https://www.elastic.co/fr/). This I/O forwards You can set `:elasticsearch/index` to your event in order to forward an event to a specific index. +### Pagerduty + +This I/O forwards events to [Pagerduty](https://pagerduty.com). + +```clojure +{:pagerduty-client {:config {:service-key #secret "pagerduty-service-key" + :source-key :service + :summary-keys [:host :service :state] + :dedup-keys [:host :service] + :http-options {:socket-timeout 5000 + :connection-timeout 5000}} + :type :pagerduty}} +``` + +- The `:service-key` parameter is your Pagerduty service (integration) key. +- `:source-key` is the event key which will be used for the alert source in the Pagerduty payload. +- `:summary-keys` is a list of keys which will be used to build the event summary. In this example, the summary would be `--`. +- `:dedup-keys` is a list of keys used to build the Pagerduty dedup key in the alert payload. + `:http-options` is an optional map for extra HTTP options (see [clj-http](https://github.com/dakrone/clj-http) for more information). + +The raw event will also be sent to Pagerduty in the `custom_details` field. The alert timestamp will be the event time, or the current time if the event has no time. + +By default, the event `:state` is used to deduce the right Pagerduty action: + +- "critical": trigger +- "ok": resolve + +You can also set a `:pagerduty/action` key to your event in order to set the action (with the `with` action for example: `(with :pagerduty/action :trigger ...)` + ## Actions The [generated documentation](/generated-doc/mirabelle.action.html) from the code contains explanations and examples about the available actions. Here is the list: @@ -136,8 +133,7 @@ The [generated documentation](/generated-doc/mirabelle.action.html) from the cod - [coll-sum](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-coll-sum) - [coll-top](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-coll-top) - [coll-where](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-coll-where) -- [critical](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-critical) -- [critical-dt](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-critical-dt) +- [cond-dt](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-cond-dt) - [ddt](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-ddt) - [ddt-pos](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-ddt-pos) - [debug](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-debug) @@ -151,12 +147,12 @@ The [generated documentation](/generated-doc/mirabelle.action.html) from the cod - [fixed-event-window](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-fixed-event-window) - [fixed-time-window](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-fixed-time-window) - [from-base64](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-from-base64) +- [from-json](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-from-json) - [include](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-include) - [increment](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-increment) -- [index](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-index) - [info](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-info) - [io](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-io) -- [json-fields](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-json-fields) +- [iterate-on](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-iterate-on) - [keep-keys](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-keep-keys) - [mean](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-mean) - [moving-event-window](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-moving-event-window) @@ -169,7 +165,7 @@ The [generated documentation](/generated-doc/mirabelle.action.html) from the cod - [project](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-project) - [publish!](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-publish!) - [rate](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-rate) -- [reaper](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-reaper) +- [ratio](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-ratio) - [reinject!](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-reinject!) - [rename-keys](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-rename-keys) - [scale](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-scale) @@ -189,9 +185,9 @@ The [generated documentation](/generated-doc/mirabelle.action.html) from the cod - [test-action](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-test-action) - [throttle](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-throttle) - [to-base64](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-to-base64) +- [to-string](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-to-string) - [top](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-top) - [under](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-under) - [untag](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-untag) -- [warning](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-warning) - [where](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-where) - [with](https://mirabelle.mcorbin.fr/generated-doc/mirabelle.action.html#var-with) diff --git a/site/mirabelle/content/howto/build/_index.md b/site/mirabelle/content/howto/build/_index.md index 72a78020..fc80ceec 100644 --- a/site/mirabelle/content/howto/build/_index.md +++ b/site/mirabelle/content/howto/build/_index.md @@ -11,7 +11,7 @@ In order to build Mirabelle, you need to install: - [Leiningen](https://leiningen.org/), the Clojure build tool. - Java. Mirabelle is tested under Java 17 (LTS). -Then, clone the Mirabelle [Git repository](https://github.com/mcorbin/mirabelle). You can now build the project with `lein uberjar`. +Then, clone the Mirabelle [Git repository](https://github.com/appclacks/mirabelle). You can now build the project with `lein uberjar`. The resulting jar will be in `target/uberjar/mirabelle--standalone.jar` @@ -39,14 +39,14 @@ Let's say your EDN (compiled) streams are in `/etc/mirabelle/streams`, your conf :console {:encoder :json}}} ``` -You should now be able to launch Mirabelle using Docker (you can check the [Docker hub](https://hub.docker.com/r/mcorbin/mirabelle/tags) to get the latest release): +You should now be able to launch Mirabelle using Docker (you can check the [Docker hub](https://hub.docker.com/r/appclacks/mirabelle/tags) to get the latest release): ``` docker run -p 5555:5555 -p 5558:5558 \ -v /etc/mirabelle/streams:/streams \ -v /etc/mirabelle/config.edn:/config/config.edn \ -e MIRABELLE_CONFIGURATION=/config/config.edn \ -appclacks/mirabelle:v0.13.0 +appclacks/mirabelle:v0.14.0 ``` ## Using Leiningen diff --git a/site/mirabelle/content/howto/pubsub/_index.md b/site/mirabelle/content/howto/pubsub/_index.md index 39f93074..f7783324 100644 --- a/site/mirabelle/content/howto/pubsub/_index.md +++ b/site/mirabelle/content/howto/pubsub/_index.md @@ -4,36 +4,25 @@ weight: 14 disableToc: false --- -The Mirabelle Websocket server allows user to subscribe to the streams [index](/howto/stream-index/) or to channels. +The Mirabelle Websocket server allows user to subscribe to channels. + +This guide uses the [https://github.com/appclacks/cl](Appclacks CLI) to interact with Mirabelle. ## An example ```clojure (streams - (stream - {:name :bar :default true} + (stream {:name :bar :default true} (where [:= :service "bar"] - (index [:host]) (publish! :my-channel)))) ``` -In this stream, we filter all events with `:service` "bar". Then, we index them, and we also call `publish!` to publish the event to a channel named `:my-channel`. - -A channel is created by default for each index (the channel name being `-index`). -You can also subscribe for events indexed into indexes belonging to streams tagged `:default` by subscribing to the `default-index` channel. +In this stream, we filter all events with `:service` "bar", then we publish them to a channel named `:my-channel`. -Users can also subscribe to other channels, not related to indexes. It's what `publish!` does. In this example, all users subscribing to the `:my-channel` channel will receive the events. +Users then subscripe to those channels. In this example, all users subscribing to the `:my-channel` channel will receive the events. -You can test that yourself by running the `websocket.py` script available [here](https://github.com/mcorbin/mirabelle/tree/master/pubsub). You will need Python 3, and to install the dependencies listed in `requirements.txt` using `pip install -r requirements.txt`. +You can test that yourself by running the `appclacks mirabelle subscribe --channel my-channel` command. You can then send an event to Mirabelle (`mirabelle event send --name http_requests_duration_seconds --metric 1 --service bar`) and it should be displayed by the `subscribe` command. -When you subscribe to a channel, you should provide a valid [where clause](/howto/stream/#filtering-events) in base64. For example, the query `[:always-true]` which matches everything would be `WzphbHdheXMtdHJ1ZV0=`. - -You can now run the script with `./websocket.py`. By default, it subscribes to the default index. You can specify a channel with `--channel` (like `--channel my-channel` here), or a query with `--query` (the query will be automatically converted to base64 by the script). +When you subscribe to a channel, you should provide a valid [where clause](/howto/stream/#filtering-events) in base64. For example, the query `[:> :metric 10]` which matches everything would be `Wzo+IDptZXRyaWMgMTBd`. The CLI will do the conversion for you: `appclacks mirabelle subscribe --channel my-channel --query '[:> :metric 10]'` will keep only events with the `metric` field greater than 10. You can also write your own scripts in various languages. You need to subscribe to `ws://:/channel/?query=`. - -### Riemann compatibility - -In order to be compatible with Riemann clients, you can also subscribe on `/index` (to subscribe to the default index), or `/index?stream=` to subscribe to an index for a dynamic stream. - - diff --git a/site/mirabelle/content/howto/stream-index/_index.md b/site/mirabelle/content/howto/stream-index/_index.md deleted file mode 100644 index 7c61ab59..00000000 --- a/site/mirabelle/content/howto/stream-index/_index.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -title: The index -weight: 12 -disableToc: false ---- - -The Mirabelle Index is an in memory map which can be used to store events. - -Each Mirabelle stream (generated from the configuration file or from the API) has its own, dedicated index instance. - -## How to push events into the index - -```clojure -(stream - {:name :bar :default true} - (where [:= :service "bar"] - (index [:host :service]))) -``` - -In this example, all events with `:service` "bar" will be added to the index. They will be indexed by their `:host` and `:service` key. - -For example, if these two events `{:host "foo" :service "bar" :time 1 :ttl 60}` and `{:host "mirabelle" :service "bar" :time 1 :ttl 60}` arrive in the system, the index would contain two keys: - -`{:host "foo" :service "bar"}` -> `{:host "foo" :service "bar" :time 1 :ttl 60}` -`{:host "mirabelle" :service "bar"}` -> `{:host "mirabelle" :service "bar" :time 1 :ttl 60}` - -If a new event arrives for an already existing key, the old key is overrided. - -You can have multiple calls to `index` in Mirabelle, and index events based on the fields you want. - -## Expiration - -In Mirabelle, events have a `:time` (when the event was generated) and optionally a TTL. The default TTL for events in the index is `120` (in seconds). You could also add another default TTL using the `with` stream. - -For example, an event with `:time` 1 and `:ttl` 120 would be expired at time 122. - -Expiration is important. First, a lot of streams which work on windows of events will remove expired events automatically. Indeed, if you use for example [coalesce](/howto/stream/#coalesce-and-project), you don't want to keep forever the latest event for a host which does not emit anymore. When the event is expired, it's removed from the action. - -Expiration can also be used to detect services which stopped emitting. - -Indeed, if an event stored in the index is expired, it will be reinjected into the stream with `:state "expired"`. You could then catch it with something like: - -```clojure -(expired - (error)) -``` - -By forwarding expired events into dedicated actions, you can for example trigger alerts based on that. - -In order to trigger event expiration from events stored into the index, we should use the `reaper` action. All streams in Riemann are using the events clocks to trigger side effects, and the reaper action will update the index clock if needed, trigger events expiration, and then reinject the events into streams: - -```clojure -(reaper 5) -(reaper 5 :custom-stream) -``` - -The `reaper` action takes as first parameter the interval (in seconds) on which the reaper stream will expire events. Don't forget you should feed the reaper action with events in order to move its clock. - -The `custom-stream` parameter is optional. By default, events are reinjected into the current stream. You can also pass a stream name to the reaper action in order to reinject events into another stream. - -I recommand you to not call forward all events to the reaper (it will impact performances). Instead, identify a couple of events arriving regularly, and forward them to the reaper in order to update the index clock. - -## Queries - -You can query the events stored into the index. The query should be a valid [where clause](/howto/stream/#filtering-events) in base64. For example `[:and [:> :metric 10] [:= :service "bar"]]` would be `WzphbmQgWzo+IDptZXRyaWMgMTBdIFs6PSA6c2VydmljZSAiYmFyIl1d`. - -You can send queries using the Mirabelle (or Riemann) TCP clients, or using the [HTTP API](http://localhost:1313/api/#query-the-index). - -## Pub Sub - -Users can subscribe to the index using Websocket, in order to see in real time which events are indexed. Check the [pubsub documentation](howto/pubsub/) for more information. - - - diff --git a/site/mirabelle/content/howto/stream/_index.md b/site/mirabelle/content/howto/stream/_index.md index f9c0ba05..fe667a78 100644 --- a/site/mirabelle/content/howto/stream/_index.md +++ b/site/mirabelle/content/howto/stream/_index.md @@ -7,6 +7,8 @@ disableToc: false In this section, you will learn about how streams work, how to define them and how to use them. Not all available actions and I/O are listed here. You can see the full list in [this section of the documentation](/howto/action-io-ref/). +This guide uses the [https://github.com/appclacks/cl](Appclacks CLI) to interact with Mirabelle. + **Concepts** - [What is a Mirabelle event](/howto/stream/#events) @@ -39,17 +41,19 @@ Mirabelle ships with a complete, extensible DSL to define streams. The DSL is he ### Events -Events are represented as an immutable map. An event has standard fields. All fields are optional. +Events are represented as an immutable map. An event has standard fields. All fields are optional and you can also add arbitrary ones, the schema is fully free. - `:host`: the event source. It can be an hostname for example. -- `:service`: What is measured. `http_requests_duration_seconds` for example. +- `:service`: the name of the service emitting the event. +- `:name`: What is measured. `http_requests_duration_seconds` for example. - `:state`: A string representing the event state. By convention, `ok`, `warning`, `critical` or `expired` are often used. - `:metric`: A number associated to the event (the value of what is measured). -- `:time`: The event time in second, as a timestamp (`1619988803` for example). It could also be a float (`1619988803,173413` for example), the Mirabelle/Riemann protocol supports microsecond resolution. +- `:time`: The event time in nanoseconds, as a timestamp (`1735047872000000000` for example). - `:description`: The event description. -- `:ttl`: The duration that the event is considered valid. See the [Index](/index) documentation for more information about the index and events expiration. - `:tags`: A list of tags associated to the event (like `["foo" "bar"]` for example). -- Extra fields can also be added if you want to. One important extra field is `:stream`. It can be used to specify on which stream the event should be send. By default, events are sent to all streags with `:default` in their configurations. +- `:attributes`: a list of arbitrary key-value attributes. + +Extra fields can also be added if you want to. One important extra field is `:stream`. It can be used to specify on which stream the event should be send. By default, events are sent to all streams with `:default` set to `true` in their configurations. ### Streams @@ -73,7 +77,7 @@ Let's now define another stream: (stream {:name :log :default true} (info)) (stream {:name :http_requests_duration} - (where [:= :service "http_requests_duration_seconds"] + (where [:= :name "http_requests_duration_seconds"] (info) (over 1.5 (with :state "critical" @@ -82,7 +86,7 @@ Let's now define another stream: In this second example, we still have our first stream named `:log`. We also have another stream, a bit more complex, named `:http_requests_duration`. -This second stream will first keep only events with services equal to "http_requests_duration_seconds" using the `where` action. +This second stream will first keep only events with `:name` equal to "http_requests_duration_seconds" using the `where` action. Then, it will log (using `info`) the events. In another branch, `over` is used tp keep only events with `:metric` greater than `1.5` (we can imagine that @@ -102,7 +106,7 @@ The Mirabelle DSL should first be compiled to an EDN datastructure before being ```clojure (streams (stream {:name :http_requests_duration} - (where [:= :service "http_requests_duration_seconds"] + (where [:= :name "http_requests_duration_seconds"] (info) (over 1.5 (with :state "critical" @@ -115,7 +119,7 @@ You then need to compile this file using this command: java -jar mirabelle.jar compile ``` -For example, let's say you have put the previous stream in a file named `stream.clj` in the `/tmp/streams` directory. +For example, let's say you have put the previous stream in a file named `stream.clj` in the `/tmp/streams` directory. If ou launch `java -jar mirabelle.jar compile /tmp/streams /tmp/compiled`, your file will be compiled and a new `stream.clj` file will be created in the destination directory (which is `/tmp/compiled` here). Let's do that. @@ -132,7 +136,7 @@ The resulting file in `/tmp/compiled/stream.clj` should be: {:action :sdo, :children ({:action :where, - :params [[:= :service "http_requests_duration_seconds"]], + :params [[:= :name "http_requests_duration_seconds"]], :children ({:action :info} {:action :over, @@ -151,30 +155,29 @@ Let's launch Mirabelle, with the `/tmp/compiled` directory referenced into the c How to launch Mirabelle is explained in [this section](/howto/build/). -Once Mirabelle started, you can send events to it. For that, you can check the [integration](/integration/) documentation section for the available clients (Riemann clients are fully compatible with Mirabelle). In this example, I will use the [Riemann C client](https://github.com/algernon/riemann-c-client) which provides a CLI and is available in many Linux package managers. +Once Mirabelle started, you can send events to it. For that, you can check the [integration](/integration/) documentation section for the available clients (Riemann clients are fully compatible with Mirabelle). In this example, I will use the [Appclacks CLI](https://github.com/appclacks/cli). You can use the `MIRABELLE_API_ENDPOINT` variable to target a specific host (default to `localhost:5558`): ``` -riemann-client send --metric-f 1 --service "http_requests_duration_seconds" --host=my-host +appclacks mirabelle event send --name http_requests_duration_seconds ``` -If I send the previous event, I should see in Mirabelle logs: +You should see in Mirabelle logs: ```json -{"@timestamp":"2021-05-01T22:48:58.786+02:00","@version":"1","message":"#riemann.codec.Event{:host \"my-host\", :service \"http_requests_duration_seconds\", :state nil, :description nil, :metric 1.0, :tags nil, :time 1.619902138786E9, :ttl nil, :x-client \"riemann-c-client\"}","logger_name":"mirabelle.action","thread_name":"defaultEventExecutorGroup-2-8","level":"INFO","level_value":20000} +{"@timestamp":"2024-12-27T09:11:44.512+01:00","@version":"1","message":"{\"host\":\"fedora\",\"time\":1735287104509143999,\"name\":\"http_requests_duration_seconds\"}","logger_name":"mirabelle.action","thread_name":"qtp2108856868-49","level":"INFO","level_value":20000,"stream":"http_requests_duration"} ``` -My event was indeed logging by the `info` action in my stream. Let's send an event with the metric greater than our threshold: +The event was indeed logging by the `info` action in my stream. Let's send an event with the metric greater than our threshold: ``` -riemann-client send --metric-f 2 --service "http_requests_duration_seconds" --host=my-host +appclacks mirabelle event send --name http_requests_duration_seconds --metric 3 ``` You will see in the Mirabelle logs: ```json -{"@timestamp":"2021-05-01T22:50:57.960+02:00","@version":"1","message":"#riemann.codec.Event{:host \"my-host\", :service \"http_requests_duration_seconds\", :state nil, :description nil, :metric 2.0, :tags nil, :time 1.61990225796E9, :ttl nil, :x-client \"riemann-c-client\"}","logger_name":"mirabelle.action","thread_name":"defaultEventExecutorGroup-2-2","level":"INFO","level_value":20000} - -{"@timestamp":"2021-05-01T22:50:57.961+02:00","@version":"1","message":"#riemann.codec.Event{:host \"my-host\", :service \"http_requests_duration_seconds\", :state \"critical\", :description nil, :metric 2.0, :tags nil, :time 1.61990225796E9, :ttl nil, :x-client \"riemann-c-client\"}","logger_name":"mirabelle.action","thread_name":"defaultEventExecutorGroup-2-2","level":"ERROR","level_value":40000} +{"@timestamp":"2024-12-27T09:12:28.635+01:00","@version":"1","message":"{\"host\":\"fedora\",\"metric\":3,\"time\":1735287148631350038,\"name\":\"http_requests_duration_seconds\"}","logger_name":"mirabelle.action","thread_name":"qtp2108856868-54","level":"INFO","level_value":20000,"stream":"http_requests_duration"} +{"@timestamp":"2024-12-27T09:12:28.635+01:00","@version":"1","message":"{\"host\":\"fedora\",\"metric\":3,\"time\":1735287148631350038,\"name\":\"http_requests_duration_seconds\",\"state\":\"critical\"}","logger_name":"mirabelle.action","thread_name":"qtp2108856868-54","level":"ERROR","level_value":40000,"stream":"http_requests_duration"} ``` The event is logged twice: one time by our `info` action, and the second time by `error` (you can see the `level` key in the log). In the second log, the `:state` was set to "critical". Our threshold works ! @@ -197,7 +200,7 @@ You can set the `PROFILE` environment variable in order to use Aero [profiles](h (streams (stream {:name :foo :default true} (where [:and - [:= :service "disk-used"] + [:= :name "disk-used"] [:> :metric #profile {:preprod 70 :prod 60 :default 90}]] @@ -212,10 +215,10 @@ You can also use other Aero build-in readers described in the Aero [readme](http **Include** -It's possible to include a configuration file in another one. Let's take this file named for example `log-service.clj`: +It's possible to include a configuration file in another one. Let's take this file named for example `log-name.clj`: ```clojure -(where [:= :service #mirabelle/var :my-service] +(where [:= :name #mirabelle/var :my-name] (info)) ``` @@ -224,16 +227,16 @@ You can then use this file using `include` in a Mirabelle stream: ```clojure (streams (stream {:name :foo :default true} - (include "log-service.clj" {:variables {:my-service "disk-used"}}) - (include "log-service.clj" {:variables {:my-service "ram-used"}}))) + (include "log-name.clj" {:variables {:my-name "disk-used"}}) + (include "log-name.clj" {:variables {:my-name "ram-used"}}))) ``` -The `#mirabelle/var` reader allows you to read a variable passed to the `include` action (here, the variable is named `:my-service`). +The `#mirabelle/var` reader allows you to read a variable passed to the `include` action (here, the variable is named `:my-name`). You can also override the default Mirabelle profile (passed as an environment variable) by passing the `;profile` key to the `include` options: ```clojure -(include "log-service.clj" {:variables {:my-service "disk-used"} +(include "log-name.clj" {:variables {:my-name "disk-used"} :profile :dev}) ``` @@ -269,10 +272,10 @@ You can now use this output named `:pagerduty-client` in a stream by using the ` If this event is set to Mirabelle: ```shell -riemann-client send --metric-d 100 --service "http_requests_duration_seconds" --state "critical" --host=myhost --attribute=environment=prod +mirabelle event send --name http_requests_duration_seconds --metric 100 --attributes=environment=prod --state critical ``` -You should see in Pagerduty a new triggered alert named `myhost - http_requests_duration_seconds - critical` containing all the informations about your event. +You should see in Pagerduty a new triggered alert named `hostname - http_requests_duration_seconds - critical` containing all the informations about your event. You can check the [I/O documentation](/howto/action-io-ref/) to have details about how the Pagerduty output can be used (to resolve alert automatically for example). @@ -368,7 +371,7 @@ For example, the `above-dt` stream will only let events pass if all events recei In this example, `above-dt` will let events pass (to log them as error) only if it receives events with `:metric` greater than 1 during more than 6O seconds. -The streams `below-dt`, `between-dt`, `outside-dt`, `critical-dt` also work that way. They are useful to avoid alerting on spikes for examples. +The streams `below-dt`, `between-dt`, `outside-dt`, `cond-dt` also work that way. They are useful to avoid alerting on spikes for examples. The `tagged-all` stream is also available to keep only events containing one tag or a set of tags: `(tagged-all "foo")` or `(tagged-all ["foo" "bar"])`. @@ -404,12 +407,12 @@ You can use `rename-keys` to rename some events keys: :environment :env}) ``` -In this example, the `:host` key will be renamed `:service` and the `:environment` key is renamed `:env`. Existing values will be overrided. +In this example, the `:host` key will be renamed `:service` and the `:environment` key is renamed `:env`. Existing values will be overrided. Nested keys are also supported, by passing a list of fields as keys or attributes (for example `[:attributes :foo]`. If you want to keep only some keys from an event (and so remove all the others), you can use `keep-keys`: ```clojure -(keep-keys [:host :service :time :metric :description :environment]) +(keep-keys [:host :service :time :metric :description [:nested :field]]) ``` Some actions can modify the `:metric` field. `increment` and `decrement` will add +1 or -1 to it, and you can use `scale` to multiply it with a value: `(scale 1000)` for example. @@ -430,11 +433,10 @@ The `percentiles` action allows you to compute percentiles (quantiles) on events ```clojure (percentiles {:percentiles [0.5 0.75 0.99] - :duration 10 - :nb-significant-digits 3} + :duration 10} ``` -This example will compute the 0.5, 0.75 and 0.99 quantiles every 10 seconds. It also supports the `:delay` parameter (to tolerate events arriving late), `:highest-trackable-value` and `:lowest-discernible-value` to bound results. +This example will compute the 0.5, 0.75 and 0.99 quantiles every 10 seconds. It also supports the `:delay` parameter (to tolerate events arriving late), `:highest-trackable-value` and `:lowest-discernible-value` to bound results, and `:nb-significant-digits` for precision (default to 3). The implementation uses the [HdrHistogram](https://github.com/HdrHistogram/HdrHistogram) library. @@ -448,7 +450,6 @@ You can compute the rate of incoming events (by counting them) using the `rate` This action will send the rate of events downstream every 20 seconds. You can also add a `:delay` parameter to tolerate late events. - #### Sum events for a time period The `:sum` stream can be used to sum events `:metrics` field for a period of time: @@ -688,6 +689,15 @@ We already saw in the [Action on lists of events: max, min, count, percentiles, `smax` and `smin` will send *for each event they receive* the maximum or minimum event. The value is never resetted. +#### Convert a value to string + +The `to-string` stream can be used to convert values associated to some keys to string: + +```clojure +(to-string [:service :state [:nested :attributes] + (info)) +``` + #### Format a string based on values If you need to create a value from others keys values, use `sformat`: diff --git a/site/mirabelle/content/howto/tests/_index.md b/site/mirabelle/content/howto/tests/_index.md index 3c524978..64b30b18 100644 --- a/site/mirabelle/content/howto/tests/_index.md +++ b/site/mirabelle/content/howto/tests/_index.md @@ -40,10 +40,10 @@ Let's write a test file for these streams. The tests file should be referenced i {:metric 101 :service "bar"}] - :tap-results {:foo [{:metric 10 - :service "foo"}] + :taps {:foo [{:metric 10 + :service "foo"}] :bar [{:metric 101 - :service "bar"}]}}} + :service "bar"}]}}} ``` We defined here a test named `:test1`. You could have add more tests to the map (or in another files), all tests are run in isolation. @@ -51,7 +51,7 @@ We defined here a test named `:test1`. You could have add more tests to the map This `:test1` key contains a map with two keys: - `:input`: the events which will be injected into the streams -- `:tap-results`: the expected contents of the tap. +- `:taps`: the expected contents of the tap. Here, we see that the tap `:foo` should contain the first event (with `:service` "foo"), and the tap `:bar` the second (with `:metric` greater than 100). @@ -82,14 +82,12 @@ Tests can also take a `:target` configuration, for example: {:metric 101 :service "bar"}] :target :foo - :tap-results {:foo [{:metric 10 - :service "foo"}]}}} + :taps {:foo [{:metric 10 + :service "foo"}]}}} ``` In this example, events in `:input` will only be injected into the `:foo` stream. -One dedicated [index](/howto/stream-index/) is created for each test. - ## Things excluded from tests In test mode, some streams behave differently than in the regular mode: diff --git a/site/mirabelle/content/integration/_index.md b/site/mirabelle/content/integration/_index.md index 2ae272d0..c72685c5 100644 --- a/site/mirabelle/content/integration/_index.md +++ b/site/mirabelle/content/integration/_index.md @@ -6,25 +6,20 @@ chapter: false All [Riemann tooling](http://riemann.io/clients.html) should in theory work with Mirabelle. -I will test with Mirabelle the Riemann integrations one by one and update this page later. +## Prometheus -## Emacs - -You can put that in your emacs configuration in order to have a good indentation for Mirabelle actions (as you can see, only a few actions are list, I will put the other ones later): +Mirabelle supports receiving events from [https://prometheus.io/](Prometheus). You can configure Prometheus to point to the Mirabelle HTTP API: ``` -(put-clojure-indent 'above-dt 2) -(put-clojure-indent 'by 1) -(put-clojure-indent 'coalesce 2) -(put-clojure-indent 'coll-percentiles 1) -(put-clojure-indent 'changed 2) -(put-clojure-indent 'fixed-time-window 1) -(put-clojure-indent 'not-expired 0) -(put-clojure-indent 'over 1) -(put-clojure-indent 'rate 0) -(put-clojure-indent 'sflatten 0) -(put-clojure-indent 'sformat 3) -(put-clojure-indent 'stream 1) -(put-clojure-indent 'with 1) -(put-clojure-indent 'where 1) +remote_write: + - url: 'http://localhost:5558/api/v1/prometheus/remote-write/default' ``` + +This configuration will send Prometheus metrics too the default streams. You can send metrics to a specific stream by replacying `default` with a stream name. + +The metric name will be set to the event `:name` key, the metric value to `:metric`, the timestamp to `:time`, all all labels will be set in the `:attributes` map. + +## Opentelemetry traces + +Mirabelle supports receiving traces through HTTP, in the [https://opentelemetry.io/docs/concepts/signals/traces/](Opentelemetry) format on the endpoint `/api/v1/opentelemetry/v1/traces/`. +For example, you could set `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:5558/api/v1/opentelemetry/v1/traces/default` to send traces to default streams in an Opentelemetry client. diff --git a/site/mirabelle/public/404.html b/site/mirabelle/public/404.html index cd0c3057..78c3e1bd 100644 --- a/site/mirabelle/public/404.html +++ b/site/mirabelle/public/404.html @@ -9,13 +9,13 @@ 404 Page not found - - - - - - - + + + + + + + - - - - - - - - -
-
-
- -
-
- - - - -
-
- -
-
- - -
-
- -
- -
- -
- -

- - The index -

- - - - - - -

The Mirabelle Index is an in memory map which can be used to store events.

-

Each Mirabelle stream (generated from the configuration file or from the API) has its own, dedicated index instance.

-

How to push events into the index

-
(stream
-  {:name :bar :default true}
-  (where [:= :service "bar"]
-    (index [:host :service])))
-

In this example, all events with :service “bar” will be added to the index. They will be indexed by their :host and :service key.

-

For example, if these two events {:host "foo" :service "bar" :time 1 :ttl 60} and {:host "mirabelle" :service "bar" :time 1 :ttl 60} arrive in the system, the index would contain two keys:

-

{:host "foo" :service "bar"} -> {:host "foo" :service "bar" :time 1 :ttl 60} -{:host "mirabelle" :service "bar"} -> {:host "mirabelle" :service "bar" :time 1 :ttl 60}

-

If a new event arrives for an already existing key, the old key is overrided.

-

You can have multiple calls to index in Mirabelle, and index events based on the fields you want.

-

Expiration

-

In Mirabelle, events have a :time (when the event was generated) and optionally a TTL. The default TTL for events in the index is 120 (in seconds). You could also add another default TTL using the with stream.

-

For example, an event with :time 1 and :ttl 120 would be expired at time 122.

-

Expiration is important. First, a lot of streams which work on windows of events will remove expired events automatically. Indeed, if you use for example coalesce, you don’t want to keep forever the latest event for a host which does not emit anymore. When the event is expired, it’s removed from the action.

-

Expiration can also be used to detect services which stopped emitting.

-

Indeed, if an event stored in the index is expired, it will be reinjected into the stream with :state "expired". You could then catch it with something like:

-
(expired
-  (error))
-

By forwarding expired events into dedicated actions, you can for example trigger alerts based on that.

-

In order to trigger event expiration from events stored into the index, we should use the reaper action. All streams in Riemann are using the events clocks to trigger side effects, and the reaper action will update the index clock if needed, trigger events expiration, and then reinject the events into streams:

-
(reaper)
-(reaper 5 :custom-stream)
-

The reaper action takes as first parameter the interval (in seconds) on which the reaper stream will expire events. Don’t forget you should feed the reaper action with events in order to move its clock.

-

The custom-stream parameter is optional. By default, events are reinjected into the current stream. You can also pass a stream name to the reaper action in order to reinject events into another stream.

-

I recommand you to not call forward all events to the reaper (it will impact performances). Instead, identify a couple of events arriving regularly, and forward them to the reaper in order to update the index clock.

-

Queries

-

You can query the events stored into the index. The query should be a valid where clause in base64. For example [:and [:> :metric 10] [:= :service "bar"]] would be WzphbmQgWzo+IDptZXRyaWMgMTBdIFs6PSA6c2VydmljZSAiYmFyIl1d.

-

You can send queries using the Mirabelle (or Riemann) TCP clients, or using the HTTP API.

-

Pub Sub

-

Users can subscribe to the index using Websocket, in order to see in real time which events are indexed. Check the pubsub documentation for more information.

- - - - - -
- -
- - -
- - -
- - - -
- -
-
-
- - - - - - - - - - - - - - - - - - - - diff --git a/site/mirabelle/public/howto/index/index.xml b/site/mirabelle/public/howto/index/index.xml deleted file mode 100644 index 45dd9d8d..00000000 --- a/site/mirabelle/public/howto/index/index.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - The index on Mirabelle - https://www.mirabelle.mcorbin.fr/howto/index/ - Recent content in The index on Mirabelle - Hugo -- gohugo.io - en-us - - diff --git a/site/mirabelle/public/howto/on-disk-queue/index.html b/site/mirabelle/public/howto/on-disk-queue/index.html deleted file mode 100644 index c1caf741..00000000 --- a/site/mirabelle/public/howto/on-disk-queue/index.html +++ /dev/null @@ -1,962 +0,0 @@ - - - - - - - - - - - - On Disk queue :: Mirabelle - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- -
-
- - - - -
-
- -
-
- - -
-
- -
- -
- -
- -

- - On Disk queue -

- - - - - - -

Warning: I’m not totally satisfied with this feature and with the current implementation. It will change in the future. Please provide me feedbacks ;)

-

Mirabelle allows you in its configuration file to optionally persist some events into a queue written on disk. The queue implementation is a Chronicle Queue.

-

You can configure the queue roll cycle in the Mirabelle configuration. The default is :half-hourly. More information about roll cycles can be found here.

-

Note: Mirabelle will not rotate/delete old files for you. You need to do it yourself.

-
(where [:= :service "foo"]
-  (disk-queue!))
-

In this example, Mirabelle will write all events with :service “foo” on disk.

-

Why is it useful ? You can use the queue for:

-
    -
  • Archive events. You could have a cronjob taking all events of the day and putting them into S3 for example. You could then reuse these events for example to reinject them later into Mirabelle.
    • -
  • Rebuilding states. Mirabelle streams will not keep their states if Mirabelle crashes. But when Mirabelle starts, all events from the ondisk queue are reinjected in the order they are in the queue into Mirabelle.
    • -
-

The reinjected events will be tagged “discard”. All stateful actions (I/O, pubsub, logger) will not be executed for discarded events. It means the events will rebuild all streams internal states but without producing side effects.

-

It’s nice, but it’s not perfect. The two big limitations today are:

-
    -
  • Events will only be reinjected into the :default streams. So be careful about that, the queue will not work with other streams. I will work on that.
    • -
  • It can be slow. What if you have 10 millions events in the queue ? Even if you process them at 100 000 events per seconds, it will take 100 seconds before Mirabelle would be fully started. Do you really want to wait minutes for your service to start ?
    • -
-

This feature will probably change in the future, i’m thinking hard about how to rebuild streams states quickly and without sacrificing streams performances. Ideas are welcomed.

- - - - - -
- -
- - -
- - -
- - - -
- -
-
-
- - - - - - - - - - - - - - - - - - - - diff --git a/site/mirabelle/public/howto/on-disk-queue/index.xml b/site/mirabelle/public/howto/on-disk-queue/index.xml deleted file mode 100644 index 61bf6267..00000000 --- a/site/mirabelle/public/howto/on-disk-queue/index.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - On Disk queue on Mirabelle - https://www.mirabelle.mcorbin.fr/howto/on-disk-queue/ - Recent content in On Disk queue on Mirabelle - Hugo -- gohugo.io - en-us - - diff --git a/site/mirabelle/public/howto/pubsub/index.html b/site/mirabelle/public/howto/pubsub/index.html index 934bb1c3..1868a596 100644 --- a/site/mirabelle/public/howto/pubsub/index.html +++ b/site/mirabelle/public/howto/pubsub/index.html @@ -12,20 +12,20 @@ Publish Subscribe :: Mirabelle - - - - - - - - - - + + + + + + + + + + - + - - - - - - - - -
-
-
- -
-
- - - - -
-
- -
-
- - -
-
- -
- -
- -
- -

- - The index -

- - - - - - -

The Mirabelle Index is an in memory map which can be used to store events.

-

Each Mirabelle stream (generated from the configuration file or from the API) has its own, dedicated index instance.

-

How to push events into the index

-
(stream
-  {:name :bar :default true}
-  (where [:= :service "bar"]
-    (index [:host :service])))
-

In this example, all events with :service “bar” will be added to the index. They will be indexed by their :host and :service key.

-

For example, if these two events {:host "foo" :service "bar" :time 1 :ttl 60} and {:host "mirabelle" :service "bar" :time 1 :ttl 60} arrive in the system, the index would contain two keys:

-

{:host "foo" :service "bar"} -> {:host "foo" :service "bar" :time 1 :ttl 60} -{:host "mirabelle" :service "bar"} -> {:host "mirabelle" :service "bar" :time 1 :ttl 60}

-

If a new event arrives for an already existing key, the old key is overrided.

-

You can have multiple calls to index in Mirabelle, and index events based on the fields you want.

-

Expiration

-

In Mirabelle, events have a :time (when the event was generated) and optionally a TTL. The default TTL for events in the index is 120 (in seconds). You could also add another default TTL using the with stream.

-

For example, an event with :time 1 and :ttl 120 would be expired at time 122.

-

Expiration is important. First, a lot of streams which work on windows of events will remove expired events automatically. Indeed, if you use for example coalesce, you don’t want to keep forever the latest event for a host which does not emit anymore. When the event is expired, it’s removed from the action.

-

Expiration can also be used to detect services which stopped emitting.

-

Indeed, if an event stored in the index is expired, it will be reinjected into the stream with :state "expired". You could then catch it with something like:

-
(expired
-  (error))
-

By forwarding expired events into dedicated actions, you can for example trigger alerts based on that.

-

In order to trigger event expiration from events stored into the index, we should use the reaper action. All streams in Riemann are using the events clocks to trigger side effects, and the reaper action will update the index clock if needed, trigger events expiration, and then reinject the events into streams:

-
(reaper 5)
-(reaper 5 :custom-stream)
-

The reaper action takes as first parameter the interval (in seconds) on which the reaper stream will expire events. Don’t forget you should feed the reaper action with events in order to move its clock.

-

The custom-stream parameter is optional. By default, events are reinjected into the current stream. You can also pass a stream name to the reaper action in order to reinject events into another stream.

-

I recommand you to not call forward all events to the reaper (it will impact performances). Instead, identify a couple of events arriving regularly, and forward them to the reaper in order to update the index clock.

-

Queries

-

You can query the events stored into the index. The query should be a valid where clause in base64. For example [:and [:> :metric 10] [:= :service "bar"]] would be WzphbmQgWzo+IDptZXRyaWMgMTBdIFs6PSA6c2VydmljZSAiYmFyIl1d.

-

You can send queries using the Mirabelle (or Riemann) TCP clients, or using the HTTP API.

-

Pub Sub

-

Users can subscribe to the index using Websocket, in order to see in real time which events are indexed. Check the pubsub documentation for more information.

- - - - - -
- -
- - -
- - -
- - - -
- -
-
-
- - - - - - - - - - - - - - - - - - - - diff --git a/site/mirabelle/public/howto/stream-index/index.xml b/site/mirabelle/public/howto/stream-index/index.xml deleted file mode 100644 index 3dd93985..00000000 --- a/site/mirabelle/public/howto/stream-index/index.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - The index on Mirabelle - https://www.mirabelle.mcorbin.fr/howto/stream-index/ - Recent content in The index on Mirabelle - Hugo -- gohugo.io - en-us - - - diff --git a/site/mirabelle/public/howto/stream/index.html b/site/mirabelle/public/howto/stream/index.html index b61f80c4..22ab4579 100644 --- a/site/mirabelle/public/howto/stream/index.html +++ b/site/mirabelle/public/howto/stream/index.html @@ -12,20 +12,20 @@ Writing streams :: Mirabelle - - - - - - - - - - + + + + + + + + + + - +