From 87efffa9510f8c7ef0a433b3f1cafa5fab3202af Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 16 Oct 2025 11:12:13 +0300 Subject: [PATCH 01/13] add network_health data stream --- .../_dev/build/docs/README.md | 12 ++ packages/cisco_meraki_metrics/changelog.yml | 5 + .../agent/stream/stream.yml.hbs | 22 ++++ .../elasticsearch/ingest_pipeline/default.yml | 66 +++++++++++ .../network_health/fields/agent.yml | 23 ++++ .../network_health/fields/base-fields.yml | 12 ++ .../network_health/fields/fields.yml | 30 +++++ .../data_stream/network_health/manifest.yml | 44 +++++++ .../network_health/sample_event.json | 76 ++++++++++++ packages/cisco_meraki_metrics/docs/README.md | 109 ++++++++++++++++++ packages/cisco_meraki_metrics/manifest.yml | 2 +- 11 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json diff --git a/packages/cisco_meraki_metrics/_dev/build/docs/README.md b/packages/cisco_meraki_metrics/_dev/build/docs/README.md index 1a55ae9e0a2..28b93f79877 100644 --- a/packages/cisco_meraki_metrics/_dev/build/docs/README.md +++ b/packages/cisco_meraki_metrics/_dev/build/docs/README.md @@ -43,3 +43,15 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur {{fields "device_health"}} {{event "device_health"}} + +### Network Health + +The `network_health` dataset provides metrics related to the overall health and performance of Meraki networks. + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +{{fields "network_health"}} + +{{event "network_health"}} diff --git a/packages/cisco_meraki_metrics/changelog.yml b/packages/cisco_meraki_metrics/changelog.yml index a829163d711..50d8b2036c8 100644 --- a/packages/cisco_meraki_metrics/changelog.yml +++ b/packages/cisco_meraki_metrics/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.0" + changes: + - description: Add `network_health` data stream + type: enhancement + link: https://github.com/elastic/integrations/pull/999 - version: "0.4.1" changes: - description: scale values in device channel utilization so they display correctly as percentages. diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..a999d22fdf8 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs @@ -0,0 +1,22 @@ +metricsets: ["network_health"] +apiBaseUrl: {{apiBaseUrl}} +apiKey: {{apiKey}} +organizations: +{{#each organizations as |organization|}} + - "{{organization}}" +{{/each}} +period: {{period}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..a9620ec0706 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,66 @@ +--- +description: Pipeline for parsing Cisco Meraki Network Health metrics. +processors: + - script: + lang: painless + source: > + // some values have unit 'percent' in the mappings; we need to scale them down from 0->100 to 0->1. + // we round to 4 decimal places to avoid floating point errors. + + if (ctx.meraki != null) { + if (ctx.meraki.uplink != null && ctx.meraki.uplink.loss != null && ctx.meraki.uplink.loss.pct != null) { + ctx.meraki.uplink.loss.pct = Math.round((ctx.meraki.uplink.loss.pct / 100) * 10000) / 10000.0; + } + + if (ctx.meraki.device != null && ctx.meraki.device.channel_utilization != null) { + def wifi0 = ctx.meraki.device.channel_utilization["2_4"]; + def wifi1 = ctx.meraki.device.channel_utilization["5"]; + + if (wifi0 != null) { + if (wifi0.utilization_80211 != null) { + wifi0.utilization_80211 = Math.round((wifi0.utilization_80211 / 100) * 10000) / 10000.0; + } + if (wifi0.utilization_non_80211 != null) { + wifi0.utilization_non_80211 = Math.round((wifi0.utilization_non_80211 / 100) * 10000) / 10000.0; + } + if (wifi0.utilization_total != null) { + wifi0.utilization_total = Math.round((wifi0.utilization_total / 100) * 10000) / 10000.0; + } + } + + if (wifi1 != null) { + if (wifi1.utilization_80211 != null) { + wifi1.utilization_80211 = Math.round((wifi1.utilization_80211 / 100) * 10000) / 10000.0; + } + if (wifi1.utilization_non_80211 != null) { + wifi1.utilization_non_80211 = Math.round((wifi1.utilization_non_80211 / 100) * 10000) / 10000.0; + } + if (wifi1.utilization_total != null) { + wifi1.utilization_total = Math.round((wifi1.utilization_total / 100) * 10000) / 10000.0; + } + } + } + } + - convert: + field: meraki.uplink.rsrp + type: float + ignore_missing: true + - convert: + field: meraki.uplink.rsrq + type: float + ignore_missing: true + - rename: + field: meraki.device.channel_utilization.2_4 + target_field: meraki.device.channel_utilization.wifi0 + ignore_missing: true + - rename: + field: meraki.device.channel_utilization.5 + target_field: meraki.device.channel_utilization.wifi1 + ignore_missing: true +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml new file mode 100644 index 00000000000..b549d5382f3 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml @@ -0,0 +1,23 @@ +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + dimension: true + description: 'Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml new file mode 100644 index 00000000000..2e74d6d8da3 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml @@ -0,0 +1,30 @@ +- name: meraki + type: group + fields: + - name: organization_id + type: keyword + dimension: true + - name: network + type: group + fields: + - name: id + type: keyword + dimension: true + - name: name + type: keyword + dimension: true + - name: vpn_peers + type: group + fields: + - name: network_id + type: keyword + dimension: true + - name: network_name + type: keyword + dimension: true + - name: usage_summary.received.bytes + type: long + unit: byte + - name: usage_summary.sent.bytes + type: long + unit: byte \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml new file mode 100644 index 00000000000..5fad1940a6d --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml @@ -0,0 +1,44 @@ +title: "Cisco Meraki Network Health Metrics" +type: metrics +streams: + - input: meraki/metrics + title: Cisco Meraki Network Health Metrics + description: Collect network health metrics from the Meraki Dashboard API with Elastic Agent. + vars: + - name: apiKey + type: text + title: Meraki Dashboard API key + secret: true + required: true + - name: organizations + type: text + title: Meraki organization IDs + multi: true + required: true + - name: apiBaseUrl + type: url + title: Meraki Dashboard API base URL + default: api.meraki.com + - name: period + type: text + title: Collection interval + default: 60s + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - cisco_meraki_metrics-network_health + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. +elasticsearch: + source_mode: synthetic + index_mode: time_series diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json b/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json new file mode 100644 index 00000000000..57d11dc63f0 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json @@ -0,0 +1,76 @@ +{ + "@timestamp": "2024-09-30T16:55:38.202Z", + "agent": { + "ephemeral_id": "11855dde-6a4a-48ce-ac32-087b1c7999a3", + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.15.2" + }, + "data_stream": { + "dataset": "cisco_meraki_metrics.network_health", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "snapshot": true, + "version": "8.15.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "cisco_meraki_metrics.network_health", + "duration": 12982553765, + "ingested": "2024-09-30T16:56:01Z", + "module": "meraki" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "c7f0ac74f5e24f78942164132c2c8ead", + "ip": "172.21.0.4", + "mac": "02-42-AC-15-00-04", + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "6.8.0-45-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "meraki": { + "organization_id": "125432", + "network": { + "name": "BKYHUM", + "vpn_peers": [ + { + "network_id": "N_837204569103482715", + "network_name": "ZXVRNE", + "usage_summary.received.bytes": 12288, + "usage_summary.sent.bytes": 12288 + }, + { + "network_id": "N_294175608239471063", + "network_name": "QWMTJL", + "usage_summary.received.bytes": 0, + "usage_summary.sent.bytes": 79872 + } + ], + "id": "L_760194835627109284" + } + }, + "metricset": { + "name": "network_health", + "period": 60000 + }, + "service": { + "type": "meraki" + } +} \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/docs/README.md b/packages/cisco_meraki_metrics/docs/README.md index 9f49428bf64..ef462bc4bf1 100644 --- a/packages/cisco_meraki_metrics/docs/README.md +++ b/packages/cisco_meraki_metrics/docs/README.md @@ -305,3 +305,112 @@ An example event for `device_health` looks as following: } } ``` + +### Network Health + +The `network_health` dataset provides metrics related to the overall health and performance of Meraki networks. + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +**Exported fields** + +| Field | Description | Type | Unit | +|---|---|---|---| +| @timestamp | Event timestamp. | date | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host mac addresses. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| meraki.network.id | | keyword | | +| meraki.network.name | | keyword | | +| meraki.network.vpn_peers.network_id | | keyword | | +| meraki.network.vpn_peers.network_name | | keyword | | +| meraki.network.vpn_peers.usage_summary.received.bytes | | long | byte | +| meraki.network.vpn_peers.usage_summary.sent.bytes | | long | byte | +| meraki.organization_id | | keyword | | + + +An example event for `network_health` looks as following: + +```json +{ + "@timestamp": "2024-09-30T16:55:38.202Z", + "agent": { + "ephemeral_id": "11855dde-6a4a-48ce-ac32-087b1c7999a3", + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.15.2" + }, + "data_stream": { + "dataset": "cisco_meraki_metrics.network_health", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "snapshot": true, + "version": "8.15.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "cisco_meraki_metrics.network_health", + "duration": 12982553765, + "ingested": "2024-09-30T16:56:01Z", + "module": "meraki" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "c7f0ac74f5e24f78942164132c2c8ead", + "ip": "172.21.0.4", + "mac": "02-42-AC-15-00-04", + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "6.8.0-45-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "meraki": { + "organization_id": "125432", + "network": { + "name": "BKYHUM", + "vpn_peers": [ + { + "network_id": "N_837204569103482715", + "network_name": "ZXVRNE", + "usage_summary.received.bytes": 12288, + "usage_summary.sent.bytes": 12288 + }, + { + "network_id": "N_294175608239471063", + "network_name": "QWMTJL", + "usage_summary.received.bytes": 0, + "usage_summary.sent.bytes": 79872 + } + ], + "id": "L_760194835627109284" + } + }, + "metricset": { + "name": "network_health", + "period": 60000 + }, + "service": { + "type": "meraki" + } +} +``` diff --git a/packages/cisco_meraki_metrics/manifest.yml b/packages/cisco_meraki_metrics/manifest.yml index 282cd1a7356..5e686025277 100644 --- a/packages/cisco_meraki_metrics/manifest.yml +++ b/packages/cisco_meraki_metrics/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.2.0 name: cisco_meraki_metrics title: Cisco Meraki Metrics -version: 0.4.1 +version: 0.5.0 description: Collect metrics from Cisco Meraki with Elastic Agent. type: integration categories: From e2527cadd99f015372b63a2c58a6a7079a951b41 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Fri, 17 Oct 2025 10:32:36 +0300 Subject: [PATCH 02/13] bump kibana version --- packages/cisco_meraki_metrics/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cisco_meraki_metrics/manifest.yml b/packages/cisco_meraki_metrics/manifest.yml index 5e686025277..60f95042608 100644 --- a/packages/cisco_meraki_metrics/manifest.yml +++ b/packages/cisco_meraki_metrics/manifest.yml @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: "^8.15.2 || ^9.0.0" + version: "^9.1.0" elastic: subscription: "basic" screenshots: From a444bdb4dc0df9c6ae019fbc37f5f355c7022124 Mon Sep 17 00:00:00 2001 From: Gabriel Pop <94497545+gpop63@users.noreply.github.com> Date: Mon, 20 Oct 2025 10:46:43 +0300 Subject: [PATCH 03/13] Update packages/cisco_meraki_metrics/changelog.yml Co-authored-by: Mykola Kmet --- packages/cisco_meraki_metrics/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cisco_meraki_metrics/changelog.yml b/packages/cisco_meraki_metrics/changelog.yml index 50d8b2036c8..df496880ac3 100644 --- a/packages/cisco_meraki_metrics/changelog.yml +++ b/packages/cisco_meraki_metrics/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add `network_health` data stream type: enhancement - link: https://github.com/elastic/integrations/pull/999 + link: https://github.com/elastic/integrations/pull/15663 - version: "0.4.1" changes: - description: scale values in device channel utilization so they display correctly as percentages. From 145818cf3330440970ba04758d289dfcfcf80323 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 28 Oct 2025 11:36:43 +0200 Subject: [PATCH 04/13] fix api base url config param --- .../data_stream/network_health/agent/stream/stream.yml.hbs | 2 +- .../data_stream/network_health/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs index a999d22fdf8..2da0e9d9849 100644 --- a/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs +++ b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs @@ -1,5 +1,5 @@ metricsets: ["network_health"] -apiBaseUrl: {{apiBaseUrl}} +apiBaseURL: {{apiBaseUrl}} apiKey: {{apiKey}} organizations: {{#each organizations as |organization|}} diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml index 5fad1940a6d..b2fd63d56c2 100644 --- a/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml +++ b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml @@ -18,7 +18,7 @@ streams: - name: apiBaseUrl type: url title: Meraki Dashboard API base URL - default: api.meraki.com + default: https://api.meraki.com - name: period type: text title: Collection interval From 3f337b0a98ff7e0f0e84e415a8c7de05893448a8 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 29 Oct 2025 11:51:38 +0200 Subject: [PATCH 05/13] remove ingest pipeline --- .../elasticsearch/ingest_pipeline/default.yml | 66 ------------------- 1 file changed, 66 deletions(-) delete mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml deleted file mode 100644 index a9620ec0706..00000000000 --- a/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml +++ /dev/null @@ -1,66 +0,0 @@ ---- -description: Pipeline for parsing Cisco Meraki Network Health metrics. -processors: - - script: - lang: painless - source: > - // some values have unit 'percent' in the mappings; we need to scale them down from 0->100 to 0->1. - // we round to 4 decimal places to avoid floating point errors. - - if (ctx.meraki != null) { - if (ctx.meraki.uplink != null && ctx.meraki.uplink.loss != null && ctx.meraki.uplink.loss.pct != null) { - ctx.meraki.uplink.loss.pct = Math.round((ctx.meraki.uplink.loss.pct / 100) * 10000) / 10000.0; - } - - if (ctx.meraki.device != null && ctx.meraki.device.channel_utilization != null) { - def wifi0 = ctx.meraki.device.channel_utilization["2_4"]; - def wifi1 = ctx.meraki.device.channel_utilization["5"]; - - if (wifi0 != null) { - if (wifi0.utilization_80211 != null) { - wifi0.utilization_80211 = Math.round((wifi0.utilization_80211 / 100) * 10000) / 10000.0; - } - if (wifi0.utilization_non_80211 != null) { - wifi0.utilization_non_80211 = Math.round((wifi0.utilization_non_80211 / 100) * 10000) / 10000.0; - } - if (wifi0.utilization_total != null) { - wifi0.utilization_total = Math.round((wifi0.utilization_total / 100) * 10000) / 10000.0; - } - } - - if (wifi1 != null) { - if (wifi1.utilization_80211 != null) { - wifi1.utilization_80211 = Math.round((wifi1.utilization_80211 / 100) * 10000) / 10000.0; - } - if (wifi1.utilization_non_80211 != null) { - wifi1.utilization_non_80211 = Math.round((wifi1.utilization_non_80211 / 100) * 10000) / 10000.0; - } - if (wifi1.utilization_total != null) { - wifi1.utilization_total = Math.round((wifi1.utilization_total / 100) * 10000) / 10000.0; - } - } - } - } - - convert: - field: meraki.uplink.rsrp - type: float - ignore_missing: true - - convert: - field: meraki.uplink.rsrq - type: float - ignore_missing: true - - rename: - field: meraki.device.channel_utilization.2_4 - target_field: meraki.device.channel_utilization.wifi0 - ignore_missing: true - - rename: - field: meraki.device.channel_utilization.5 - target_field: meraki.device.channel_utilization.wifi1 - ignore_missing: true -on_failure: - - set: - field: event.kind - value: pipeline_error - - append: - field: error.message - value: '{{{ _ingest.on_failure_message }}}' From a6877bc9ff48cdeb7e9148d9b6f71bd25bf73819 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 29 Oct 2025 11:51:46 +0200 Subject: [PATCH 06/13] remove base fields --- .../network_health/fields/base-fields.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml deleted file mode 100644 index 7c798f4534c..00000000000 --- a/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml +++ /dev/null @@ -1,12 +0,0 @@ -- name: data_stream.type - type: constant_keyword - description: Data stream type. -- name: data_stream.dataset - type: constant_keyword - description: Data stream dataset. -- name: data_stream.namespace - type: constant_keyword - description: Data stream namespace. -- name: '@timestamp' - type: date - description: Event timestamp. From 78a9d04e3fb4f209f943fc196babc63e28885195 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 29 Oct 2025 11:52:07 +0200 Subject: [PATCH 07/13] add missing fields --- .../network_health/fields/fields.yml | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml index 2e74d6d8da3..870a5545c63 100644 --- a/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml @@ -27,4 +27,68 @@ unit: byte - name: usage_summary.sent.bytes type: long - unit: byte \ No newline at end of file + unit: byte + - name: jitter_summaries + type: group + fields: + - name: jitter_avg + type: double + description: Average jitter in milliseconds + unit: ms + - name: jitter_max + type: double + description: Maximum jitter in milliseconds + unit: ms + - name: jitter_min + type: double + description: Minimum jitter in milliseconds + unit: ms + - name: receiver_uplink + type: keyword + - name: sender_uplink + type: keyword + - name: latency_summaries + type: group + fields: + - name: latency_avg.ms + type: double + unit: ms + - name: latency_max.ms + type: double + unit: ms + - name: latency_min.ms + type: double + unit: ms + - name: receiver_uplink + type: keyword + - name: sender_uplink + type: keyword + - name: loss_percentage_summaries + type: group + fields: + - name: loss_avg.pct + type: double + unit: percent + - name: loss_max.pct + type: double + unit: percent + - name: loss_min.pct + type: double + unit: percent + - name: receiver_uplink + type: keyword + - name: sender_uplink + type: keyword + - name: mos_summaries + type: group + fields: + - name: mos_avg + type: double + - name: mos_max + type: double + - name: mos_min + type: double + - name: receiver_uplink + type: keyword + - name: sender_uplink + type: keyword \ No newline at end of file From 779c4c6766e2597129e216d0d910ce41ab1fb4e2 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 29 Oct 2025 12:36:22 +0200 Subject: [PATCH 08/13] Revert "remove base fields" This reverts commit a6877bc9ff48cdeb7e9148d9b6f71bd25bf73819. --- .../network_health/fields/base-fields.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. From b2050cac513d747c312dfaccc828aae816c1c29f Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 29 Oct 2025 12:37:20 +0200 Subject: [PATCH 09/13] update readme --- packages/cisco_meraki_metrics/docs/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/cisco_meraki_metrics/docs/README.md b/packages/cisco_meraki_metrics/docs/README.md index ef462bc4bf1..7cdb04a385a 100644 --- a/packages/cisco_meraki_metrics/docs/README.md +++ b/packages/cisco_meraki_metrics/docs/README.md @@ -327,6 +327,26 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | meraki.network.id | | keyword | | | meraki.network.name | | keyword | | +| meraki.network.vpn_peers.jitter_summaries.jitter_avg | Average jitter in milliseconds | double | ms | +| meraki.network.vpn_peers.jitter_summaries.jitter_max | Maximum jitter in milliseconds | double | ms | +| meraki.network.vpn_peers.jitter_summaries.jitter_min | Minimum jitter in milliseconds | double | ms | +| meraki.network.vpn_peers.jitter_summaries.receiver_uplink | | keyword | | +| meraki.network.vpn_peers.jitter_summaries.sender_uplink | | keyword | | +| meraki.network.vpn_peers.latency_summaries.latency_avg.ms | | double | ms | +| meraki.network.vpn_peers.latency_summaries.latency_max.ms | | double | ms | +| meraki.network.vpn_peers.latency_summaries.latency_min.ms | | double | ms | +| meraki.network.vpn_peers.latency_summaries.receiver_uplink | | keyword | | +| meraki.network.vpn_peers.latency_summaries.sender_uplink | | keyword | | +| meraki.network.vpn_peers.loss_percentage_summaries.loss_avg.pct | | double | percent | +| meraki.network.vpn_peers.loss_percentage_summaries.loss_max.pct | | double | percent | +| meraki.network.vpn_peers.loss_percentage_summaries.loss_min.pct | | double | percent | +| meraki.network.vpn_peers.loss_percentage_summaries.receiver_uplink | | keyword | | +| meraki.network.vpn_peers.loss_percentage_summaries.sender_uplink | | keyword | | +| meraki.network.vpn_peers.mos_summaries.mos_avg | | double | | +| meraki.network.vpn_peers.mos_summaries.mos_max | | double | | +| meraki.network.vpn_peers.mos_summaries.mos_min | | double | | +| meraki.network.vpn_peers.mos_summaries.receiver_uplink | | keyword | | +| meraki.network.vpn_peers.mos_summaries.sender_uplink | | keyword | | | meraki.network.vpn_peers.network_id | | keyword | | | meraki.network.vpn_peers.network_name | | keyword | | | meraki.network.vpn_peers.usage_summary.received.bytes | | long | byte | From 8d64761ca6583ef92c81bc584087d0a0c68aae3c Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 29 Oct 2025 12:40:29 +0200 Subject: [PATCH 10/13] fix fields --- .../data_stream/network_health/fields/fields.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml index 870a5545c63..90e5de7ceba 100644 --- a/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml @@ -33,16 +33,10 @@ fields: - name: jitter_avg type: double - description: Average jitter in milliseconds - unit: ms - name: jitter_max type: double - description: Maximum jitter in milliseconds - unit: ms - name: jitter_min type: double - description: Minimum jitter in milliseconds - unit: ms - name: receiver_uplink type: keyword - name: sender_uplink From 6b1945172a7039a7ef8cc8fff4116ac801d24126 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 29 Oct 2025 12:40:41 +0200 Subject: [PATCH 11/13] update readme --- packages/cisco_meraki_metrics/docs/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cisco_meraki_metrics/docs/README.md b/packages/cisco_meraki_metrics/docs/README.md index 7cdb04a385a..ac8d0e72dbc 100644 --- a/packages/cisco_meraki_metrics/docs/README.md +++ b/packages/cisco_meraki_metrics/docs/README.md @@ -327,9 +327,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | meraki.network.id | | keyword | | | meraki.network.name | | keyword | | -| meraki.network.vpn_peers.jitter_summaries.jitter_avg | Average jitter in milliseconds | double | ms | -| meraki.network.vpn_peers.jitter_summaries.jitter_max | Maximum jitter in milliseconds | double | ms | -| meraki.network.vpn_peers.jitter_summaries.jitter_min | Minimum jitter in milliseconds | double | ms | +| meraki.network.vpn_peers.jitter_summaries.jitter_avg | | double | | +| meraki.network.vpn_peers.jitter_summaries.jitter_max | | double | | +| meraki.network.vpn_peers.jitter_summaries.jitter_min | | double | | | meraki.network.vpn_peers.jitter_summaries.receiver_uplink | | keyword | | | meraki.network.vpn_peers.jitter_summaries.sender_uplink | | keyword | | | meraki.network.vpn_peers.latency_summaries.latency_avg.ms | | double | ms | From 00c2e22c96896cc470c04fc779fd0e191584de99 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 30 Oct 2025 14:12:31 +0200 Subject: [PATCH 12/13] remove redundant agent ecs fields --- .../network_health/fields/agent.yml | 23 ------------------- packages/cisco_meraki_metrics/docs/README.md | 3 --- 2 files changed, 26 deletions(-) delete mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml deleted file mode 100644 index b549d5382f3..00000000000 --- a/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml +++ /dev/null @@ -1,23 +0,0 @@ -- name: host - title: Host - group: 2 - description: 'A host is defined as a general computing instance. - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' - type: group - fields: - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - dimension: true - description: 'Name of the host. - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/docs/README.md b/packages/cisco_meraki_metrics/docs/README.md index ac8d0e72dbc..26cf15ebe04 100644 --- a/packages/cisco_meraki_metrics/docs/README.md +++ b/packages/cisco_meraki_metrics/docs/README.md @@ -322,9 +322,6 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host mac addresses. | keyword | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | meraki.network.id | | keyword | | | meraki.network.name | | keyword | | | meraki.network.vpn_peers.jitter_summaries.jitter_avg | | double | | From f2cbf11ad49b1c53e5ce6782b37203b9183e4f07 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Fri, 31 Oct 2025 12:47:53 +0200 Subject: [PATCH 13/13] Revert "remove redundant agent ecs fields" This reverts commit 00c2e22c96896cc470c04fc779fd0e191584de99. --- .../network_health/fields/agent.yml | 23 +++++++++++++++++++ packages/cisco_meraki_metrics/docs/README.md | 3 +++ 2 files changed, 26 insertions(+) create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml new file mode 100644 index 00000000000..b549d5382f3 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml @@ -0,0 +1,23 @@ +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + dimension: true + description: 'Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/docs/README.md b/packages/cisco_meraki_metrics/docs/README.md index 26cf15ebe04..ac8d0e72dbc 100644 --- a/packages/cisco_meraki_metrics/docs/README.md +++ b/packages/cisco_meraki_metrics/docs/README.md @@ -322,6 +322,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host mac addresses. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | meraki.network.id | | keyword | | | meraki.network.name | | keyword | | | meraki.network.vpn_peers.jitter_summaries.jitter_avg | | double | |