Merge pull request #1 from splunk/event-hubs

Merge Event Hub functions

Author: JasonConger

.gitignore

@@ -91,4 +91,7 @@ out
 bin
 obj
 appsettings.json
-local.settings.json
+local.settings.json
+
+# Workstation
+.DS_Store

.vscode/settings.json

@@ -1,8 +1,9 @@
 {
-    "azureFunctions.deploySubpath": "graph",
     "azureFunctions.postDeployTask": "npm install",
     "azureFunctions.projectLanguage": "JavaScript",
     "azureFunctions.projectRuntime": "~3",
     "debug.internalConsoleOptions": "neverOpen",
-    "azureFunctions.preDeployTask": "npm prune"
+    "azureFunctions.preDeployTask": "npm prune",
+    "azureFunctions.deploySubpath": "event-hubs-hec",
+    "azureFunctions.projectSubpath": "event-hubs-hec"
 }

.vscode/tasks.json

@@ -4,19 +4,19 @@
 		{
 			"type": "func",
 			"command": "host start",
-			"problemMatcher": "$func-watch",
+			"problemMatcher": "$func-node-watch",
 			"isBackground": true,
 			"dependsOn": "npm install",
 			"options": {
-				"cwd": "${workspaceFolder}/graph"
+				"cwd": "${workspaceFolder}/event-hubs-hec"
 			}
 		},
 		{
 			"type": "shell",
 			"label": "npm install",
 			"command": "npm install",
 			"options": {
-				"cwd": "${workspaceFolder}/graph"
+				"cwd": "${workspaceFolder}/event-hubs-hec"
 			}
 		},
 		{
@@ -25,7 +25,7 @@
 			"command": "npm prune --production",
 			"problemMatcher": [],
 			"options": {
-				"cwd": "${workspaceFolder}/graph"
+				"cwd": "${workspaceFolder}/event-hubs-hec"
 			}
 		}
 	]

README.md

@@ -11,7 +11,23 @@ Each available set of functions in this repository is contained within its own f
 | Functions | Location | Description |
 | --------- | -------- | ----------- |
 | Microsoft Teams | [graph](https://github.com/splunk/azure-functions-splunk/tree/master/graph) | Collects [Microsoft Teams call records]( https://docs.microsoft.com/en-us/graph/api/resources/callrecords-callrecord).  This data can be used with the [Microsoft 365 App for Splunk]( https://splunkbase.splunk.com/app/3786/) and/or the [RWI – Executive Dashboard]( https://splunkbase.splunk.com/app/4952/) |
-
+| Azure Event Hubs | [event-hubs-hec](https://github.com/splunk/azure-functions-splunk/tree/master/event-hubs) | These Azure Functions are triggered by events arriving on an Azure Event Hub.  The functions then process the events and send the event to a listening Splunk HTTP Event Collector |
+
+## Setting a Project Subpath
+[Multiple Azure Function projects](https://github.com/Microsoft/vscode-azurefunctions/wiki/Multiple-function-projects) exist in this repository.  In order to debug a specific function project, set the `azureFunctions.deploySubpath` and `azureFunctions.projectSubpath` parameters in `settings.json` to the appropriate path.
+
+For example, to run and debug the `Graph` functions use the following `settings.json`
+```
+{
+    "azureFunctions.postDeployTask": "npm install",
+    "azureFunctions.projectLanguage": "JavaScript",
+    "azureFunctions.projectRuntime": "~3",
+    "debug.internalConsoleOptions": "neverOpen",
+    "azureFunctions.preDeployTask": "npm prune",
+    "azureFunctions.deploySubpath": "graph",
+    "azureFunctions.projectSubpath": "graph"
+}
+```
 
 ## Support
 

event-hubs-hec/.funcignore

@@ -0,0 +1,7 @@
+*.js.map
+*.ts
+.git*
+.vscode
+local.settings.json
+test
+tsconfig.json

event-hubs-hec/.gitignore

@@ -0,0 +1,94 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build output
+.nuxt
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TypeScript output
+dist
+out
+
+# Azure Functions artifacts
+bin
+obj
+appsettings.json
+local.settings.json

event-hubs-hec/README.md

@@ -0,0 +1,80 @@
+# Azure Functions for Sending Event Hub data to a Splunk HTTP Event Collector
+Events arriving on an Azure Event Hub can trigger serverless Azure Functions.  Azure Functions can further process the raw events in near real-time.
+
+<a href="https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fsplunk%2Fazure-functions-splunk%2Fevent-hubs%2Fevent-hubs-hec%2Fdeploy%2FazureDeploy.json/createUIDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2Fsplunk%2Fazure-functions-splunk%2Fevent-hubs%2Fevent-hubs-hec%2Fdeploy%2FazureDeploy.portal.json" target="_blank">
+<img src="https://aka.ms/deploytoazurebutton"/>
+</a>
+
+This repository contains a collection of Azure Functions for:
+* Processing events as they arrive on an Event Hub
+* Separating batched events (events in a `records[]` array) into individual events
+* Formatting events in the `event` format for a Splunk HTTP Event Collector
+* Sending event data to Splunk via [HTTP Event Collector](https://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector)
+* Writing event data to a Storage Blob if data cannot successfully be sent to Splunk
+  * The [Splunk Add-on for Microsoft Cloud Services](https://splunkbase.splunk.com/app/3110/) can be utilized to retrieve Storage Blob data
+
+## Getting Started
+
+### 1. Create an HTTP Event Collector token in your Spunk Environment
+An HTTP Event Collector receives data pushed from the Azure Functions.  Refer to the Splunk documentation for [setting up an HTTP Event Collector input](https://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector) in your Splunk Enterprise or Splunk Cloud environment.
+
+### 2. Create an Event Hub Namespace
+An Event Hub Namespace will contain one or more Event Hubs.  Refer to the Microsoft documentation for [Event Hub Namespace setup instructions](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create).
+
+### 3. Send data to an Event Hub
+Microsoft Azure uses [diagnostics settings](https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/diagnostic-settings) to define data export and destination rules.  Each resource to be monitored must have a diagnostic setting.  Diagnostic settings can be defined using the Azure portal, PowerShell, [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/monitor/diagnostic-settings?view=azure-cli-latest), [Resource Manager templates](https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/resource-manager-diagnostic-settings), REST API, or an Azure Policy.
+
+* [Sending Azure Activity log data to an Event Hub using the Azure Portal walkthrough](docs/activity_log_diagnostic_settings.md)
+* [Sending Azure Diagnostic logs and metrics to an Event Hub](https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/diagnostic-settings)
+* [Sending Azure Active Directory logs to an Event Hub](docs/azure_ad_diagnostic_settings.md)
+* Sending Azure Virtual Machine data to an Event Hub
+  * [Windows VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-windows)
+  * [Linux VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux)
+
+
+### 4. Deploy the functions to Azure
+
+Use the "Deploy to Azure" button above to deploy the Azure Functions from this repo to your Azure account.  During setup, you will be prompted for the following information:
+
+* Event Hub Namespace
+* Event Hub consumer group for each hub monitored
+* Splunk sourcetype or sourcetype base for each hub monitored
+  * Note: see section below about sourcetypes
+* Splunk [HTTP Event Collector](https://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector) Endpoint
+* Splunk [HTTP Event Collector](https://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector) Token
+
+## Splunk Sourcetypes
+### Azure Active Directory Sourcetypes
+Functions that collect Azure Active Directory data use a sourcetype base.  The category of the Azure Active Directory event is appended to the sourcetype base to construct the full sourcetype.
+
+**Example**
+
+The default sourcetype base for Azure Active Directory Sign-in and Audit events is `azure:aad`
+
+A sign-in event with a category of `SignInLogs` will have a sourcetype of `azure:aad:signinlogs`
+
+An audit event with a category of `AuditLogs` will have a sourcetype of `azure:aad:auditlogs`
+
+### Diagnostic Logs
+Functions that collect diagnostic log data attempt to construct a sourcetype based on the `resourceId` of the event.  The logic for this sourcetype construction can be found in the `getSourceType` function in the [./helpers/splunk.js file](helpers/splunk.js).  The following steps are used to construct the sourcetype:
+
+* A regular expression is used to extract two groups after the text `/PROVIDERS`
+  * Example `/PROVIDERS/`**`MICROSOFT.RESOURCES/DEPLOYMENTS/`**
+* Periods (`.`) and forward slashes (`/`) are replaced with colons (`:`)
+* The event category is appended
+
+**Example**
+
+An event with a `resourceId` of `/SUBSCRIPTIONS/subscription ID/RESOURCEGROUPS/group/PROVIDERS/MICROSOFT.RESOURCES/DEPLOYMENTS/FAILURE-ANOMALIES-ALERT-RULE-DEPLOYMENT-12345678` will have a sourcetype of `azure:resources:deployments:administrative`
+
+If a sourcetype cannot be constructed from the event, the specified default sourcetype entered at setup will be used.
+
+
+## Securing Azure Function settings
+Microsoft stores the above values as [application settings](https://docs.microsoft.com/en-us/azure/azure-functions/functions-how-to-use-azure-function-app-settings#settings). These settings are stored encrypted, but you may opt to transfer one or more of these settings to a Key Vault. Refer to the following documentation for details on this procedure:
+
+* [Use Key Vault references for App Service and Azure Functions](https://docs.microsoft.com/en-us/azure/app-service/app-service-key-vault-references)
+
+
+## Support
+This software is released as-is. Splunk provides no warranty and no support on this software. If you have any issues with the software, please file an issue on the repository.

event-hubs-hec/aad-logs/function.json

@@ -0,0 +1,21 @@
+{
+  "bindings": [
+    {
+      "type": "eventHubTrigger",
+      "name": "eventHubMessages",
+      "direction": "in",
+      "eventHubName": "%AAD_LOG_HUB_NAME%",
+      "connection": "EVENTHUB_CONNECTION_STRING",
+      "cardinality": "many",
+      "consumerGroup": "%AAD_LOG_CONSUMER_GROUP%",
+      "dataType": "string"
+    },
+    {
+      "name": "outputBlob",
+      "type": "blob",
+      "path": "undeliverable-events/%AAD_LOG_SOURCETYPE%-{rand-guid}",
+      "connection": "BLOB_CONNECTION_STRING",
+      "direction": "out"
+    }
+  ]
+}

event-hubs-hec/aad-logs/index.js

@@ -0,0 +1,30 @@
+/*
+Copyright 2020 Splunk Inc. 
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+const splunk = require('../helpers/splunk');
+module.exports = async function (context, eventHubMessages) {
+    
+    for (const event of eventHubMessages) {
+        await splunk
+                .sendToHEC(event, process.env["AAD_LOG_SOURCETYPE"])
+                .catch(err => {
+                    context.log.error(`Error posting to Splunk HTTP Event Collector: ${err}`);
+                    
+                    // If the event was not successfully sent to Splunk, drop the event in a storage blob
+                    context.bindings.outputBlob = event;
+                })
+    }
+    context.done();
+};

event-hubs-hec/aad-signin-logs-non-interactive/function.json

@@ -0,0 +1,21 @@
+{
+  "bindings": [
+    {
+      "type": "eventHubTrigger",
+      "name": "eventHubMessages",
+      "direction": "in",
+      "eventHubName": "%AAD_NON_INTERACTIVE_SIGNIN_LOG_HUB_NAME%",
+      "connection": "EVENTHUB_CONNECTION_STRING",
+      "cardinality": "many",
+      "consumerGroup": "%AAD_NON_INTERACTIVE_SIGNIN_LOG_CONSUMER_GROUP%",
+      "dataType": "string"
+    },
+    {
+      "name": "outputBlob",
+      "type": "blob",
+      "path": "undeliverable-events/%AAD_NON_INTERACTIVE_SIGNIN_LOG_SOURCETYPE%-{rand-guid}",
+      "connection": "BLOB_CONNECTION_STRING",
+      "direction": "out"
+    }
+  ]
+}

event-hubs-hec/aad-signin-logs-non-interactive/index.js

@@ -0,0 +1,30 @@
+/*
+Copyright 2020 Splunk Inc. 
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+const splunk = require('../helpers/splunk');
+module.exports = async function (context, eventHubMessages) {
+    
+    for (const event of eventHubMessages) {
+        await splunk
+                .sendToHEC(event, process.env["AAD_NON_INTERACTIVE_SIGNIN_LOG_SOURCETYPE"])
+                .catch(err => {
+                    context.log.error(`Error posting to Splunk HTTP Event Collector: ${err}`);
+                    
+                    // If the event was not successfully sent to Splunk, drop the event in a storage blob
+                    context.bindings.outputBlob = event;
+                })
+    }
+    context.done();
+};

event-hubs-hec/aad-signin-logs-service-principal/function.json

@@ -0,0 +1,21 @@
+{
+  "bindings": [
+    {
+      "type": "eventHubTrigger",
+      "name": "eventHubMessages",
+      "direction": "in",
+      "eventHubName": "%AAD_SERVICE_PRINCIPAL_SIGNIN_LOG_HUB_NAME%",
+      "connection": "EVENTHUB_CONNECTION_STRING",
+      "cardinality": "many",
+      "consumerGroup": "%AAD_SERVICE_PRINCIPAL_SIGNIN_LOG_CONSUMER_GROUP%",
+      "dataType": "string"
+    },
+    {
+      "name": "outputBlob",
+      "type": "blob",
+      "path": "undeliverable-events/%AAD_SERVICE_PRINCIPAL_SIGNIN_LOG_SOURCETYPE%-{rand-guid}",
+      "connection": "BLOB_CONNECTION_STRING",
+      "direction": "out"
+    }
+  ]
+}