work in progress for #449, handle multiple netbox sites. upload and l…

…ogstash parts are mostly working

config/netbox-common.env.example

@@ -2,7 +2,8 @@
 #   can also be configured in the env_file files for netbox* services
 # Whether or not Logstash will enrich network traffic metadata via NetBox API calls
 NETBOX_ENRICHMENT=false
-# The name of the default "site" to be created upon NetBox initialization, and to be queried for enrichment
+# The name of the default "site" to be created upon NetBox initialization, and to be used for enrichment
+#   if not otherwise specified for the source of the network traffic data
 NETBOX_DEFAULT_SITE=Malcolm
 # Whether or not unobserved network entities in Logstash data will be used to populate NetBox
 NETBOX_AUTO_POPULATE=false

docs/asset-interaction-analysis.md

@@ -51,9 +51,7 @@ As Zeek logs and Suricata alerts are parsed and enriched (if the `NETBOX_ENRICHM
 
 For Malcolm's purposes, both physical devices and virtualized hosts will be stored as described above: the `device_type` field can be used to distinguish between them.
 
-NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/). Sites can have overlapping IP address ranges. The value of the `NETBOX_DEFAULT_SITE` variable in [environment variable in `netbox-common.env`](malcolm-config.md#MalcolmConfigEnvVars) will be used as a query parameter for these enrichment lookups.
-
-This feature was implemented as described in [idaholab/Malcolm#132](https://github.com/idaholab/Malcolm/issues/132).
+NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/). Sites can have overlapping IP address ranges. The site to associate with network traffic can be specified when [PCAP is uploaded](upload.md#Upload) or when configuring [live analysis](live-analysis.md#LiveAnalysis). If not otherwise specified, the value of the `NETBOX_DEFAULT_SITE` variable in [environment variable in `netbox-common.env`](malcolm-config.md#MalcolmConfigEnvVars) will be used for these enrichment lookups.
 
 ## <a name="NetBoxCompare"></a>Compare and highlight discrepancies between NetBox inventory and observed network traffic
 

docs/malcolm-config.md

@@ -54,7 +54,7 @@ Although the configuration script automates many of the following configuration
 * **`netbox-common.env`**, `netbox.env`, `netbox-secret.env`, `netbox-postgres.env`, `netbox-redis-cache.env` and `netbox-redis.env` - settings related to [NetBox](https://netbox.dev/) and [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis)
     - `NETBOX_DISABLED` - if set to `true`, Malcolm will **not** start and manage a [NetBox](asset-interaction-analysis.md#AssetInteractionAnalysis) instance (default `true`)
     - `NETBOX_ENRICHMENT` - if set to `true`, Logstash will [enrich network traffic metadata](asset-interaction-analysis.md#NetBoxEnrichment) via NetBox API calls
-    - `NETBOX_DEFAULT_SITE` - specifies the default NetBox [site name](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/) for use when [enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment) (default `Malcolm`)
+    - `NETBOX_DEFAULT_SITE` - specifies the default NetBox [site name](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/) for use when [enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment) if a specific site is not otherwise specified for the source of the data (default `Malcolm`)
     - `NETBOX_AUTO_POPULATE` - if set to `true`, Logstash will [populate the NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic
     - `NETBOX_AUTO_CREATE_PREFIX` - if set to `true`, Logstash will automatically create private subnet prefixes in the [NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic
     - `NETBOX_DEFAULT_AUTOCREATE_MANUFACTURER` - if set to `true`, new manufacturer entries will be created in the NetBox database when [matching device manufacturers to OUIs](asset-interaction-analysis.md#NetBoxPopPassiveOUIMatch) (default `true`)

file-upload/php/submit.php

@@ -93,6 +93,12 @@ function handle_transfer_ids_post($ids) {
         if (isset($_POST["tags"]) && (strlen($_POST["tags"]) > 0)) {
             $new_name_prefix = $_POST["tags"] . ",USERTAG,";
         }
+        if (isset($_POST["site-dropdown"]) && (strlen($_POST["site-dropdown"]) > 0) && ((is_int($_POST["site-dropdown"])) || (ctype_digit($_POST["site-dropdown"])))) {
+            if (strlen($new_name_prefix) > 0) {
+                $new_name_prefix = $new_name_prefix . ',';
+            }
+            $new_name_prefix = $new_name_prefix . 'NBSITEID' . $_POST["site-dropdown"] . ",";
+        }
 
         $files = $transfer->getFiles(defined('TRANSFER_PROCESSOR') ? TRANSFER_PROCESSOR : null);
         if($files != null){

file-upload/site/index.html

@@ -55,6 +55,14 @@
         .imgcenter {
             width: 50%;
         }
+        .site-dropdown {
+            display: none; /* hide initially */
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
+            border-radius: 12px;
+            border: 1px solid #909497;
+            font-size: 14px;
+            padding: 12px;
+        }
     </style>
 </head>
 <body>
@@ -63,7 +71,14 @@ <h1 class="panel-title">Network Traffic Artifact Upload</h1>
 
     <form id="commit_files" action="server/php/submit.php" method="post" enctype="multipart/form-data">
 
-        <input type="text" name="tags"/>
+        <div style="display: flex;">
+            <div style="flex-grow: 1;"><input type="text" name="tags"/></div>
+            <div>
+                <select name="site-dropdown" id="site-dropdown" class="site-dropdown" title="NetBox site">
+                    <!-- Options will be populated dynamically -->
+                </select>
+            </div>
+        </div>
         <br>
         <button type="submit" id="commit_button" class="button button_rounded">Commit Uploaded Files</button>
         <input type="file" name="filepond[]" multiple>
@@ -155,6 +170,58 @@ <h1 class="panel-title">Network Traffic Artifact Upload</h1>
             }
             setInterval(checkSubmitEnabler, 5000);
 
+            const siteDropdown = document.getElementById('site-dropdown');
+
+            // Function to populate the NetBox site dropdown
+            function populateSiteDropdown(data) {
+                // Clear any existing options
+                siteDropdown.innerHTML = '';
+
+                // Create and append options from the API result data
+                if (data.results && data.results.length > 0) {
+                    let lowestId = data.results[0].id;
+                    let lowestIdIndex = 0;
+
+                    // Find the index of the item with the lowest id
+                    data.results.forEach((item, index) => {
+                        if (item.id < lowestId) {
+                            lowestId = item.id;
+                            lowestIdIndex = index;
+                        }
+                    });
+
+                    // Create and append options from the data
+                    data.results.forEach((item, index) => {
+                        const option = document.createElement('option');
+                        option.value = item.id;
+                        option.textContent = item.display;
+                        // Automatically select the option with the lowest id
+                        if (index === lowestIdIndex) {
+                            option.selected = true;
+                        }
+                        siteDropdown.appendChild(option);
+                    });
+
+                    // Make the dropdown visible
+                    siteDropdown.style.display = 'block';
+
+                } else {
+                    // Hide the dropdown if no results
+                    siteDropdown.style.display = 'none';
+                }
+            }
+
+            // Fetch data from the NetBox dcim/sites REST API
+            fetch('/netbox/api/dcim/sites/?format=json')
+                .then(response => response.json())
+                .then(data => {
+                    populateSiteDropdown(data);
+                })
+                .catch(error => {
+                    console.error('Error fetching NetBox sites:', error);
+                    // Hide the dropdown on error
+                    siteDropdown.style.display = 'none';
+                });
 
         })
     </script>

filebeat/scripts/filebeat-process-zeek-folder.sh

@@ -11,7 +11,7 @@
 #   - are not in use (fuser -s)
 # 1. move file to processed/ (preserving original subdirectory heirarchy, if any)
 # 2. calculate tags based on splitting the file path and filename (splitting on
-#    on [, -/_])
+#    on ",-/_.")
 
 FILEBEAT_PREPARE_PROCESS_COUNT=${FILEBEAT_PREPARE_PROCESS_COUNT:-1}
 
@@ -79,6 +79,7 @@ if mkdir $LOCKDIR; then
         USERTAG=false
 
         TAGS=()
+        TAGS_PRESERVED=()
         IFS=",-/_." read -r -a SOURCESPLIT <<< $(echo "$FILENAME" | sed "s/\.[^.]*$//")
         echo "\"$FILENAME\" -> \"${DESTNAME}\""
         for index in "${!SOURCESPLIT[@]}"
@@ -87,14 +88,17 @@ if mkdir $LOCKDIR; then
           if ! in_array TAGS "$TAG_CANDIDATE"; then
             if [[ "$TAG_CANDIDATE" = "USERTAG" ]]; then
               USERTAG=true
-            elif [[ -n $TAG_CANDIDATE && ! $TAG_CANDIDATE =~ ^[0-9-]+$ && $TAG_CANDIDATE != "tar" && $TAG_CANDIDATE != "AUTOZEEK" && ! $TAG_CANDIDATE =~ ^AUTOCARVE ]]; then
+            elif [[ -n "$TAG_CANDIDATE" && ! "$TAG_CANDIDATE" =~ ^[0-9-]+$ && "$TAG_CANDIDATE" != "tar" && "$TAG_CANDIDATE" != "AUTOZEEK" && ! "$TAG_CANDIDATE" =~ ^AUTOCARVE ]]; then
               TAGS+=("${TAG_CANDIDATE}")
+              if [[ "$TAG_CANDIDATE" =~ ^NBSITEID[0-9-]+$ ]]; then
+                TAGS_PRESERVED+=("${TAG_CANDIDATE}")
+              fi
             fi
           fi
         done
 
         if [[ "$ZEEK_LOG_AUTO_TAG" != "true" ]] && [[ "$USERTAG" != "true" ]]; then
-          TAGS=()
+          TAGS=("${TAGS_PRESERVED[@]}")
         fi
 
         mkdir -p "$DESTDIR"

logstash/pipelines/beats/11_beats_logs.conf

@@ -913,8 +913,10 @@ filter {
       code => "
         if fileParts = event.get('[log][file][path]').split('/').last.match(/^(?:.*?)(?:\((.*)\))?(?:\.\d{4}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2})?\.evtx\.json/i) then
           filenameTags = fileParts.captures[0].split(',')
-          filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(evtx|json)s?\s*\z/i))}
+          nbsiteid = filenameTags.find { |str| str[/NBSITEID(\d+)/] }[/NBSITEID(\d+)/, 1].to_i rescue nil
+          filenameTags.delete_if{|v| ((v == nil) or (v == '')  or (v !~ /\D/) or (v =~ /\A\s*NBSITEID/) or (v =~ /\A\s*(evtx|json)s?\s*\z/i))}
           event.set('[@metadata][evtx_log_tags]', filenameTags.uniq) unless (filenameTags.length == 0)
+          event.set('[@metadata][nbsiteid]', nbsiteid) unless (nbsiteid.to_i == 0)
         end"
     }
     if ([@metadata][evtx_log_tags]) { mutate { id => "mutate_merge_evtx_log_tags"

logstash/pipelines/enrichment/21_netbox.conf

@@ -51,6 +51,7 @@ filter {
           "enabled_env" => "NETBOX_ENRICHMENT"
           "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+          "lookup_site_id" => "[@metadata][nbsiteid]"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
           "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -76,6 +77,7 @@ filter {
             "enabled_env" => "NETBOX_ENRICHMENT"
             "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
             "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+            "lookup_site_id" => "[@metadata][nbsiteid]"
             "lookup_site_env" => "NETBOX_DEFAULT_SITE"
             "netbox_token_env" => "SUPERUSER_API_TOKEN"
             "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -100,6 +102,7 @@ filter {
               "enabled_env" => "NETBOX_ENRICHMENT"
               "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
               "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+              "lookup_site_id" => "[@metadata][nbsiteid]"
               "lookup_site_env" => "NETBOX_DEFAULT_SITE"
               "netbox_token_env" => "SUPERUSER_API_TOKEN"
               "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -121,6 +124,7 @@ filter {
               "enabled_env" => "NETBOX_ENRICHMENT"
               "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
               "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+              "lookup_site_id" => "[@metadata][nbsiteid]"
               "lookup_site_env" => "NETBOX_DEFAULT_SITE"
               "netbox_token_env" => "SUPERUSER_API_TOKEN"
               "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -148,6 +152,7 @@ filter {
             "enabled_env" => "NETBOX_ENRICHMENT"
             "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
             "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+            "lookup_site_id" => "[@metadata][nbsiteid]"
             "lookup_site_env" => "NETBOX_DEFAULT_SITE"
             "netbox_token_env" => "SUPERUSER_API_TOKEN"
             "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -169,6 +174,7 @@ filter {
             "enabled_env" => "NETBOX_ENRICHMENT"
             "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
             "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+            "lookup_site_id" => "[@metadata][nbsiteid]"
             "lookup_site_env" => "NETBOX_DEFAULT_SITE"
             "netbox_token_env" => "SUPERUSER_API_TOKEN"
             "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -198,6 +204,7 @@ filter {
           "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
           "auto_prefix_env" => "NETBOX_AUTO_CREATE_PREFIX"
+          "lookup_site_id" => "[@metadata][nbsiteid]"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
           "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -217,6 +224,7 @@ filter {
           "enabled_env" => "NETBOX_ENRICHMENT"
           "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+          "lookup_site_id" => "[@metadata][nbsiteid]"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
           "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -243,6 +251,7 @@ filter {
           "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
           "auto_prefix_env" => "NETBOX_AUTO_CREATE_PREFIX"
+          "lookup_site_id" => "[@metadata][nbsiteid]"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "netbox_token_env" => "SUPERUSER_API_TOKEN"
           "cache_size_env" => "NETBOX_CACHE_SIZE"
@@ -262,6 +271,7 @@ filter {
           "enabled_env" => "NETBOX_ENRICHMENT"
           "verbose_env" => "NETBOX_ENRICHMENT_VERBOSE"
           "debug_env" => "NETBOX_ENRICHMENT_DEBUG"
+          "lookup_site_id" => "[@metadata][nbsiteid]"
           "lookup_site_env" => "NETBOX_DEFAULT_SITE"
           "lookup_service_env" => "NETBOX_ENRICHMENT_LOOKUP_SERVICE"
           "lookup_service_port_source" => "[destination][port]"

logstash/pipelines/suricata/11_suricata_logs.conf

@@ -17,8 +17,10 @@ filter {
               tags = fileParts.captures[0]
               unless tags.nil?
                 filenameTags = tags.split(',')
-                filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(eve|pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i))}
+                nbsiteid = filenameTags.find { |str| str[/NBSITEID(\d+)/] }[/NBSITEID(\d+)/, 1].to_i rescue nil
+                filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*NBSITEID/) or (v =~ /\A\s*(eve|pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i))}
                 event.set('[@metadata][suricata_log_tags]', filenameTags.uniq) unless (filenameTags.length == 0)
+                event.set('[@metadata][nbsiteid]', nbsiteid) unless (nbsiteid.to_i == 0)
               end
             end"
         }

logstash/pipelines/zeek/10_zeek_prep.conf

@@ -38,8 +38,10 @@ filter {
       id => "ruby_zeek_prune_tags"
       code => "
         filenameTags = event.get('[@metadata][zeek_log_tags]').split(',')
-        filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))}
+        nbsiteid = filenameTags.find { |str| str[/NBSITEID(\d+)/] }[/NBSITEID(\d+)/, 1].to_i rescue nil
+        filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*NBSITEID/i) or (v =~ /\A\s*(autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))}
         event.set('[@metadata][zeek_log_tags]', filenameTags.uniq) unless (filenameTags.length == 0)
+        event.set('[@metadata][nbsiteid]', nbsiteid) unless (nbsiteid.to_i == 0)
       "
     }
     if ([@metadata][zeek_log_tags]) { mutate { id => "mutate_merge_zeek_log_tags"