Integrating fluent-bit input plugins for Linux/Windows (#1144)

* Implement input plugin setup + perf and containerinventory plugins (#1112)

* migrate perf and containterinventory plugins

---------

Co-authored-by: Amol Agrawal <[email protected]>

* add var to livenessprobe (#1116)

Co-authored-by: Amol Agrawal <[email protected]>

* Add perf and container inventory plugins for Windows and integrating them to named pipe (#1113)

Add perf and container inventory plugins for Windows and integrating them to named pipe (#1113)

* Resolving comments

* Solving a bug

* Using the new input plugins only for windows msi mode

* Remove the duplicate flow of logs

* Not update the config files when legacy and resource optimization is on

* Not updating the config map for geneva as well

* Solving timestamp bug and add windows_rs.conf files

* Var name change bug

* Use the stat of a file instead of reading from file

* address PR comments - I

* update fluent-bit to 2.1.10

* update trivyignore

* fix bug in confg parsing

* add shared library

* revert fluent-bit upgrade in windows

* Adding a configmap option for disabling windows fluent-bit

* Adding a configmap option for disabling windows fluent-bit

* Solving a bug

* Resolving comments

* Solving RS bug

* Solving a bug

* Resolving comments

* remove linux option from cm

* Update trivyignore

* Sign the containerinventory.so and perf.so files

* Signing windows binary files

* Update the go mod tidy for the ci_prod changes

* Update go.mod to 1.21

* bug

* Update fluent-bit conf

* Update the fluent-bit conf file

* fluentd starting two times

---------

Co-authored-by: Amol Agrawal <[email protected]>
Co-authored-by: Amol Agrawal <[email protected]>
Co-authored-by: Janvi Jatakia (from Dev Box) <[email protected]>

.gitignore

@@ -29,4 +29,6 @@ source/plugins/go/src/*.h
 *_mock.go
 *_log.txt
 *.log
-*.byebug_history
+*.byebug_history
+go.work.sum
+go.work

.pipelines/azure_pipeline_mergedbranches.yaml

@@ -355,6 +355,12 @@ jobs:
         echo "Extract out oms"
         docker cp signingContainer:C:\opt\amalogswindows\out_oms.so .
 
+        echo "Extract containerinventory.so"
+        docker cp signingContainer:C:\opt\fluent-bit\bin\containerinventory.so .
+
+        echo "Extract perf.so"
+        docker cp signingContainer:C:\opt\fluent-bit\bin\perf.so .
+
         echo "Removing container..."
         docker rm signingContainer
 
@@ -558,6 +564,12 @@ jobs:
         echo "Extract out oms"
         docker cp signingContainer:C:\opt\amalogswindows\out_oms.so .
 
+        echo "Extract containerinventory.so"
+        docker cp signingContainer:C:\opt\fluent-bit\bin\containerinventory.so .
+
+        echo "Extract perf.so"
+        docker cp signingContainer:C:\opt\fluent-bit\bin\perf.so .
+
         echo "Removing container..."
         docker rm signingContainer
 

.trivyignore

@@ -13,6 +13,7 @@ GHSA-m425-mq94-257g
 CVE-2023-46129
 CVE-2023-47090
 CVE-2024-21626
+CVE-2023-50658
 
 # ruby HIGH
 CVE-2017-10784

build/common/installer/conf/azm-containers-input-plugins.conf

@@ -0,0 +1,5 @@
+[PLUGINS]
+    Path /opt/fluent-bit/bin/containerinventory.so
+
+[PLUGINS]
+    Path /opt/fluent-bit/bin/perf.so

build/common/installer/scripts/fluent-bit-conf-customizer.rb

@@ -5,11 +5,24 @@
 @fluent_bit_common_conf_path = "/etc/opt/microsoft/docker-cimprov/fluent-bit-common.conf"
 
 @os_type = ENV["OS_TYPE"]
+@isWindows = false
 if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
+  @isWindows = true
   @fluent_bit_conf_path = "/etc/fluent-bit/fluent-bit.conf"
   @fluent_bit_common_conf_path = "/etc/fluent-bit/fluent-bit-common.conf"
 end
 
+@using_aad_msi_auth = false
+if !ENV["USING_AAD_MSI_AUTH"].nil? && !ENV["USING_AAD_MSI_AUTH"].empty? && ENV["USING_AAD_MSI_AUTH"].strip.casecmp("true") == 0
+  @using_aad_msi_auth = true
+end
+
+@geneva_logs_integration = false
+if !ENV["GENEVA_LOGS_INTEGRATION"].nil? && !ENV["GENEVA_LOGS_INTEGRATION"].empty? && ENV["GENEVA_LOGS_INTEGRATION"].strip.casecmp("true") == 0
+  @geneva_logs_integration = true
+end
+
+
 @default_service_interval = "15"
 @default_mem_buf_limit = "10"
 
@@ -35,6 +48,21 @@ def substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
     return new_contents
 end
 
+def substituteResourceOptimization(resourceOptimizationEnabled, new_contents)
+  #Update the config file only in two conditions: 1. Linux and resource optimization is enabled 2. Windows and using aad msi auth and not using geneva logs integration
+  if (!@isWindows && !resourceOptimizationEnabled.nil? && resourceOptimizationEnabled.to_s.downcase == "true") || (@isWindows && @using_aad_msi_auth && !@geneva_logs_integration)
+    puts "config::Starting to substitute the placeholders in fluent-bit.conf file for resource optimization"
+    if (@isWindows)
+      new_contents = new_contents.gsub("#${ResourceOptimizationPluginFile}", "plugins_file  /etc/fluent-bit/azm-containers-input-plugins.conf")
+    else
+      new_contents = new_contents.gsub("#${ResourceOptimizationPluginFile}", "plugins_file  /etc/opt/microsoft/docker-cimprov/azm-containers-input-plugins.conf")
+    end
+    new_contents = new_contents.gsub("#${ResourceOptimizationFBConfigFile}", "@INCLUDE fluent-bit-input.conf")
+  end
+
+  return new_contents
+end
+
 def substituteFluentBitPlaceHolders
   begin
     # Replace the fluentbit config file with custom values if present
@@ -47,6 +75,8 @@ def substituteFluentBitPlaceHolders
     ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"]
     multilineLogging = ENV["AZMON_MULTILINE_ENABLED"]
     stacktraceLanguages = ENV["AZMON_MULTILINE_LANGUAGES"]
+    resourceOptimizationEnabled = ENV["AZMON_RESOURCE_OPTIMIZATION_ENABLED"]
+    windowsFluentBitDisabled = ENV["AZMON_WINDOWS_FLUENT_BIT_DISABLED"]
 
     serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
@@ -84,6 +114,10 @@ def substituteFluentBitPlaceHolders
     end
 
     new_contents = substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
+
+    if !@isWindows || (@isWindows && !windowsFluentBitDisabled.nil? && windowsFluentBitDisabled.to_s.downcase == "false")
+      new_contents = substituteResourceOptimization(resourceOptimizationEnabled, new_contents)
+    end
     File.open(@fluent_bit_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in fluent-bit.conf file"
 

build/common/installer/scripts/tomlparser-agent-config.rb

@@ -96,6 +96,8 @@
 @ignoreProxySettings = false
 
 @multiline_enabled = "false"
+@resource_optimization_enabled = false
+@windows_fluent_bit_disabled = false
 
 @waittime_port_25226 = 45
 @waittime_port_25228 = 120
@@ -351,6 +353,26 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         puts "Using config map value: AZMON_MULTILINE_ENABLED = #{@multiline_enabled}"
       end
 
+      if !@controllerType.nil? && !@controllerType.empty? && @controllerType.strip.casecmp(@daemonset) == 0 && @containerType.nil?
+        resource_optimization_config = parsedConfig[:agent_settings][:resource_optimization]
+        if !resource_optimization_config.nil?
+          resource_optimization_enabled = resource_optimization_config[:enabled]
+          if !resource_optimization_enabled.nil? && (!!resource_optimization_enabled == resource_optimization_enabled) #Checking for Boolean type, since 'Boolean' is not defined as a type in ruby
+            @resource_optimization_enabled = resource_optimization_enabled
+          end
+          puts "Using config map value: AZMON_RESOURCE_OPTIMIZATION_ENABLED = #{@resource_optimization_enabled}"
+        end
+      end
+
+      windows_fluent_bit_config = parsedConfig[:agent_settings][:windows_fluent_bit]
+      if !windows_fluent_bit_config.nil?
+        windows_fluent_bit_disabled = windows_fluent_bit_config[:disabled]
+        if !windows_fluent_bit_disabled.nil? && windows_fluent_bit_disabled.downcase == "true"
+          @windows_fluent_bit_disabled = true
+        end
+        puts "Using config map value: AZMON_WINDOWS_FLUENT_BIT_DISABLED = #{@windows_fluent_bit_disabled}"
+      end
+
       network_listener_waittime_config = parsedConfig[:agent_settings][:network_listener_waittime]
       if !network_listener_waittime_config.nil?
         waittime = network_listener_waittime_config[:tcp_port_25226]
@@ -479,6 +501,9 @@ def populateSettingValuesFromConfigMap(parsedConfig)
     file.write("export AZMON_MULTILINE_ENABLED=#{@multiline_enabled}\n")
   end
 
+  file.write("export AZMON_RESOURCE_OPTIMIZATION_ENABLED=#{@resource_optimization_enabled}\n")
+  file.write("export AZMON_WINDOWS_FLUENT_BIT_DISABLED=#{@windows_fluent_bit_disabled}\n")
+
   file.write("export WAITTIME_PORT_25226=#{@waittime_port_25226}\n")
   file.write("export WAITTIME_PORT_25228=#{@waittime_port_25228}\n")
   file.write("export WAITTIME_PORT_25229=#{@waittime_port_25229}\n")
@@ -567,6 +592,13 @@ def get_command_windows(env_variable_name, env_variable_value)
       commands = get_command_windows("AZMON_MULTILINE_ENABLED", @multiline_enabled)
       file.write(commands)
     end
+    if @resource_optimization_enabled
+      commands = get_command_windows("AZMON_RESOURCE_OPTIMIZATION_ENABLED", @resource_optimization_enabled)
+      file.write(commands)
+    end
+
+    commands = get_command_windows("AZMON_WINDOWS_FLUENT_BIT_DISABLED", @windows_fluent_bit_disabled)
+    file.write(commands)
     # Close file after writing all environment variables
     file.close
     puts "****************End Config Processing********************"

build/linux/Makefile

@@ -87,6 +87,12 @@ INSTALLER_TMPDIR := $(INTERMEDIATE_DIR)/installer_tmp
 
 # GO Source dir for custom fluent bit plugin
 GO_SOURCE_DIR := $(SOURCE_DIR)/plugins/go/src
+GO_SOURCE_INPUT_DIR := $(SOURCE_DIR)/plugins/go/input
+
+CMETRICS_VERSION = 0.6.0
+AMD64_DEB_HEADERS = cmetrics_$(CMETRICS_VERSION)_amd64-headers.deb
+AMD64_DEB = cmetrics_$(CMETRICS_VERSION)_amd64.deb
+AMD64_URL = https://github.com/fluent/cmetrics/releases/download/v$(CMETRICS_VERSION)
 
 ifeq ($(ENABLE_DEBUG),1)
 PROV_DEBUG_FLAGS := -g
@@ -103,7 +109,7 @@ OUTPUT_PACKAGE_PREFIX=docker-cimprov-$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER
 # Build targets
 # TODO - fix plugintests and enable it back
 # all : PROVIDER_STATUS KIT_STATUS kit fluentbitplugin rubypluginstests
-all : PROVIDER_STATUS KIT_STATUS kit fluentbitplugin
+all : PROVIDER_STATUS KIT_STATUS kit fluentbitinputplugin fluentbitplugin
 
 clean :
 	$(RMDIR) $(BASE_DIR)/build/cppunit_result.* $(BASE_DIR)/installer/intermediate $(BASE_DIR)/intermediate $(BASE_DIR)/target
@@ -112,6 +118,34 @@ clean :
 
 distclean : clean
 
+# Define the target
+install_cmetrics:
+# For amd64 architecture
+ifeq ($(PF_ARCH),amd64)
+	# Download and install header files
+	wget $(AMD64_URL)/$(AMD64_DEB_HEADERS)
+	sudo dpkg -i $(AMD64_DEB_HEADERS)
+	# Download and install binary
+	wget $(AMD64_URL)/$(AMD64_DEB)
+	sudo dpkg -i $(AMD64_DEB)
+endif
+
+# For arm64 architecture
+ifeq ($(PF_ARCH),arm64)
+	sudo apt install cmake bison flex -y
+	# Clone the repository and build from source
+	git clone --recursive https://github.com/calyptia/cmetrics.git
+	cd cmetrics && \
+	git checkout v$(CMETRICS_VERSION) && \
+	git submodule sync && \
+	git -c protocol.version=2 submodule update --init --force --depth=1 && \
+	git submodule foreach git config --local gc.auto 0 && \
+	cmake -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \
+	-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc . && \
+	make && \
+	make install
+endif
+
 PROVIDER_STATUS:
 	@echo "========================= Performing Building provider"
 	@echo "clean up everything under: $(INTERMEDIATE_BASE_DIR) to avoid picking up old binaries"
@@ -130,6 +164,15 @@ fluentbitplugin :
 	make -C $(GO_SOURCE_DIR) fbplugin arch=$(PF_ARCH)
 	$(COPY) $(GO_SOURCE_DIR)/out_oms.so $(INTERMEDIATE_DIR)
 
+fluentbitinputplugin:
+	@echo "========================= Building fluentbit input go plugin"
+	$(MKPATH) $(INTERMEDIATE_DIR)
+
+	make -C $(GO_SOURCE_INPUT_DIR)/containerinventory containerinventory arch=$(PF_ARCH)
+	$(COPY) $(GO_SOURCE_INPUT_DIR)/containerinventory/containerinventory.so $(INTERMEDIATE_DIR)
+	make -C $(GO_SOURCE_INPUT_DIR)/perf perf arch=$(PF_ARCH)
+	$(COPY) $(GO_SOURCE_INPUT_DIR)/perf/perf.so $(INTERMEDIATE_DIR)
+
 rubypluginstests :
 	@echo "========================= Installing pre-reqs for running tests"
 	sudo apt-add-repository ppa:brightbox/ruby-ng -y
@@ -148,7 +191,7 @@ rubypluginstests :
 # Assume we ALWAYS build DPKG, but only build RPM if --enable-ulinux is speified in configure.
 
 kit : CONTAINERLIB_FILENAME = libcontainer.so
-kit : fluentbitplugin
+kit : install_cmetrics fluentbitinputplugin fluentbitplugin
 
 
 	@echo "========================= Performing Building RPM and DPKG packages"

build/linux/installer/conf/fluent-bit-common.conf

@@ -84,3 +84,11 @@
     Retry_Limit                     10
     TelemetryPushIntervalSeconds    300
     Match                           oms.container.*
+
+[OUTPUT]
+    Name                            oms
+    Alias                           oms_input_plugin_output
+    EnableTelemetry                 true
+    Retry_Limit                     10
+    TelemetryPushIntervalSeconds    300
+    Match                           oneagent.containerInsights.*

build/linux/installer/conf/fluent-bit-input.conf

@@ -0,0 +1,9 @@
+[INPUT]
+    Name containerinventory
+    Tag  oneagent.containerInsights.CONTAINER_INVENTORY_BLOB
+    run_interval 60
+
+[INPUT]
+    Name perf
+    Tag  oneagent.containerInsights.LINUX_PERF_BLOB
+    run_interval 60

build/linux/installer/conf/fluent-bit.conf

@@ -10,6 +10,10 @@
     Log_Level     info
     Parsers_File  /etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf
     Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
+    #${ResourceOptimizationPluginFile}
+
+#${ResourceOptimizationFBConfigFile}
+
 
 [INPUT]
     Name tail

build/linux/installer/conf/kube.conf

@@ -4,6 +4,8 @@
     root_dir /var/opt/microsoft/docker-cimprov/state
    </system>
 
+   #@include windows_rs_containerinventory.conf
+
    #perf
    <match **LINUX_PERF_BLOB**>
     @type forward
@@ -64,32 +66,6 @@
      @log_level info
     </filter>
 
-    #containerinventory for windows containers
-    <match **CONTAINER_INVENTORY_BLOB**>
-      @type forward
-      @id out_ci_fwd
-      @log_level debug
-      send_timeout 30
-      connect_timeout 30
-      heartbeat_type none
-      <server>
-        host 0.0.0.0
-        port "#{ENV['MDSD_FLUENT_SOCKET_PORT']}"
-      </server>
-     <buffer>
-      @type file
-      overflow_action drop_oldest_chunk
-      chunk_limit_size 4m
-      queue_limit_length "#{ENV['FLUENTD_QUEUE_LIMIT_LENGTH']}"
-      flush_interval "#{ENV['FLUENTD_FLUSH_INTERVAL']}"
-      retry_max_times 10
-      retry_wait 5s
-      retry_max_interval 5m
-      flush_thread_count 5
-     </buffer>
-     keepalive true
-    </match>
-
 
   <worker "#{ENV['FLUENTD_POD_INVENTORY_WORKER_ID']}">
     #Kubernetes pod inventory
@@ -317,13 +293,7 @@
      @log_level debug
     </source>
 
-    #cadvisor perf- Windows nodes
-    <source>
-     @type win_cadvisor_perf
-     tag oneagent.containerInsights.LINUX_PERF_BLOB
-     run_interval 60
-     @log_level debug
-    </source>
+    #@include windows_rs_perf.conf
 
     #Kubernetes object state - deployments
     <source>

build/linux/installer/conf/windows_rs_containerinventory.conf

@@ -0,0 +1,25 @@
+  #containerinventory for windows containers
+  <match **CONTAINER_INVENTORY_BLOB**>
+    @type forward
+    @id out_ci_fwd
+    @log_level debug
+    send_timeout 30
+    connect_timeout 30
+    heartbeat_type none
+    <server>
+      host 0.0.0.0
+      port "#{ENV['MDSD_FLUENT_SOCKET_PORT']}"
+    </server>
+    <buffer>
+    @type file
+    overflow_action drop_oldest_chunk
+    chunk_limit_size 4m
+    queue_limit_length "#{ENV['FLUENTD_QUEUE_LIMIT_LENGTH']}"
+    flush_interval "#{ENV['FLUENTD_FLUSH_INTERVAL']}"
+    retry_max_times 10
+    retry_wait 5s
+    retry_max_interval 5m
+    flush_thread_count 5
+    </buffer>
+    keepalive true
+  </match>

build/linux/installer/conf/windows_rs_perf.conf

@@ -0,0 +1,7 @@
+  #cadvisor perf- Windows nodes
+  <source>
+    @type win_cadvisor_perf
+    tag oneagent.containerInsights.LINUX_PERF_BLOB
+    run_interval 60
+    @log_level debug
+  </source>