From b5104b7beb342f438ef8f6fb257fdbd5da68b796 Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Fri, 10 Oct 2025 15:13:02 -0400 Subject: [PATCH 1/4] plugin_proxy: change proxydef initialization to use calloc instead of malloc Changing intialization to use calloc instead of malloc so that emtpy fields are 0 initialized. Specifically for the event_type addition. Signed-off-by: jmccormick7 --- src/flb_plugin_proxy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_plugin_proxy.c b/src/flb_plugin_proxy.c index 2ec84b546c5..da8465adbb8 100644 --- a/src/flb_plugin_proxy.c +++ b/src/flb_plugin_proxy.c @@ -612,7 +612,7 @@ struct flb_plugin_proxy *flb_plugin_proxy_create(const char *dso_path, int type, return NULL; } - proxy->def = flb_malloc(sizeof(struct flb_plugin_proxy_def)); + proxy->def = flb_calloc(1, sizeof(struct flb_plugin_proxy_def)); if (!proxy->def) { flb_errno(); dlclose(handle); From 7eef919a5d2f2adfefcb12298073386270adc721 Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Wed, 12 Nov 2025 12:17:07 -0500 Subject: [PATCH 2/4] runtime_shell: Adding test for go plugins This commit adds a runtime shell test for the go plugin interface. A dummy go output plugin is added to the directories, and a test that builds the plugin shared object and then tests that it works and writes the output. This test ensures that the go proxy interface is working as expected. This was tested in the devcontainer using ctest. Signed-off-by: jmccormick7 --- tests/runtime_shell/CMakeLists.txt | 4 +- tests/runtime_shell/conf/proxy_logs_test.conf | 21 ++++ .../conf/proxy_metrics_test.conf | 0 .../go_plugins/build_test_plugins.sh | 111 ++++++++++++++++++ tests/runtime_shell/go_plugins/go.mod | 8 ++ tests/runtime_shell/go_plugins/go.sum | 5 + tests/runtime_shell/go_plugins/logs_output.go | 47 ++++++++ tests/runtime_shell/go_plugins/test_logs_go.h | 92 +++++++++++++++ tests/runtime_shell/proxy_logs_expect.sh | 38 ++++++ 9 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 tests/runtime_shell/conf/proxy_logs_test.conf create mode 100644 tests/runtime_shell/conf/proxy_metrics_test.conf create mode 100755 tests/runtime_shell/go_plugins/build_test_plugins.sh create mode 100644 tests/runtime_shell/go_plugins/go.mod create mode 100644 tests/runtime_shell/go_plugins/go.sum create mode 100644 tests/runtime_shell/go_plugins/logs_output.go create mode 100644 tests/runtime_shell/go_plugins/test_logs_go.h create mode 100755 tests/runtime_shell/proxy_logs_expect.sh diff --git a/tests/runtime_shell/CMakeLists.txt b/tests/runtime_shell/CMakeLists.txt index 30ae87a6b96..f201bb46910 100644 --- a/tests/runtime_shell/CMakeLists.txt +++ b/tests/runtime_shell/CMakeLists.txt @@ -15,6 +15,7 @@ set(UNIT_TESTS_SH in_syslog_uds_stream_plaintext_expect.sh processor_conditional.sh processor_invalid.sh + proxy_logs_expect.sh ) # Prepare list of unit tests @@ -27,6 +28,7 @@ foreach(script ${UNIT_TESTS_SH}) "FLB_ROOT=${PROJECT_SOURCE_DIR};\ FLB_RUNTIME_SHELL_PATH=${CMAKE_CURRENT_SOURCE_DIR};\ FLB_RUNTIME_SHELL_CONF=${CMAKE_CURRENT_SOURCE_DIR}/conf;\ -FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit" +FLB_BIN=${CMAKE_BINARY_DIR}/bin/fluent-bit;\ +FLB_BUILD=${CMAKE_BINARY_DIR}" ) endforeach() diff --git a/tests/runtime_shell/conf/proxy_logs_test.conf b/tests/runtime_shell/conf/proxy_logs_test.conf new file mode 100644 index 00000000000..2946088e6f6 --- /dev/null +++ b/tests/runtime_shell/conf/proxy_logs_test.conf @@ -0,0 +1,21 @@ +[SERVICE] + Flush 1 + Grace 2 + Log_Level info + Daemon Off + +[INPUT] + Name dummy + Dummy {"message": "test log entry", "level": "info"} + Samples 3 + Tag test.logs + +[OUTPUT] + Name test_logs_go + Match test.logs + +[OUTPUT] + Name file + Match test.logs + File ${SIGNAL_FILE_PATH} + mkdir on diff --git a/tests/runtime_shell/conf/proxy_metrics_test.conf b/tests/runtime_shell/conf/proxy_metrics_test.conf new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/runtime_shell/go_plugins/build_test_plugins.sh b/tests/runtime_shell/go_plugins/build_test_plugins.sh new file mode 100755 index 00000000000..3b1ab4e882e --- /dev/null +++ b/tests/runtime_shell/go_plugins/build_test_plugins.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Build script for Go test plugins +set -e + +GO_PLUGIN_DIR="${FLB_ROOT}/tests/runtime_shell/go_plugins" +BUILD_DIR="${FLB_ROOT}/build" + +install_go_if_needed() { + if ! command -v go &> /dev/null; then + echo "Go not found, installing Go..." + + ARCH=$(uname -m) + case $ARCH in + x86_64) GO_ARCH="amd64" ;; + aarch64|arm64) GO_ARCH="arm64" ;; + *) echo "Unsupported architecture: $ARCH"; exit 1 ;; + esac + + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + GO_VERSION="1.25.4" + GO_TARBALL="go${GO_VERSION}.${OS}-${GO_ARCH}.tar.gz" + GO_URL="https://golang.org/dl/${GO_TARBALL}" + + echo "Downloading Go from $GO_URL..." + + TEMP_DIR=$(mktemp -d) + cd "$TEMP_DIR" + + if command -v curl > /dev/null 2>&1; then + curl -L -O "$GO_URL" + else + echo "Neither wget nor curl is available to download Go." + exit 1 + fi + + echo "Extracting Go tarball..." + ls -la + + if [ ! -f "$GO_TARBALL" ]; then + echo "Failed to download Go tarball." + exit 1 + fi + + tar -xzf "$GO_TARBALL" + + if [ -w "/usr/local" ]; then + sudo rm -rf /usr/local/go + sudo mv go /usr/local/go + export PATH="/usr/local/go/bin:$PATH" + else + echo "No write permission to /usr/local. Installing Go to $HOME/.local/go" + mkdir -p "$HOME/.local" + rm -rf "$HOME/.local/go" + mv go "$HOME/.local/go" + export PATH="$HOME/.local/go/bin:$PATH" + fi + cd - > /dev/null + rm -rf "$TEMP_DIR" + echo "Go installed successfully." + go version +else + echo "Go is already installed." +fi +} + +verify_go_cgo() { + echo "Verifying Go CGO support..." + if ! go env CGO_ENABLED | grep -q "1"; then + echo "Warning: CGO is not enabled. Attempting to enable CGO..." + export CGO_ENABLED=1 + fi + + TEMP_GO_FILE=$(mktemp --suffix=.go) + cat > "$TEMP_GO_FILE" << 'EOF' +package main +import "C" +//export TestFunc +func TestFunc() {} +func main() {} +EOF + TEMP_SO_FILE=$(mktemp --suffix=.so) + if go build -buildmode=c-shared -o "$TEMP_SO_FILE" "$TEMP_GO_FILE" 2> /dev/null; then + echo "CGO is enabled and working." + rm -f "$TEMP_GO_FILE" "$TEMP_SO_FILE" + else + echo "Error: CGO is not enabled or not working properly. Please ensure you have a C compiler installed." + rm -f "$TEMP_GO_FILE" "$TEMP_SO_FILE" + exit 1 + fi +} + +build_go_plugins() { + echo "Building Go test plugins..." + + echo "Building logs output plugin..." + cd "$GO_PLUGIN_DIR" + CGO_ENABLED=1 GO111MODULE=on go build -buildmode=c-shared -v -ldflags="-s -w" -o $BUILD_DIR/test_logs_go.so logs_output.go + if [ $? -eq 0 ]; then + echo "Go test plugins built successfully!" + echo "Logs plugin: $BUILD_DIR/test_logs_go.so" + else + echo "Failed to build Go test plugins." + exit 1 + fi +} + +echo "Setting up Go build environment..." +install_go_if_needed +verify_go_cgo +build_go_plugins \ No newline at end of file diff --git a/tests/runtime_shell/go_plugins/go.mod b/tests/runtime_shell/go_plugins/go.mod new file mode 100644 index 00000000000..ff0a7df9ff6 --- /dev/null +++ b/tests/runtime_shell/go_plugins/go.mod @@ -0,0 +1,8 @@ +module github.com/fluent/fluent-bit/tests/runtime_shell/go_plugins + +go 1.25.1 + +require ( + github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c + github.com/ugorji/go/codec v1.1.7 // indirect +) \ No newline at end of file diff --git a/tests/runtime_shell/go_plugins/go.sum b/tests/runtime_shell/go_plugins/go.sum new file mode 100644 index 00000000000..523259f04ad --- /dev/null +++ b/tests/runtime_shell/go_plugins/go.sum @@ -0,0 +1,5 @@ +github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c h1:yKN46XJHYC/gvgH2UsisJ31+n4K3S7QYZSfU2uAWjuI= +github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c/go.mod h1:L92h+dgwElEyUuShEwjbiHjseW410WIcNz+Bjutc8YQ= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= diff --git a/tests/runtime_shell/go_plugins/logs_output.go b/tests/runtime_shell/go_plugins/logs_output.go new file mode 100644 index 00000000000..21ac6b5d6ef --- /dev/null +++ b/tests/runtime_shell/go_plugins/logs_output.go @@ -0,0 +1,47 @@ +package main + +import ( + "C" + "fmt" + "unsafe" + + "github.com/fluent/fluent-bit-go/output" +) + +//export FLBPluginRegister +func FLBPluginRegister(def unsafe.Pointer) int { + // Register as logs-only output plugin + return output.FLBPluginRegister(def, "test_logs_go", "Test Go Output Plugin for Logs") +} + +//export FLBPluginInit +func FLBPluginInit(plugin unsafe.Pointer) int { + return output.FLB_OK +} + +//export FLBPluginFlushCtx +func FLBPluginFlushCtx(ctx, data unsafe.Pointer, length C.int, tag *C.char) int { + // Write to a stdout to verify it received data + dec := output.NewDecoder(data, int(length)) + var logrecords []string + for { + ret, _, record := output.GetRecord(dec) + if ret != 0 { + break + } + logrecords = append(logrecords, fmt.Sprintf("%v", record)) + } + for _, record := range logrecords { + fmt.Printf("%s\n", record) + } + + return output.FLB_OK +} + +//export FLBPluginExit +func FLBPluginExit() int { + return output.FLB_OK +} + +func main() { +} diff --git a/tests/runtime_shell/go_plugins/test_logs_go.h b/tests/runtime_shell/go_plugins/test_logs_go.h new file mode 100644 index 00000000000..6b6b8bb27cf --- /dev/null +++ b/tests/runtime_shell/go_plugins/test_logs_go.h @@ -0,0 +1,92 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +extern size_t _GoStringLen(_GoString_ s); +extern const char *_GoStringPtr(_GoString_ s); +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#if !defined(__cplusplus) || _MSVC_LANG <= 201402L +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +#include +typedef std::complex GoComplex64; +typedef std::complex GoComplex128; +#endif +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern GoInt FLBPluginRegister(void* def); +extern GoInt FLBPluginInit(void* plugin); +extern GoInt FLBPluginFlushCtx(void* ctx, void* data, int length, char* tag); +extern GoInt FLBPluginExit(void); + +#ifdef __cplusplus +} +#endif diff --git a/tests/runtime_shell/proxy_logs_expect.sh b/tests/runtime_shell/proxy_logs_expect.sh new file mode 100755 index 00000000000..08148d7542c --- /dev/null +++ b/tests/runtime_shell/proxy_logs_expect.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Setup environment if not already set +if [ -z "$FLB_BIN" ]; then + FLB_ROOT=${FLB_ROOT:-$(cd $(dirname $0)/../.. && pwd)} + FLB_BIN=${FLB_BIN:-$FLB_ROOT/build/bin/fluent-bit} +fi + +echo "Using Fluent Bit at: $FLB_BIN" + +. $FLB_RUNTIME_SHELL_PATH/go_plugins/build_test_plugins.sh + +test_proxy_logs_compatibility() { + export SIGNAL_FILE_PATH="/tmp/flb_signal_logs_$$.txt" + STDOUT_OUTPUT_FILE="/tmp/test_logs_stdout_$$.txt" + + rm -f "$STDOUT_OUTPUT_FILE" "$SIGNAL_FILE_PATH" + + $FLB_BIN -e $FLB_ROOT/build/test_logs_go.so -c $FLB_RUNTIME_SHELL_CONF/proxy_logs_test.conf > "$STDOUT_OUTPUT_FILE" 2>&1 & + FLB_PID=$! + + sleep 3 + + if [ -f "$STDOUT_OUTPUT_FILE" ]; then + echo "SUCCESS: Captured Fluent Bit output" + echo "Output contents:" + cat "$STDOUT_OUTPUT_FILE" + else + echo "FAIL: No stdout output captured" + return 1 + fi + + # Clean up + rm -f "$STDOUT_OUTPUT_FILE" "$SIGNAL_FILE_PATH" +} + +# Load the runtime shell environment +. $FLB_RUNTIME_SHELL_PATH/runtime_shell.env From 2b6272499a58b81504834f86d800460ee7427f8f Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Wed, 12 Nov 2025 12:41:27 -0500 Subject: [PATCH 3/4] runtime_shell: Apply coderabit fixes Signed-off-by: jmccormick7 --- tests/runtime_shell/go_plugins/build_test_plugins.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/runtime_shell/go_plugins/build_test_plugins.sh b/tests/runtime_shell/go_plugins/build_test_plugins.sh index 3b1ab4e882e..cb6e21ce697 100755 --- a/tests/runtime_shell/go_plugins/build_test_plugins.sh +++ b/tests/runtime_shell/go_plugins/build_test_plugins.sh @@ -45,7 +45,9 @@ install_go_if_needed() { tar -xzf "$GO_TARBALL" if [ -w "/usr/local" ]; then - sudo rm -rf /usr/local/go + if [ -d /usr/local/go ]; then + sudo rm -rf /usr/local/go + fi sudo mv go /usr/local/go export PATH="/usr/local/go/bin:$PATH" else From b94c8347d660a17842c00cce77e7b0a256ce3373 Mon Sep 17 00:00:00 2001 From: jmccormick7 Date: Fri, 21 Nov 2025 12:52:58 -0500 Subject: [PATCH 4/4] runtime_shell: making go proxy test Linux only This commit makes the go proxy a test that only runs on Linux using the CMAKE_SYSTEM_NAME. It also removes an autogenerated header file from the go compile that should not have been commited. Signed-off-by: jmccormick7 --- tests/runtime_shell/CMakeLists.txt | 5 +- tests/runtime_shell/go_plugins/test_logs_go.h | 92 ------------------- 2 files changed, 4 insertions(+), 93 deletions(-) delete mode 100644 tests/runtime_shell/go_plugins/test_logs_go.h diff --git a/tests/runtime_shell/CMakeLists.txt b/tests/runtime_shell/CMakeLists.txt index f201bb46910..4893c23d9c3 100644 --- a/tests/runtime_shell/CMakeLists.txt +++ b/tests/runtime_shell/CMakeLists.txt @@ -15,9 +15,12 @@ set(UNIT_TESTS_SH in_syslog_uds_stream_plaintext_expect.sh processor_conditional.sh processor_invalid.sh - proxy_logs_expect.sh ) +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND UNIT_TESTS_SH proxy_logs_expect.sh) +endif() + # Prepare list of unit tests foreach(script ${UNIT_TESTS_SH}) add_test(NAME ${script} diff --git a/tests/runtime_shell/go_plugins/test_logs_go.h b/tests/runtime_shell/go_plugins/test_logs_go.h deleted file mode 100644 index 6b6b8bb27cf..00000000000 --- a/tests/runtime_shell/go_plugins/test_logs_go.h +++ /dev/null @@ -1,92 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package command-line-arguments */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -extern size_t _GoStringLen(_GoString_ s); -extern const char *_GoStringPtr(_GoString_ s); -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#if !defined(__cplusplus) || _MSVC_LANG <= 201402L -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -#include -typedef std::complex GoComplex64; -typedef std::complex GoComplex128; -#endif -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - -extern GoInt FLBPluginRegister(void* def); -extern GoInt FLBPluginInit(void* plugin); -extern GoInt FLBPluginFlushCtx(void* ctx, void* data, int length, char* tag); -extern GoInt FLBPluginExit(void); - -#ifdef __cplusplus -} -#endif