How to build librdkafka #1995

UebelAndre · 2023-06-07T11:34:47Z

UebelAndre
Jun 7, 2023
Maintainer

I saw this incredible thread in the slack channel and thought I'd re-post it here to make it more discoverable

David Lopez:

has someone been able to build the the rdkafka library using the cmake-build feature using bazel? Im getting panics after the following warnings:
Building and linking librdkafka statically
CMake Error: Could not find CMAKE_ROOT !!!
CMake has most likely not been installed correctly.
Modules directory not found in
I defined CMAKE_ROOT and CMAKE_MODULE_PATH in my shell but seems like bazel is ignoring it and it is using a different cmake version that the one that I have installed in my system (edited)

This then lead to a hero's reponse

Brian Myers:

Here's what my librdkafka looks like to help get you started. The others (zstd, lz4, zlib, openssl) should have examples if you google for them. I don't recall seeing any good librdkafka examples, so pasting here

load("@bazel_skylib//rules:write_file.bzl", "write_file")
load("@rules_cc//cc:defs.bzl", "cc_library")

alias(
    name = "librdkafka",
    actual = "rdkafka",
    visibility = ["//visibility:public"],
)

alias(
    name = "librdkafka++",
    actual = "rdkafka++",
    visibility = ["//visibility:public"],
)

write_file(
    name = "config_dot_h",
    out = "config.h",
    content = [
        "#define WITHOUT_OPTIMIZATION 0",
        "#define ENABLE_DEVEL 0",
        "#define ENABLE_REFCNT_DEBUG 0",
        "#define HAVE_ATOMICS_32 1",
        "#define HAVE_ATOMICS_32_SYNC 0",
        "#if (HAVE_ATOMICS_32)",
        "# if (HAVE_ATOMICS_32_SYNC)",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_ATOMICS_64 1",
        "#define HAVE_ATOMICS_64_SYNC 0",
        "#if (HAVE_ATOMICS_64)",
        "# if (HAVE_ATOMICS_64_SYNC)",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_RAND_R 1",
        "#define HAVE_REGEX 1",
        "#define HAVE_STRNDUP 1",
        "#define WITH_C11THREADS 0",
        "#define WITH_CRC32C_HW 0",
        "#define WITH_HDRHISTOGRAM 1",
        "#define WITH_LIBDL 1",
        "#define WITH_LZ4_EXT 1",
        "#define WITH_PKGCONFIG 1",
        "#define WITH_PLUGINS 1",
        "#define WITH_SASL 0",
        "#define WITH_SASL_CYRUS 0",
        "#define WITH_SASL_OAUTHBEARER 0",
        "#define WITH_SASL_SCRAM 0",
        "#define WITH_SNAPPY 1",
        "#define WITH_SOCKEM 1",
        "#define WITH_SSL 1",
        "#define WITH_ZLIB 1",
        "#define WITH_ZSTD 1",
    ] + select({
        "@//bazel/constraints/os:darwin": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 1",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 0",
            "#define SOLIB_EXT \".dylib\"",
            # The compiler here is incorrect, but I don't think it matters
            "#define BUILT_WITH  \"CMAKE AppleClang AppleClang PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "@//bazel/constraints/os:linux": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 0",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 1",
            "#define SOLIB_EXT \".so\"",
            "#define BUILT_WITH  \"CMAKE GNU GNU PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "//conditions:default": [],
    }),
)

cc_library(
    name = "config_header",
    hdrs = [
        "config.h",
    ],
)

cc_library(
    name = "rdkafka_header",
    hdrs = [
        "src/rdkafka.h",
    ],
)

cc_library(
    name = "rdkafka",
    srcs = glob(
        include = [
            "src/*.c",
            "src/*.h",
        ],
        exclude = [
            # don't use sasl
            "src/rdkafka_sasl_cyrus.c",
            "src/rdkafka_sasl_oauthbearer.c",
            "src/rdkafka_sasl_scram.c",
            "src/rdkafka_sasl_win32.c",
            # no need for win32
            "src/rdwin32.h",
            "src/win32_config.h",
            # don't use built-in laz4
            "src/lz4.c",
            "src/lz4frame.c",
            "src/lz4hc.c",
        ],
    ),
    hdrs = [
        "src/rdkafka.h",
    ],
    copts = [
        "-fPIC",
        "-Wall",
        # Keep log spew down when compiling with LLVM
        "-Wno-unused-but-set-variable",
    ],
    include_prefix = "librdkafka",
    strip_include_prefix = "src",
    deps = [
        ":config_header",
        "@com_github__facebook__zstd//:libzstd",
        "@com_github__lz4__lz4//:liblz4",
        "@com_github__openssl__openssl//:openssl",
        "@net_zlib__zlib//:libz",
    ],
)

cc_library(
    name = "rdkafka++",
    srcs = glob([
        "src-cpp/*.cpp",
        "src-cpp/*.h",
    ]),
    hdrs = [
        "src-cpp/rdkafkacpp.h",
    ],
    strip_include_prefix = "src-cpp",
    deps = [
        ":config_header",
        ":rdkafka_header",
    ],
)

I only have to support macos and linux by the way. Good luck if you care about windows
Oh, and make sure you pull the version of librdkafka that rdkafka-sys uses. rdkafka 0.28.0 uses https://github.com/benesch/librdkafka/archive/30633e59a9072b7b9c76411c281578e0125d5f65.tar.gz

Huge thanks to Brian for sharing!

UebelAndre · 2025-03-10T20:25:24Z

UebelAndre
Mar 10, 2025
Maintainer Author

There's now https://github.com/scasagrande/bazel-examples/tree/main/rdkafka-sys !

"""
This file contains a native bazel build for librdkafka.

This implementation is a modified version of this one that was posted
in the bazel slack:
https://bazelbuild.slack.com/archives/CSV56UT0F/p1685744596942669?thread_ts=1685743903.751189&cid=CSV56UT0F

While messy, there is nothing actually special about it. The standard
librdkafka build systems (cmake and mklove) will construct this `config.h`
file that we have just manually defined here in this BUILD.bazel file.
"""

load("@bazel_skylib//rules:write_file.bzl", "write_file")
load("@rules_cc//cc:defs.bzl", "cc_library")

alias(
    name = "librdkafka",
    actual = "rdkafka",
    visibility = ["//visibility:public"],
)

alias(
    name = "librdkafka++",
    actual = "rdkafka++",
    visibility = ["//visibility:public"],
)

write_file(
    name = "config_dot_h",
    out = "config.h",
    content = [
        "#define WITHOUT_OPTIMIZATION 0",
        "#define ENABLE_DEVEL 0",
        "#define ENABLE_REFCNT_DEBUG 0",
        "#define HAVE_ATOMICS_32 1",
        "#define HAVE_ATOMICS_32_SYNC 0",
        "#if (HAVE_ATOMICS_32)",
        "# if (HAVE_ATOMICS_32_SYNC)",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_ATOMICS_64 1",
        "#define HAVE_ATOMICS_64_SYNC 0",
        "#if (HAVE_ATOMICS_64)",
        "# if (HAVE_ATOMICS_64_SYNC)",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __sync_ ## OP1 ## _and_ ## OP2(PTR, VAL)",
        "# else",
        "#  define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST)",
        "# endif",
        "#endif",
        "#define HAVE_RAND_R 1",
        "#define HAVE_REGEX 1",
        "#define HAVE_STRNDUP 1",
        "#define WITH_C11THREADS 0",
        "#define WITH_CRC32C_HW 0",
        "#define WITH_HDRHISTOGRAM 1",
        "#define WITH_LIBDL 1",
        "#define WITH_LZ4_EXT 1",
        "#define WITH_PKGCONFIG 1",
        "#define WITH_PLUGINS 1",
        "#define WITH_SASL 0",  # flip for sasl support
        "#define WITH_SASL_CYRUS 0",  # flip for sasl support
        "#define WITH_SASL_OAUTHBEARER 0",
        "#define WITH_SASL_SCRAM 0",
        "#define WITH_SNAPPY 1",
        "#define WITH_SOCKEM 1",
        "#define WITH_SSL 1",
        "#define WITH_ZLIB 0",
        "#define WITH_ZSTD 0",
    ] + select({
        "@platforms//os:macos": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 1",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 0",
            "#define SOLIB_EXT \".dylib\"",
            # The compiler here is incorrect, but I don't think it matters
            "#define BUILT_WITH  \"CMAKE AppleClang AppleClang PKGCONFIG HDRHISTOGRAM LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
            #            "#define BUILT_WITH  \"CMAKE AppleClang AppleClang PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "@platforms//os:linux": [
            "#define HAVE_PTHREAD_SETNAME_DARWIN 0",
            "#define HAVE_PTHREAD_SETNAME_FREEBSD 0",
            "#define HAVE_PTHREAD_SETNAME_GNU 1",
            "#define SOLIB_EXT \".so\"",
            "#define BUILT_WITH  \"CMAKE GNU GNU PKGCONFIG HDRHISTOGRAM LIBDL PLUGINS SSL SASL_CYRUS SASL_SCRAM LZ4_EXT SNAPPY SOCKEM\"",
            #            "#define BUILT_WITH  \"CMAKE GNU GNU PKGCONFIG HDRHISTOGRAM ZLIB ZSTD LIBDL PLUGINS SSL LZ4_EXT SNAPPY SOCKEM\"",
        ],
        "//conditions:default": [],
    }),
)

cc_library(
    name = "config_header",
    hdrs = [
        "config.h",
    ],
)

cc_library(
    name = "rdkafka_header",
    hdrs = [
        "src/rdkafka.h",
    ],
)

cc_library(
    name = "rdkafka",
    srcs = glob(
        include = [
            "src/*.c",
            "src/*.h",
        ],
        exclude = [
            # Comment out to include sasl support
            "src/rdkafka_sasl_cyrus.c",
            "src/rdkafka_sasl_scram.c",

            # don't use sasl oauthbearer, it needs libcurl-dev
            "src/rdkafka_sasl_oauthbearer.c",
            "src/rdkafka_sasl_oauthbearer_oidc.c",
            "src/rdhttp.c",

            # no need for win32
            "src/rdkafka_sasl_win32.c",
            "src/rdwin32.h",
            "src/win32_config.h",

            # don't use built-in lz4
            "src/lz4.c",
            "src/lz4frame.c",
            "src/lz4hc.c",

            # don't use zstd
            "src/rdkafka_zstd.c",

            # don't use zlib
            "src/rdgz.c",

            # These are also defined in @lz4//:lib/xxhash.c
            "src/rdxxhash.c",
        ],
    ),
    hdrs = [
        "src/rdkafka.h",
    ],
    #    copts = [
    #        "-fPIC",
    #        "-Wall",
    #        # Keep log spew down when compiling with LLVM
    #        "-Wno-unused-but-set-variable",
    #    ],
    data = [":config.h"],
    include_prefix = "librdkafka",
    strip_include_prefix = "src",
    deps = [
        ":config_header",
        "@lz4",
        "@lz4//:lz4_frame",
        "@openssl",
    ],
)

cc_library(
    name = "rdkafka++",
    srcs = glob([
        "src-cpp/*.cpp",
        "src-cpp/*.h",
    ]),
    hdrs = [
        "src-cpp/rdkafkacpp.h",
    ],
    strip_include_prefix = "src-cpp",
    deps = [
        ":config_header",
        ":rdkafka_header",
    ],
)

0 replies

UebelAndre · 2025-03-10T20:26:27Z

UebelAndre
Mar 10, 2025
Maintainer Author

And for bzlmod, seems like there's https://github.com/bazelbuild/bazel-central-registry/tree/eb0fdf46f22a8df0a8a66e61716a27068888ea55/modules/librdkafka

0 replies

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

How to build librdkafka #1995

{{title}}

Replies: 2 comments

{{title}}

{{title}}

Select a reply

How to build librdkafka #1995

UebelAndre Jun 7, 2023 Maintainer

Replies: 2 comments

UebelAndre Mar 10, 2025 Maintainer Author

UebelAndre Mar 10, 2025 Maintainer Author

UebelAndre
Jun 7, 2023
Maintainer

UebelAndre
Mar 10, 2025
Maintainer Author

UebelAndre
Mar 10, 2025
Maintainer Author