Skip to content

Commit 4aed32f

Browse files
jiaoew1991jiaoew1991
authored
Use librdkafka for all platform (milvus-io#25538)
Signed-off-by: Enwei Jiao <[email protected]>
1 parent 50454be commit 4aed32f

File tree

11 files changed

+59
-77
lines changed

11 files changed

+59
-77
lines changed

Diff for: ‎.github/workflows/code-checker.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
ubuntu:
3232
name: ${{ matrix.name }}
3333
runs-on: ${{ matrix.os }}
34-
timeout-minutes: 120
34+
timeout-minutes: 180
3535
strategy:
3636
fail-fast: false
3737
matrix:
@@ -78,7 +78,7 @@ jobs:
7878
name: Code Checker CentOS 7
7979
# Run in centos docker
8080
runs-on: ubuntu-20.04
81-
timeout-minutes: 120
81+
timeout-minutes: 180
8282
steps:
8383
- name: Checkout
8484
uses: actions/checkout@v2

Diff for: ‎Makefile

+5-14
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ OBJPREFIX := "github.com/milvus-io/milvus/cmd/milvus"
1818
INSTALL_PATH := $(PWD)/bin
1919
LIBRARY_PATH := $(PWD)/lib
2020
OS := $(shell uname -s)
21-
ARCH := $(shell arch)
2221
mode = Release
2322
disk_index = OFF
2423
useasan = false
@@ -34,14 +33,14 @@ milvus: build-cpp print-build-info
3433
@source $(PWD)/scripts/setenv.sh && \
3534
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
3635
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
37-
${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
36+
-tags dynamic -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
3837

3938
milvus-gpu: build-cpp-gpu print-gpu-build-info
4039
@echo "Building Milvus-gpu ..."
4140
@source $(PWD)/scripts/setenv.sh && \
4241
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
4342
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS_GPU)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
44-
${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
43+
-tags dynamic -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
4544

4645
get-build-deps:
4746
@(env bash $(PWD)/scripts/install_deps.sh)
@@ -100,30 +99,22 @@ meta-migration:
10099
@source $(PWD)/scripts/setenv.sh && \
101100
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
102101
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
103-
${AARCH64_FLAG} -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null
102+
-tags dynamic -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null
104103

105104
INTERATION_PATH = $(PWD)/tests/integration
106105
integration-test:
107106
@echo "Building integration tests ..."
108107
@source $(PWD)/scripts/setenv.sh && \
109108
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
110109
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
111-
${APPLE_SILICON_FLAG} -o $(INSTALL_PATH)/integration-test $(INTERATION_PATH)/ 1>/dev/null
110+
-tags dynamic -o $(INSTALL_PATH)/integration-test $(INTERATION_PATH)/ 1>/dev/null
112111

113112
BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev")
114113
BUILD_TAGS_GPU = ${BUILD_TAGS}-gpu
115114
BUILD_TIME = $(shell date -u)
116115
GIT_COMMIT = $(shell git rev-parse --short HEAD)
117116
GO_VERSION = $(shell go version)
118117

119-
ifeq ($(ARCH),arm64)
120-
AARCH64_FLAG = -tags dynamic
121-
endif
122-
123-
ifeq ($(ARCH),aarch64)
124-
AARCH64_FLAG = -tags dynamic
125-
endif
126-
127118
print-build-info:
128119
$(shell git config --global --add safe.directory '*')
129120
@echo "Build Tag: $(BUILD_TAGS)"
@@ -363,4 +354,4 @@ generate-mockery: getdeps
363354
#internal/proxy
364355
$(PWD)/bin/mockery --name=LBPolicy --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_policy.go --structname=MockLBPolicy --with-expecter --outpkg=proxy --inpackage
365356
$(PWD)/bin/mockery --name=LBBalancer --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_balancer.go --structname=MockLBBalancer --with-expecter --outpkg=proxy --inpackage
366-
$(PWD)/bin/mockery --name=shardClientMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_shardclient_manager.go --structname=MockShardClientManager --with-expecter --outpkg=proxy --inpackage
357+
$(PWD)/bin/mockery --name=shardClientMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_shardclient_manager.go --structname=MockShardClientManager --with-expecter --outpkg=proxy --inpackage

Diff for: ‎internal/core/conanfile.py

+2
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,11 @@ class MilvusConan(ConanFile):
5151
"folly/2023.05.22.02@milvus/dev",
5252
"google-cloud-cpp/2.5.0@milvus/dev",
5353
"opentelemetry-cpp/1.8.1.1@milvus/dev",
54+
"librdkafka/1.9.1",
5455
)
5556
generators = ("cmake", "cmake_find_package")
5657
default_options = {
58+
"librdkafka:shared": True,
5759
"rocksdb:shared": True,
5860
"rocksdb:with_zstd": True,
5961
"arrow:parquet": True,

Diff for: ‎internal/core/thirdparty/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ add_subdirectory(knowhere)
4242

4343
add_subdirectory(boost_ext)
4444
add_subdirectory(rocksdb)
45+
add_subdirectory(rdkafka)
4546
add_subdirectory(simdjson)
4647

4748
if (LINUX)

Diff for: ‎internal/core/thirdparty/rdkafka/CMakeLists.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
milvus_add_pkg_config("rdkafka")
2+
3+
install(DIRECTORY "${CONAN_INCLUDE_DIRS_LIBRDKAFKA}/librdkafka/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/librdkafka/")

Diff for: ‎internal/core/thirdparty/rdkafka/rdkafka.pc.in

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
libdir=@CMAKE_INSTALL_FULL_LIBDIR@
2+
includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
3+
Name: Rdkafka
4+
Description: RdKafka
5+
Version: @RDKAFKA_VERSION@
6+
7+
Libs: -L${libdir} -lrdkafka
8+
# Libs.private: -lz -lbz2
9+
Cflags: -I${includedir}

Diff for: ‎internal/core/thirdparty/rocksdb/CMakeLists.txt

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,3 @@
1-
if (MSYS)
2-
configure_file(rocksdb-win.pc.in "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" @ONLY)
3-
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
4-
else()
5-
milvus_add_pkg_config("rocksdb")
6-
endif()
1+
milvus_add_pkg_config("rocksdb")
72

83
install(DIRECTORY "${CONAN_INCLUDE_DIRS_ROCKSDB}/rocksdb/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/rocksdb/")

Diff for: ‎internal/core/thirdparty/rocksdb/rocksdb-win.pc.in

-9
This file was deleted.

Diff for: ‎scripts/run_go_codecov.sh

+2-5
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,15 @@ echo "Running unittest under ./internal & ./pkg"
3030

3131
# starting the timer
3232
beginTime=`date +%s`
33-
if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then
34-
APPLE_SILICON_FLAG="-tags dynamic"
35-
fi
3633
for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do
37-
go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
34+
go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
3835
if [ -f profile.out ]; then
3936
grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO}
4037
rm profile.out
4138
fi
4239
done
4340
for d in $(go list ./pkg/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do
44-
go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
41+
go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
4542
if [ -f profile.out ]; then
4643
grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO}
4744
rm profile.out

Diff for: ‎scripts/run_go_unittest.sh

+33-37
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@ if [[ $(uname -s) == "Darwin" ]]; then
2626
export MallocNanoZone=0
2727
fi
2828

29-
if [[ "$(uname -m)" == "arm64" ]]; then
30-
APPLE_SILICON_FLAG="-tags dynamic"
31-
fi
32-
3329
# ignore MinIO,S3 unittests
3430
MILVUS_DIR="${ROOT_DIR}/internal/"
3531
PKG_DIR="${ROOT_DIR}/pkg/"
@@ -64,106 +60,106 @@ done
6460

6561
function test_proxy()
6662
{
67-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast -count=1
68-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1
63+
go test -race -cover -tags dynamic "${MILVUS_DIR}/proxy/..." -failfast -count=1
64+
go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1
6965
}
7066

7167
function test_querynode()
7268
{
73-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynodev2/..." -failfast -count=1
74-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querynode/..." -failfast -count=1
69+
go test -race -cover -tags dynamic "${MILVUS_DIR}/querynodev2/..." -failfast -count=1
70+
go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/querynode/..." -failfast -count=1
7571
}
7672

7773

7874
function test_kv()
7975
{
80-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/kv/..." -failfast -count=1
76+
go test -race -cover -tags dynamic "${MILVUS_DIR}/kv/..." -failfast -count=1
8177
}
8278

8379
function test_mq()
8480
{
85-
go test -race -cover ${APPLE_SILICON_FLAG} $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast -count=1
81+
go test -race -cover -tags dynamic $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast -count=1
8682
}
8783

8884
function test_storage()
8985
{
90-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/storage" -failfast -count=1
86+
go test -race -cover -tags dynamic "${MILVUS_DIR}/storage" -failfast -count=1
9187
}
9288

9389
function test_allocator()
9490
{
95-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/allocator/..." -failfast -count=1
91+
go test -race -cover -tags dynamic "${MILVUS_DIR}/allocator/..." -failfast -count=1
9692
}
9793

9894
function test_tso()
9995
{
100-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/tso/..." -failfast -count=1
96+
go test -race -cover -tags dynamic "${MILVUS_DIR}/tso/..." -failfast -count=1
10197
}
10298

10399
function test_config()
104100
{
105-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/config/..." -failfast -count=1
101+
go test -race -cover -tags dynamic "${MILVUS_DIR}/config/..." -failfast -count=1
106102
}
107103

108104
function test_util()
109105
{
110-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/funcutil/..." -failfast -count=1
111-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/paramtable/..." -failfast -count=1
112-
go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/util/retry/..." -failfast -count=1
113-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/sessionutil/..." -failfast -count=1
114-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/typeutil/..." -failfast -count=1
115-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/importutil/..." -failfast -count=1
106+
go test -race -cover -tags dynamic "${MILVUS_DIR}/util/funcutil/..." -failfast -count=1
107+
go test -race -cover -tags dynamic "${MILVUS_DIR}/util/paramtable/..." -failfast -count=1
108+
go test -race -cover -tags dynamic "${PKG_DIR}/util/retry/..." -failfast -count=1
109+
go test -race -cover -tags dynamic "${MILVUS_DIR}/util/sessionutil/..." -failfast -count=1
110+
go test -race -cover -tags dynamic "${MILVUS_DIR}/util/typeutil/..." -failfast -count=1
111+
go test -race -cover -tags dynamic "${MILVUS_DIR}/util/importutil/..." -failfast -count=1
116112
}
117113

118114
function test_pkg()
119115
{
120-
go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/common/..." -failfast -count=1
121-
go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/config/..." -failfast -count=1
122-
go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/log/..." -failfast -count=1
123-
go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/mq/..." -failfast -count=1
124-
go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/tracer/..." -failfast -count=1
125-
go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/util/..." -failfast -count=1
116+
go test -race -cover -tags dynamic "${PKG_DIR}/common/..." -failfast -count=1
117+
go test -race -cover -tags dynamic "${PKG_DIR}/config/..." -failfast -count=1
118+
go test -race -cover -tags dynamic "${PKG_DIR}/log/..." -failfast -count=1
119+
go test -race -cover -tags dynamic "${PKG_DIR}/mq/..." -failfast -count=1
120+
go test -race -cover -tags dynamic "${PKG_DIR}/tracer/..." -failfast -count=1
121+
go test -race -cover -tags dynamic "${PKG_DIR}/util/..." -failfast -count=1
126122
}
127123

128124
function test_datanode
129125
{
130126

131-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datanode/..." -failfast -count=1
132-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datanode/..." -failfast -count=1
127+
go test -race -cover -tags dynamic "${MILVUS_DIR}/datanode/..." -failfast -count=1
128+
go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/datanode/..." -failfast -count=1
133129

134130
}
135131

136132
function test_indexnode()
137133
{
138-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexnode/..." -failfast -count=1
134+
go test -race -cover -tags dynamic "${MILVUS_DIR}/indexnode/..." -failfast -count=1
139135
}
140136

141137
function test_rootcoord()
142138
{
143-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/rootcoord/..." -failfast -count=1
144-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/rootcoord" -failfast
139+
go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/rootcoord/..." -failfast -count=1
140+
go test -race -cover -tags dynamic "${MILVUS_DIR}/rootcoord" -failfast
145141
}
146142

147143
function test_datacoord()
148144
{
149-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datacoord/..." -failfast -count=1
150-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datacoord/..." -failfast -count=1
145+
go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/datacoord/..." -failfast -count=1
146+
go test -race -cover -tags dynamic "${MILVUS_DIR}/datacoord/..." -failfast -count=1
151147
}
152148

153149
function test_querycoord()
154150
{
155-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querycoord/..." -failfast -count=1
156-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querycoordv2/..." -failfast -count=1
151+
go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/querycoord/..." -failfast -count=1
152+
go test -race -cover -tags dynamic "${MILVUS_DIR}/querycoordv2/..." -failfast -count=1
157153
}
158154

159155
#function test_indexcoord()
160156
#{
161-
#go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexcoord/..." -failfast
157+
#go test -race -cover -tags dynamic "${MILVUS_DIR}/indexcoord/..." -failfast
162158
#}
163159

164160
function test_metastore()
165161
{
166-
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/metastore/..." -failfast -count=1
162+
go test -race -cover -tags dynamic "${MILVUS_DIR}/metastore/..." -failfast -count=1
167163
}
168164

169165
function test_all()

Diff for: ‎scripts/run_intergration_test.sh

+1-4
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,10 @@ echo "mode: atomic" > ${FILE_COVERAGE_INFO}
2828

2929
# starting the timer
3030
beginTime=`date +%s`
31-
if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then
32-
APPLE_SILICON_FLAG="-tags dynamic"
33-
fi
3431

3532
for d in $(go list ./tests/integration/...); do
3633
echo "$d"
37-
go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" -timeout=20m
34+
go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" -timeout=20m
3835
if [ -f profile.out ]; then
3936
grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO}
4037
rm profile.out

0 commit comments

Comments
 (0)