Skip to content

Commit 2ced666

Browse files
jiaoew1991jiaoew1991
authored
Simplify makefile (milvus-io#23583)
Signed-off-by: Enwei Jiao <[email protected]>
1 parent 546080d commit 2ced666

File tree

8 files changed

+202
-202
lines changed

8 files changed

+202
-202
lines changed

‎.github/workflows/main.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ jobs:
8888
restore-keys: ubuntu${{ matrix.ubuntu }}-conan-
8989
- name: Build
9090
run: |
91-
./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage generated-proto-go-without-cpp"
91+
./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage"
9292
- run: |
9393
zip -r code.zip . -x "./.docker/*"
9494
- name: Archive code

‎Makefile

+24-43
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ OS := $(shell uname -s)
2121
ARCH := $(shell arch)
2222
mode = Release
2323
disk_index = OFF
24-
useasan = false
24+
useasan = false
2525
ifeq (${USE_ASAN}, true)
26-
useasan = true
26+
useasan = true
2727
endif
2828

2929
export GIT_BRANCH=master
@@ -58,19 +58,6 @@ tools/bin/revive: tools/check/go.mod
5858
cppcheck:
5959
@(env bash ${PWD}/scripts/core_build.sh -l)
6060

61-
# put generate proto as a separated target because build cpp have different cases like with unittest.
62-
generated-proto-go-without-cpp: export protoc:=${PWD}/cmake_build/bin/protoc
63-
generated-proto-go-without-cpp:
64-
@mkdir -p ${GOPATH}/bin
65-
@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/[email protected])
66-
@(env bash $(PWD)/scripts/proto_gen_go.sh)
67-
68-
generated-proto-go: build-cpp generated-proto-go-without-cpp
69-
70-
check-proto-product-only:
71-
@(env bash $(PWD)/scripts/check_proto_product.sh)
72-
check-proto-product: generated-proto-go check-proto-product-only
73-
7461

7562
fmt:
7663
ifdef GO_DIFF_FILES
@@ -150,13 +137,6 @@ print-gpu-build-info:
150137
@echo "Git Commit: $(GIT_COMMIT)"
151138
@echo "Go Version: $(GO_VERSION)"
152139

153-
embd-milvus: build-cpp-embd print-build-info
154-
@echo "Building **Embedded** Milvus ..."
155-
@source $(PWD)/scripts/setenv.sh && \
156-
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
157-
GO111MODULE=on $(GO) build -ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
158-
${AARCH64_FLAG} -buildmode=c-shared -o $(INSTALL_PATH)/embd-milvus.so $(PWD)/pkg/embedded/embedded.go 1>/dev/null
159-
160140
update-milvus-api: download-milvus-proto
161141
@echo "Update milvus/api version ..."
162142
@(env bash $(PWD)/scripts/update-api-version.sh)
@@ -165,26 +145,35 @@ download-milvus-proto:
165145
@echo "Download milvus-proto repo ..."
166146
@(env bash $(PWD)/scripts/download_milvus_proto.sh)
167147

168-
build-cpp: download-milvus-proto
148+
build-3rdparty:
149+
@echo "Build 3rdparty ..."
150+
@(env bash $(PWD)/scripts/3rdparty_build.sh)
151+
152+
generated-proto: download-milvus-proto build-3rdparty
153+
@echo "Generate proto ..."
154+
@mkdir -p ${GOPATH}/bin
155+
@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/[email protected])
156+
@(env bash $(PWD)/scripts/generate_proto.sh)
157+
158+
build-cpp: generated-proto
169159
@echo "Building Milvus cpp library ..."
170160
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
171161

172-
build-cpp-gpu: download-milvus-proto
162+
build-cpp-gpu: generated-proto
173163
@echo "Building Milvus cpp gpu library ..."
174164
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
175165

176-
build-cpp-embd: download-milvus-proto
177-
@echo "Building **Embedded** Milvus cpp library ..."
178-
@(env bash $(PWD)/scripts/core_build.sh -b -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
179-
180-
build-cpp-with-unittest: download-milvus-proto
166+
build-cpp-with-unittest: generated-proto
181167
@echo "Building Milvus cpp library with unittest ..."
182168
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
183169

184-
build-cpp-with-coverage: download-milvus-proto
170+
build-cpp-with-coverage: generated-proto
185171
@echo "Building Milvus cpp library with coverage and unittest ..."
186172
@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -a ${useasan} -c -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
187173

174+
check-proto-product: generated-proto
175+
@(env bash $(PWD)/scripts/check_proto_product.sh)
176+
188177

189178
# Run the tests.
190179
unittest: test-cpp test-go
@@ -274,11 +263,6 @@ codecov-cpp: build-cpp-with-coverage
274263
@echo "Running cpp coverage..."
275264
@(env bash $(PWD)/scripts/run_cpp_codecov.sh)
276265

277-
# Package docker image locally.
278-
# TODO: fix error occur at starting up
279-
docker: install
280-
./build/build_image.sh
281-
282266
# Build each component and install binary to $GOPATH/bin.
283267
install: milvus
284268
@echo "Installing binary to './bin'"
@@ -298,7 +282,6 @@ gpu-install: milvus-gpu
298282
-cp -r -P $(PWD)/internal/core/output/lib64/*.so* $(LIBRARY_PATH) 2>/dev/null
299283
@echo "Installation successful."
300284

301-
302285
clean:
303286
@echo "Cleaning up all the generated files"
304287
@find . -name '*.test' | xargs rm -fv
@@ -351,10 +334,10 @@ generate-mockery: getdeps
351334
# internal/datanode
352335
$(PWD)/bin/mockery --name=Allocator --dir=$(PWD)/internal/datanode/allocator/ --output=$(PWD)/internal/datanode/allocator --filename=mock_allocator.go --with-expecter --structname=MockAllocator --outpkg=allocator --inpackage
353336
# internal/querycoordv2
354-
$(PWD)/bin/mockery --name=QueryNodeServer --dir=$(PWD)/internal/proto/querypb/ --output=$(PWD)/internal/querycoordv2/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNodeServer
355-
$(PWD)/bin/mockery --name=Broker --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_broker.go --with-expecter --structname=MockBroker --outpkg=meta
356-
$(PWD)/bin/mockery --name=Scheduler --dir=$(PWD)/internal/querycoordv2/task --output=$(PWD)/internal/querycoordv2/task --filename=mock_scheduler.go --with-expecter --structname=MockScheduler --outpkg=task --inpackage
357-
$(PWD)/bin/mockery --name=Cluster --dir=$(PWD)/internal/querycoordv2/session --output=$(PWD)/internal/querycoordv2/session --filename=mock_cluster.go --with-expecter --structname=MockCluster --outpkg=session --inpackage
337+
$(PWD)/bin/mockery --name=QueryNodeServer --dir=$(PWD)/internal/proto/querypb/ --output=$(PWD)/internal/querycoordv2/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNodeServer
338+
$(PWD)/bin/mockery --name=Broker --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_broker.go --with-expecter --structname=MockBroker --outpkg=meta
339+
$(PWD)/bin/mockery --name=Scheduler --dir=$(PWD)/internal/querycoordv2/task --output=$(PWD)/internal/querycoordv2/task --filename=mock_scheduler.go --with-expecter --structname=MockScheduler --outpkg=task --inpackage
340+
$(PWD)/bin/mockery --name=Cluster --dir=$(PWD)/internal/querycoordv2/session --output=$(PWD)/internal/querycoordv2/session --filename=mock_cluster.go --with-expecter --structname=MockCluster --outpkg=session --inpackage
358341
$(PWD)/bin/mockery --name=Store --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_store.go --with-expecter --structname=MockStore --outpkg=meta --inpackage
359342
$(PWD)/bin/mockery --name=Balance --dir=$(PWD)/internal/querycoordv2/balance --output=$(PWD)/internal/querycoordv2/balance --filename=mock_balancer.go --with-expecter --structname=MockBalancer --outpkg=balance --inpackage
360343
$(PWD)/bin/mockery --name=Controller --dir=$(PWD)/internal/querycoordv2/dist --output=$(PWD)/internal/querycoordv2/dist --filename=mock_controller.go --with-expecter --structname=MockController --outpkg=dist --inpackage
@@ -370,7 +353,5 @@ generate-mockery: getdeps
370353
$(PWD)/bin/mockery --name=Manager --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_manager.go --with-expecter --outpkg=cluster --structname=MockManager --inpackage
371354
$(PWD)/bin/mockery --name=Loader --dir=$(PWD)/internal/querynodev2/segments --output=$(PWD)/internal/querynodev2/segments --filename=mock_loader.go --with-expecter --outpkg=segments --structname=MockLoader --inpackage
372355
$(PWD)/bin/mockery --name=Worker --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_worker.go --with-expecter --outpkg=worker --structname=MockWorker --inpackage
373-
$(PWD)/bin/mockery --name=ShardDelegator --dir=$(PWD)/internal/querynodev2/delegator/ --output=$(PWD)/internal/querynodev2/delegator/ --filename=mock_delegator.go --with-expecter --outpkg=delegator --structname=MockShardDelegator --inpackage
374-
375-
ci-ut: build-cpp-with-coverage generated-proto-go-without-cpp codecov-cpp codecov-go
356+
$(PWD)/bin/mockery --name=ShardDelegator --dir=$(PWD)/internal/querynodev2/delegator/ --output=$(PWD)/internal/querynodev2/delegator/ --filename=mock_delegator.go --with-expecter --outpkg=delegator --structname=MockShardDelegator --inpackage
376357

‎internal/core/src/pb/CMakeLists.txt

+6-43
Original file line numberDiff line numberDiff line change
@@ -10,51 +10,14 @@
1010
# or implied. See the License for the specific language governing permissions and limitations under the License
1111

1212
find_package(Protobuf REQUIRED)
13-
set( proto_file_names
14-
schema.proto
15-
common.proto
16-
)
17-
18-
set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/")
19-
20-
FOREACH(proto_file ${proto_file_names})
21-
STRING( REGEX REPLACE "[^/]proto" "" proto_src_file_name ${proto_file} )
22-
LIST( APPEND milvus_proto_srcs "${proto_src_file_name}.pb.cc" )
2313

24-
ADD_CUSTOM_COMMAND(
25-
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h"
26-
"${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc"
27-
COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/
28-
--cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/
29-
${proto_file}
30-
DEPENDS ${PROTO_PATH}/${proto_file} )
31-
ENDFOREACH(proto_file)
32-
33-
set( proto_file_names
34-
segcore.proto
35-
index_cgo_msg.proto
36-
plan.proto
14+
add_library(milvus_proto STATIC
15+
common.pb.cc
16+
index_cgo_msg.pb.cc
17+
plan.pb.cc
18+
schema.pb.cc
19+
segcore.pb.cc
3720
)
38-
39-
set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../proto/")
40-
set( PROTO_API_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/" )
41-
42-
FOREACH(proto_file ${proto_file_names})
43-
STRING( REGEX REPLACE "[^/]proto" "" proto_src_file_name ${proto_file} )
44-
LIST( APPEND milvus_proto_srcs "${proto_src_file_name}.pb.cc" )
45-
46-
ADD_CUSTOM_COMMAND(
47-
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h"
48-
"${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc"
49-
COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/
50-
--proto_path=${PROTO_API_PATH}/
51-
--cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/
52-
${proto_file}
53-
DEPENDS ${PROTO_PATH}/${proto_file} )
54-
55-
ENDFOREACH(proto_file)
56-
57-
add_library(milvus_proto STATIC ${milvus_proto_srcs})
5821
message(STATUS "milvus proto sources: " ${milvus_proto_srcs})
5922

6023
target_link_libraries( milvus_proto PUBLIC ${CONAN_LIBS} )

‎scripts/3rdparty_build.sh

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the LF AI & Data foundation under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
19+
SOURCE="${BASH_SOURCE[0]}"
20+
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
21+
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
22+
SOURCE="$(readlink "$SOURCE")"
23+
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
24+
done
25+
ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
26+
CPP_SRC_DIR="${ROOT_DIR}/internal/core"
27+
BUILD_OUTPUT_DIR="${ROOT_DIR}/cmake_build"
28+
29+
if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
30+
mkdir ${BUILD_OUTPUT_DIR}
31+
fi
32+
33+
source ${ROOT_DIR}/scripts/setenv.sh
34+
pushd ${BUILD_OUTPUT_DIR}
35+
36+
export CONAN_REVISIONS_ENABLED=1
37+
if [[ ! `conan remote list` == *default-conan-local* ]]; then
38+
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
39+
fi
40+
unameOut="$(uname -s)"
41+
case "${unameOut}" in
42+
Darwin*)
43+
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; }
44+
;;
45+
Linux*)
46+
echo "Running on ${OS_NAME}"
47+
export CPU_TARGET=avx
48+
GCC_VERSION=`gcc -dumpversion`
49+
if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
50+
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; }
51+
else
52+
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
53+
fi
54+
;;
55+
*)
56+
echo "Cannot build on windows"
57+
;;
58+
esac
59+
60+
popd

‎scripts/core_build.sh

+2-41
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ done
149149
if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
150150
mkdir ${BUILD_OUTPUT_DIR}
151151
fi
152+
source ${ROOT_DIR}/scripts/setenv.sh
152153

153154
CMAKE_GENERATOR="Unix Makefiles"
154155

@@ -188,47 +189,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
188189
exit 0
189190
fi
190191

191-
export CONAN_REVISIONS_ENABLED=1
192-
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
193-
unameOut="$(uname -s)"
194-
case "${unameOut}" in
195-
Darwin*)
196-
# detect llvm version by valid list
197-
for llvm_version in 15 14 NOT_FOUND ; do
198-
if brew ls --versions llvm@${llvm_version} > /dev/null; then
199-
break
200-
fi
201-
done
202-
if [ "${llvm_version}" = "NOT_FOUND" ] ; then
203-
echo "valid llvm(14 or 15) not installed"
204-
exit 1
205-
fi
206-
llvm_prefix="$(brew --prefix llvm@${llvm_version})"
207-
export CLANG_TOOLS_PATH="${llvm_prefix}/bin"
208-
export CC="ccache ${llvm_prefix}/bin/clang"
209-
export CXX="ccache ${llvm_prefix}/bin/clang++"
210-
export ASM="${llvm_prefix}/bin/clang"
211-
export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include"
212-
export CXXFLAGS=${CFLAGS}
213-
export LDFLAGS="-L$(brew --prefix libomp)/lib"
214-
215-
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; }
216-
;;
217-
Linux*)
218-
echo "Running on ${OS_NAME}"
219-
export CPU_TARGET=avx
220-
GCC_VERSION=`gcc -dumpversion`
221-
if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
222-
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; }
223-
else
224-
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
225-
fi
226-
;;
227-
*)
228-
echo "Cannot build on windows"
229-
;;
230-
esac
231-
232192
arch=$(uname -m)
233193
CMAKE_CMD="cmake \
234194
${CMAKE_EXTRA_ARGS} \
@@ -250,6 +210,7 @@ ${CMAKE_EXTRA_ARGS} \
250210
-DUSE_ASAN=${USE_ASAN} \
251211
${CPP_SRC_DIR}"
252212

213+
echo "CC $CC"
253214
echo ${CMAKE_CMD}
254215
${CMAKE_CMD} -G "${CMAKE_GENERATOR}"
255216

0 commit comments

Comments
 (0)