Skip to content

Commit 5ad89ff

Browse files
czs007czs007
authored
Add offline build logic for arrow dependency (milvus-io#17266)
Signed-off-by: zhenshan.cao <[email protected]>
1 parent b09359b commit 5ad89ff

File tree

3 files changed

+152
-16
lines changed

3 files changed

+152
-16
lines changed

‎.github/workflows/code-checker.yaml

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -43,37 +43,37 @@ jobs:
4343
uses: actions/checkout@v2
4444
- name: Cache CCache Volumes
4545
if: ${{ matrix.os == 'ubuntu-18.04' }}
46-
uses: actions/cache@v1
46+
uses: actions/cache@v3
4747
with:
4848
path: .docker/amd64-ubuntu18.04-ccache
4949
key: ubuntu18.04-ccache-${{ hashFiles('internal/core/**') }}
5050
restore-keys: ubuntu18.04-ccache-
5151
- name: Cache Third Party
5252
if: ${{ matrix.os == 'ubuntu-18.04' }}
53-
uses: actions/cache@v1
53+
uses: actions/cache@v3
5454
with:
5555
path: .docker/thirdparty
56-
key: ubuntu18.04-thirdparty-${{ hashFiles('internal/core/**') }}
56+
key: ubuntu18.04-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
5757
restore-keys: ubuntu18.04-thirdparty-
5858
- name: Mac Cache Third Party
5959
if: ${{ matrix.os == 'macos-11' }}
60-
uses: actions/cache@v1
60+
uses: actions/cache@v3
6161
with:
6262
path: |
6363
/tmp/thirdparty
64-
cmake_build/3rdparty_download
65-
key: macos-11-thirdparty-${{ hashFiles('internal/core/**') }}
64+
cmake_build/3rdparty_download/download
65+
key: macos-11-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
6666
restore-keys: macos-11-thirdparty-
6767
- name: Mac Cache CCache Volumes
6868
if: ${{ matrix.os == 'macos-11' }}
69-
uses: actions/cache@v1
69+
uses: actions/cache@v3
7070
with:
7171
path: /var/tmp/ccache
7272
key: macos-11-ccache-${{ hashFiles('internal/core/**') }}
7373
restore-keys: macos-11-ccache-
7474
- name: Cache Go Mod Volumes
7575
if: ${{ matrix.os == 'ubuntu-18.04' }}
76-
uses: actions/cache@v1
76+
uses: actions/cache@v3
7777
with:
7878
path: .docker/amd64-ubuntu18.04-go-mod
7979
key: ubuntu18.04-go-mod-${{ hashFiles('**/go.sum') }}
@@ -85,7 +85,7 @@ jobs:
8585
go-version: '~1.17.1'
8686
- name: Mac Cache Go Mod Volumes
8787
if: ${{ matrix.os == 'macos-11' }}
88-
uses: actions/cache@v1
88+
uses: actions/cache@v3
8989
with:
9090
path: ~/go/pkg/mod
9191
key: macos-11-go-mod-${{ hashFiles('**/go.sum') }}
@@ -110,7 +110,7 @@ jobs:
110110
mkdir -p /var/tmp/ccache
111111
fi
112112
brew install boost libomp ninja tbb openblas ccache
113-
make check-proto-product && make verifiers
113+
make check-proto-product && make verifiers && sh scripts/collect_arrow_dep.sh -f /tmp/thirdparty
114114
centos:
115115
name: Code Checker CentOS 7
116116
# Run in centos docker
@@ -120,19 +120,19 @@ jobs:
120120
- name: Checkout
121121
uses: actions/checkout@v2
122122
- name: Cache CCache Volumes
123-
uses: actions/cache@v2
123+
uses: actions/cache@v3
124124
with:
125125
path: .docker/amd64-centos7-ccache
126126
key: centos7-ccache-${{ hashFiles('internal/core/**') }}
127127
restore-keys: centos7-ccache-
128128
- name: Cache Third Party
129-
uses: actions/cache@v2
129+
uses: actions/cache@v3
130130
with:
131131
path: .docker/thirdparty
132-
key: centos7-thirdparty-${{ hashFiles('internal/core/**') }}
132+
key: centos7-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
133133
restore-keys: centos7-thirdparty-
134134
- name: Cache Go Mod Volumes
135-
uses: actions/cache@v2
135+
uses: actions/cache@v3
136136
with:
137137
path: .docker/amd64-centos7-go-mod
138138
key: centos7-go-mod-${{ hashFiles('**/go.sum') }}
@@ -161,15 +161,15 @@ jobs:
161161
- name: Checkout
162162
uses: actions/checkout@v2
163163
- name: Cache go
164-
uses: actions/cache@v2
164+
uses: actions/cache@v3
165165
with:
166166
path: |
167167
%LocalAppData%\go-build
168168
~/go/pkg/mod
169169
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
170170
restore-keys: ${{ runner.os }}-go-
171171
- name: Cache ccache
172-
uses: actions/cache@v2
172+
uses: actions/cache@v3
173173
with:
174174
path: |
175175
${{ github.workspace }}/.ccache

‎scripts/collect_arrow_dep.sh

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/usr/bin/env bash
2+
3+
# Licensed to the LF AI & Data foundation under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
19+
# assume we are in milvus directory
20+
21+
CUSTOM_THIRDPARTY_PATH=
22+
23+
while getopts "f:h" arg; do
24+
case $arg in
25+
f)
26+
CUSTOM_THIRDPARTY_PATH=$OPTARG
27+
;;
28+
29+
h) # help
30+
echo "
31+
32+
parameter:
33+
-f: custom paths of thirdparty downloaded files(default: NULL)
34+
-h: help
35+
36+
usage:
37+
./collect_arrow_dep.sh -f\${CUSTOM_THIRDPARTY_PATH} [-h]
38+
"
39+
exit 0
40+
;;
41+
?)
42+
echo "ERROR! unknown argument"
43+
exit 1
44+
;;
45+
esac
46+
done
47+
48+
49+
if [ -z "$CUSTOM_THIRDPARTY_PATH" ]
50+
then
51+
echo "
52+
parameter:
53+
-f: custom paths of thirdparty downloaded files(default: NULL)
54+
-h: help
55+
56+
usage:
57+
./core_build.sh -f\${CUSTOM_THIRDPARTY_PATH} [-h]
58+
"
59+
exit 0
60+
fi
61+
62+
if [ ! -d "$CUSTOM_THIRDPARTY_PATH" ]
63+
then
64+
echo "${CUSTOM_THIRDPARTY_PATH} is not a directory"
65+
exit 0
66+
fi
67+
68+
SOURCE="${BASH_SOURCE[0]}"
69+
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
70+
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
71+
SOURCE="$(readlink "$SOURCE")"
72+
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
73+
done
74+
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
75+
76+
BUILD_OUTPUT_DIR="${SCRIPTS_DIR}/../cmake_build"
77+
ARROW_DIR=${BUILD_OUTPUT_DIR}/thirdparty/arrow
78+
ARROWBIN_DIR=${BUILD_OUTPUT_DIR}/thirdparty/arrow/arrow-bin
79+
80+
echo ${ARROWBIN_DIR}
81+
82+
pushd ${ARROWBIN_DIR}
83+
targetNames=(
84+
"jemalloc-5.2.1.tar.bz2"
85+
"thrift-0.13.0.tar.gz"
86+
"utf8proc-v2.7.0.tar.gz"
87+
"xsimd-7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz"
88+
"zstd-v1.5.1.tar.gz"
89+
)
90+
91+
srcNames=(
92+
"jemalloc_ep-prefix/src/jemalloc-5.2.1.tar.bz2"
93+
"thrift_ep-prefix/src/thrift-0.13.0.tar.gz"
94+
"utf8proc_ep-prefix/src/v2.7.0.tar.gz"
95+
"src/7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz"
96+
"zstd_ep-prefix/src/v1.5.1.tar.gz"
97+
)
98+
99+
for i in "${!srcNames[@]}"; do
100+
if test -f "${srcNames[i]}"; then
101+
echo "${srcNames[i]} exists. start to copy to ${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}"
102+
echo cp -f "${srcNames[i]}" "${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}"
103+
cp -f "${srcNames[i]}" "${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}"
104+
fi
105+
done
106+
popd

‎scripts/core_build.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,30 @@ echo ${CMAKE_CMD}
204204
${CMAKE_CMD} -G "${CMAKE_GENERATOR}"
205205

206206

207+
# enable offline build of arrow dependency if files exist.
208+
arrowDepKeys=(
209+
"ARROW_JEMALLOC_URL"
210+
"ARROW_THRIFT_URL"
211+
"ARROW_UTF8PROC_URL"
212+
"ARROW_XSIMD_URL"
213+
"ARROW_ZSTD_URL"
214+
)
215+
arrowDepValues=(
216+
"jemalloc-5.2.1.tar.bz2"
217+
"thrift-0.13.0.tar.gz"
218+
"utf8proc-v2.7.0.tar.gz"
219+
"xsimd-7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz"
220+
"zstd-v1.5.1.tar.gz"
221+
)
222+
for i in "${!arrowDepValues[@]}"; do
223+
if test -f "${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]}"; then
224+
echo "${arrowDepValues[$i]} exists."
225+
export ${arrowDepKeys[$i]}=${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]}
226+
fi
227+
done
228+
229+
set
230+
207231
if [[ ${RUN_CPPLINT} == "ON" ]]; then
208232
# cpplint check
209233
make lint
@@ -233,4 +257,10 @@ else
233257
make -j ${jobs} install || exit 1
234258
fi
235259

260+
if command -v ccache &> /dev/null
261+
then
262+
ccache -s
263+
exit
264+
fi
265+
236266
popd

0 commit comments

Comments
 (0)