[ML-202] Update oneAPI Base Toolkit version and prepare for OAP 1.3.1 release (#203)

* oneapi 2022.1 changes: cpu_icc => cpu/libccl.so.1.0, libtbb.so.12.5, clang => dpcpp

* [ML-202] Update oneAPI Base Toolkit version and prepare for OAP 1.3.1 release

* Update deps version for Ubuntu and README

* Remove cached oneAPI in workflow

* Update RELEASE

* Adjust compiler: gcc for CPU and dpcpp for GPU

* nit

* Modify CI checks

* Update .github/workflows/ci-tests.yml

Co-authored-by: Wu, Xiaochang <[email protected]>

Author: Hong

.github/workflows/ci-checks-build.yml

@@ -62,7 +62,7 @@ jobs:
             ~/.m2/repository
             /opt/intel/oneapi
             ~/opt
-          key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2021.4.0
+          key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2022.1.2
           restore-keys: |
             ${{ runner.os }}-
       - name: Build Test for CPU and GPU

.github/workflows/ci-tests.yml

@@ -4,7 +4,7 @@ on: [push, pull_request]
 
 jobs:
   local-test:
-    name: Local Test for Units
+    name: Local Test for Units (CPU)
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
@@ -20,15 +20,15 @@ jobs:
             ~/.m2/repository
             /opt/intel/oneapi
             ~/opt
-          key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2021.4.0
+          key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2022.1.2
           restore-keys: |
             ${{ runner.os }}-
       - name: Local Test
         run: |
           ${{github.workspace}}/dev/ci/ci-local-test.sh
 
   cluster-test:
-    name: Cluster Test for Examples
+    name: Cluster Test for Examples (CPU)
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
@@ -44,7 +44,7 @@ jobs:
             ~/.m2/repository
             /opt/intel/oneapi
             ~/opt
-          key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2021.4.0
+          key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2022.1.2
           restore-keys: |
             ${{ runner.os }}-
       - name: Cluster Test

.vscode/c_cpp_properties.json

@@ -9,7 +9,7 @@
                 "${JAVA_HOME}/include/**"
             ],
             "defines": [],
-            "compilerPath": "${CMPLR_ROOT}/linux/bin/clang",
+            "compilerPath": "${CMPLR_ROOT}/linux/bin/dpcpp",
             "cStandard": "c17",
             "cppStandard": "c++14",
             "intelliSenseMode": "clang-x64"

README.md

@@ -24,7 +24,7 @@ You can find the all the OAP MLlib documents on the [project web page](https://o
 
 ### Java/Scala Users Preferred
 
-Use a pre-built OAP MLlib JAR to get started. You can firstly download OAP package from [OAP-JARs-Tarball](https://github.com/oap-project/oap-tools/releases/download/v1.3.0/oap-1.3.0-bin.tar.gz) and extract this Tarball to get `oap-mllib-x.x.x.jar` under `oap-x.x.x-bin-spark-x.x.x/jars`.
+Use a pre-built OAP MLlib JAR to get started. You can firstly download OAP package from [OAP-JARs-Tarball](https://github.com/oap-project/oap-tools/releases/download/v1.3.1/oap-1.3.1-bin.tar.gz) and extract this Tarball to get `oap-mllib-x.x.x.jar` under `oap-x.x.x-bin-spark-x.x.x/jars`.
 
 Then you can refer to the following [Running](#running) section to try out.
 
@@ -118,7 +118,7 @@ We use [Apache Maven](https://maven.apache.org/) to manage and build source code
 * JDK 8.0+
 * Apache Maven 3.6.2+
 * GNU GCC 4.8.5+
-* Intel® oneAPI Base Toolkit (>=2021.4.0) Components :
+* Intel® oneAPI Base Toolkit (>=2022.1) Components :
     - DPC++/C++ Compiler (dpcpp/clang++)
     - Data Analytics Library (oneDAL)
     - Threading Building Blocks (oneTBB)
@@ -158,7 +158,7 @@ We suggest you to source `setvars.sh` script into current shell to setup buildin
     $ source /opt/intel/oneapi/setvars.sh
 ```
 
-You can also refer to [this CI script](dev/ci-test.sh) to setup the building environments.
+You can also refer to [this CI script](dev/ci/ci-build-test.sh) to setup the building environments.
 
 If you prefer to buid your own open source [oneDAL](https://github.com/oneapi-src/oneDAL), [oneTBB](https://github.com/oneapi-src/oneTBB), [oneCCL](https://github.com/oneapi-src/oneCCL) versions rather than use the ones included in oneAPI Base Toolkit, you can refer to the related build instructions and manually source `setvars.sh` accordingly.
 

RELEASE

@@ -1,3 +1,3 @@
-OAP_MLLIB_VERSION=1.3.0
+OAP_MLLIB_VERSION=1.3.1
 SPARK_VERSION=3.2.0
 PLATFORM_PROFILE=CPU_ONLY_PROFILE

dev/install-build-deps-centos.sh

@@ -15,7 +15,7 @@ EOF
   sudo mv /tmp/oneAPI.repo /etc/yum.repos.d  
   # sudo yum groupinstall -y "Development Tools"
   # sudo yum install -y cmake
-  sudo yum install -y intel-oneapi-dpcpp-cpp-2021.4.0 intel-oneapi-dal-devel-2021.4.0 intel-oneapi-tbb-devel-2021.4.0 intel-oneapi-ccl-devel-2021.4.0 intel-oneapi-mpi-devel-2021.4.0
+  sudo yum install -y intel-oneapi-dpcpp-cpp-2022.0.2 intel-oneapi-dal-devel-2021.5.3  intel-oneapi-tbb-devel-2021.5.1 intel-oneapi-ccl-devel-2021.5.1 intel-oneapi-mpi-devel-2021.5.1
 else
   echo "oneAPI components already installed!"
 fi  

dev/install-build-deps-ubuntu.sh

@@ -9,7 +9,7 @@ if [ ! -d /opt/intel/oneapi ]; then
   echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
   sudo apt-get update
   # sudo apt-get install -y build-essential cmake
-  sudo apt-get install -y intel-oneapi-dpcpp-cpp-2021.4.0 intel-oneapi-dal-devel-2021.4.0 intel-oneapi-tbb-devel-2021.4.0 intel-oneapi-ccl-devel-2021.4.0 intel-oneapi-mpi-devel-2021.4.0
+  sudo apt-get install -y intel-oneapi-dpcpp-cpp-2022.0.2 intel-oneapi-dal-devel-2021.5.3  intel-oneapi-tbb-devel-2021.5.1 intel-oneapi-ccl-devel-2021.5.1 intel-oneapi-mpi-devel-2021.5.1
 else
   echo "oneAPI components already installed!"
 fi

dev/prepare-build-deps-gpu.sh

@@ -48,7 +48,7 @@ TARGET_DIR=./src/main/resources/lib
 
 rm -f $TARGET_DIR/*.so*
 
-cp $CCL_ROOT/lib/cpu_icc/libccl.so.1.0 $TARGET_DIR/libccl.so.1
+cp $CCL_ROOT/lib/cpu/libccl.so.1.0 $TARGET_DIR/libccl.so.1
 
 cp $I_MPI_ROOT/libfabric/lib/libfabric.so.1 $TARGET_DIR/libfabric.so.1
 cp $I_MPI_ROOT/libfabric/lib/prov/libsockets-fi.so $TARGET_DIR
@@ -61,8 +61,8 @@ cp $I_MPI_ROOT/lib/release_mt/libmpi.so.12.0.0 $TARGET_DIR/libmpi.so.12
 
 cp $DAALROOT/lib/intel64/libJavaAPI.so.1.1 $TARGET_DIR/libJavaAPI.so
 
-cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.4 $TARGET_DIR/libtbb.so.12
-cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.4 $TARGET_DIR/libtbbmalloc.so.2
+cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.5 $TARGET_DIR/libtbb.so.12
+cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.5 $TARGET_DIR/libtbbmalloc.so.2
 
 # SYCL libs
 cp $CMPLR_ROOT/linux/compiler/lib/intel64_lin/libintlc.so.5 $TARGET_DIR

dev/prepare-build-deps.sh

@@ -48,7 +48,7 @@ TARGET_DIR=./src/main/resources/lib
 
 rm -f $TARGET_DIR/*.so*
 
-cp $CCL_ROOT/lib/cpu_icc/libccl.so.1.0 $TARGET_DIR/libccl.so.1
+cp $CCL_ROOT/lib/cpu/libccl.so.1.0 $TARGET_DIR/libccl.so.1
 
 cp $I_MPI_ROOT/libfabric/lib/libfabric.so.1 $TARGET_DIR/libfabric.so.1
 cp $I_MPI_ROOT/libfabric/lib/prov/libsockets-fi.so $TARGET_DIR
@@ -61,7 +61,7 @@ cp $I_MPI_ROOT/lib/release_mt/libmpi.so.12.0.0 $TARGET_DIR/libmpi.so.12
 
 cp $DAALROOT/lib/intel64/libJavaAPI.so.1.1 $TARGET_DIR/libJavaAPI.so
 
-cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.4 $TARGET_DIR/libtbb.so.12
-cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.4 $TARGET_DIR/libtbbmalloc.so.2
+cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.5 $TARGET_DIR/libtbb.so.12
+cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.5 $TARGET_DIR/libtbbmalloc.so.2
 
 echo oneAPI Toolkit version: $(basename $CCL_ROOT) > $TARGET_DIR/VERSION

examples/als/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.intel.oap</groupId>
   <artifactId>oap-mllib-examples</artifactId>
-  <version>1.3.0</version>
+  <version>1.3.1</version>
   <packaging>jar</packaging>
 
   <name>ALSExample</name>

examples/correlation/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.intel.oap</groupId>
   <artifactId>oap-mllib-examples</artifactId>
-  <version>1.3.0</version>  
+  <version>1.3.1</version>
   <packaging>jar</packaging>
 
   <name>CorrelationExample</name>

examples/kmeans/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.intel.oap</groupId>
   <artifactId>oap-mllib-examples</artifactId>
-  <version>1.3.0</version>
+  <version>1.3.1</version>
   <packaging>jar</packaging>
 
   <name>KMeansExample</name>

examples/linear-regression/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.intel.oap</groupId>
   <artifactId>oap-mllib-examples</artifactId>
-  <version>1.3.0</version>  
+  <version>1.3.1</version>
   <packaging>jar</packaging>
 
   <name>LinearRegressionExample</name>

examples/naive-bayes/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.intel.oap</groupId>
   <artifactId>oap-mllib-examples</artifactId>
-  <version>1.3.0</version>  
+  <version>1.3.1</version>
   <packaging>jar</packaging>
 
   <name>NaiveBayesExample</name>

examples/pca/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.intel.oap</groupId>
   <artifactId>oap-mllib-examples</artifactId>
-  <version>1.3.0</version>  
+  <version>1.3.1</version>
   <packaging>jar</packaging>
 
   <name>PCAExample</name>

examples/summarizer/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.intel.oap</groupId>
   <artifactId>oap-mllib-examples</artifactId>
-  <version>1.3.0</version>
+  <version>1.3.1</version>
   <packaging>jar</packaging>
 
   <name>SummaryStatisticsExample</name>

mllib-dal/build.sh

@@ -128,7 +128,15 @@ echo DAALROOT=$DAALROOT
 echo TBBROOT=$TBBROOT
 echo CCL_ROOT=$CCL_ROOT
 echo Maven Version: $(mvn -v | head -n 1 | cut -f3 -d" ")
-echo Clang Version: $(clang -dumpversion)
+
+if [[ $PLATFORM_PROFILE == CPU_ONLY_PROFILE ]]
+then
+  echo GCC Version: $(gcc -dumpversion)
+elif [[ $PLATFORM_PROFILE == CPU_GPU_PROFILE ]]
+then
+  echo DPCPP Version: $(dpcpp -dumpversion)
+fi
+
 echo Spark Version: $SPARK_VERSION
 echo Platform Profile: $PLATFORM_PROFILE
 echo =============================

mllib-dal/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.intel.oap</groupId>
     <artifactId>oap-mllib</artifactId>
-    <version>1.3.0</version>
+    <version>1.3.1</version>
     <name>OAP Project OAP MLlib</name>
     <packaging>jar</packaging>
     <url>https://github.com/oap-project/oap-mllib.git</url>
@@ -17,7 +17,7 @@
         <scala.binary.version>2.12</scala.binary.version>
         <scalatest.version>3.2.9</scalatest.version>
         <spark.version>3.2.0</spark.version>
-        <dal.version>2021.4.0.83</dal.version>
+        <dal.version>2021.5.0.91</dal.version>
         <assembly.description>src/assembly/assembly.xml</assembly.description>
     </properties>
 

mllib-dal/src/main/native/Makefile

@@ -12,8 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CC := clang
-CXX := clang++
+# Use gcc for CPU and dpcpp for GPU
+ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE)
+  CC := gcc
+  CXX := g++
+else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE)
+  CC := dpcpp
+  CXX := dpcpp
+endif
+
 RM := rm -rf
 
 PLATFORM_PROFILE ?= CPU_ONLY_PROFILE
@@ -35,13 +42,13 @@ endif
 
 INCS := -I $(JAVA_HOME)/include \
         -I $(JAVA_HOME)/include/linux \
-        -I $(CCL_ROOT)/include/cpu_icc \
+        -I $(CCL_ROOT)/include/cpu \
         -I $(DAALROOT)/include \
         -I ./javah \
         -I ./
 
 # Use static link if possible, TBB is only available as dynamic libs
-LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu_icc -lccl \
+LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \
         -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \
         -L$(DAALROOT)/lib/intel64 -l:libonedal_core.a -l:libonedal_thread.a  \
         -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc