Update pipline for docs (#477)

.ci/pipeline/ci.yml

@@ -3,26 +3,64 @@ variables:
     value: https://registrationcenter-download.intel.com/akdlm/irc_nas/17453/w_BaseKit_p_2021.1.0.2664_offline.exe
   - name: WINDOWS_DPCPP_COMPONENTS
     value: intel.oneapi.win.dpcpp-compiler
+  - name: python.version
+    value: '3.7'
+  - name: 'PYTHON'
+    value: python
+  - name: 'ARGS'
+    value: '1'
 
 jobs:
-- job: Linux
+- job: Linux_Docs
   pool:
     vmImage: 'ubuntu-20.04'
   steps:
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(python.version)'
+    inputs:
+      versionSpec: '$(python.version)'
   - script: |
       conda update -y -q conda
-      conda create -q -y -n CB -c conda-forge python=3.7 conda-build=3.21.1 conda-verify
-    displayName: Create Anaconda environment
+      conda create -q -y -n CB -c intel python=$(python.version) dal-devel impi-devel
+    displayName: 'Conda create'
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
-      conda build --override-channels -c intel -c conda-forge --numpy=1.19 conda-recipe
-    displayName: conda build
+      pip install -r requirements-dev.txt
+      pip install -r requirements-doc.txt
+      pip install -r requirements-test.txt
+    displayName: 'Install requirements'
+  - script: |
+      . /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate CB
+      export DALROOT=$CONDA_PREFIX
+      ./conda-recipe/build.sh
+    displayName: 'Build daal4py'
+  - script: |
+      . /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate CB
+      ./conda-recipe/run_test.sh
+    displayName: Testing
+  - script: |
+      . /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate CB
+      cd doc
+      make html
+    displayName: 'Build documentation'
+  - script: |
+      cp -R doc/_build $(Build.ArtifactStagingDirectory)/html
+    displayName: 'Copy build'
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: 'daal4py-documentation'
+      targetPath: '$(Build.ArtifactStagingDirectory)/html'
 
 - job: Linux_DPCPP
   pool:
     vmImage: 'ubuntu-20.04'
   steps:
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(python.version)'
   - script: |
       wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
       sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
@@ -41,7 +79,7 @@ jobs:
     displayName: 'apt-get'
   - script: |
       conda update -y -q conda
-      conda create -q -y -n CB -c conda-forge python=3.7 conda-build=3.21.1 conda-verify
+      conda create -q -y -n CB -c conda-forge python=$(python.version) conda-build=3.21.1 conda-verify
     displayName: Create Anaconda environment
   - script: |
       export DPCPPROOT=/opt/intel/oneapi/compiler/latest
@@ -59,26 +97,18 @@ jobs:
       sudo chown -R $USER $CONDA
       conda config --set always_yes yes --set changeps1 no
       conda update -q conda
-      conda create -n CB python=3.7
+      conda create -n CB python=$(python.version)
       conda install -n CB -q cython scipy pytest pandas pyyaml joblib numpydoc jinja2 numpy clang-tools lightgbm scikit-learn mpich
       conda install -n CB -q --override-channels -c intel dal dal-include
     displayName: Create Anaconda environment
   - script: |
       source activate CB
       export DALROOT=$CONDA_PREFIX
-      export MPIROOT=$CONDA_PREFIX
-      export MACOSX_DEPLOYMENT_TARGET=10.15
-      export LDFLAGS="${LDFLAGS//-Wl,-dead_strip_dylibs}"
-      export LDFLAGS_LD="${LDFLAGS_LD//-dead_strip_dylibs}"
-      export LDSHARED="-bundle -undefined dynamic_lookup -flto -Wl,-export_dynamic -Wl,-pie -Wl,-headerpad_max_install_names"
-      python setup.py install --single-version-externally-managed --record=record.txt
+      ./conda-recipe/build.sh
     displayName: Conda build
   - script: |
       source activate CB
-      python -c "import daal4py"
-      mpirun -n 4 python -m unittest discover -v -s tests -p spmd*.py
-      pytest --pyargs daal4py/sklearn/
-      python tests/run_tests.py
+      ./conda-recipe/run_test.sh
     displayName: Testing
   - script: |
       source activate CB
@@ -88,14 +118,15 @@ jobs:
 - job: Windows_DPCPP
   pool:
     vmImage: 'windows-latest'
-
   steps:
+  - task: UsePythonVersion@0
+    displayName: 'Use Python $(python.version)'
   - script: .ci/scripts/install_windows.bat $(WINDOWS_BASEKIT_URL) $(WINDOWS_DPCPP_COMPONENTS)
     timeoutInMinutes: 15
     displayName: oneAPI environment
   - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
     displayName: Add conda to PATH
-  - script: conda create -q -y -n CB python=3.7 conda-build conda-verify
+  - script: conda create -q -y -n CB python=$(python.version) conda-build conda-verify
     displayName: Create Anaconda environment
   - script: |
       call activate CB

.gitattributes

@@ -0,0 +1,4 @@
+# Shell scripts require LF
+*.sh    text eol=lf
+# Batch scripts require CRLF
+*.bat   text eol=crlf

conda-recipe/run_test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 #*******************************************************************************
-# Copyright 2014-2020 Intel Corporation
+# Copyright 2014-2021 Intel Corporation
 # All Rights Reserved.
 #
 # This software is licensed under the Apache License, Version 2.0 (the
@@ -18,27 +18,7 @@
 # limitations under the License.
 #*******************************************************************************
 
-# if dpc++ vars path is specified
-if [ ! -z "${DPCPPROOT}" ]; then
-    source ${DPCPPROOT}/env/vars.sh
-fi
-
-# if DAALROOT is specified
-if [ "${DAALROOT}" != "" ] && [ "${DALROOT}" == "" ] ; then
-    export DALROOT="${DAALROOT}"
-fi
-
-if [ "${DALROOT}" != "" ]; then
-    conda remove daal --force -y
-    source ${DALROOT}/env/vars.sh
-fi
-
-# if TBBROOT is specified
-if [ ! -z "${TBBROOT}" ]; then
-    conda remove tbb --force -y
-    if [ -e ${TBBROOT}/env/vars.sh ]; then
-        source ${TBBROOT}/env/vars.sh
-    else
-        export LD_LIBRARY_PATH=${TBBROOT}/lib/intel64:${LD_LIBRARY_PATH}
-    fi
-fi
+python -c "import daal4py"
+mpirun -n 4 python -m unittest discover -v -s tests -p spmd*.py
+pytest --pyargs daal4py/sklearn/
+python tests/run_tests.py

examples/svm_multiclass_batch.py

@@ -41,7 +41,7 @@ def main(readcsv=read_csv, method='defaultDense'):
     algorithm = d4p.multi_class_classifier_training(nClasses=nClasses,
                                                     training=d4p.svm_training(method='thunder'),
                                                     prediction=d4p.svm_prediction())
-    
+
     # Pass data to training. Training result provides model
     train_result = algorithm.compute(train_data, train_labels)
     assert train_result.model.NumberOfFeatures == nFeatures
@@ -52,21 +52,20 @@ def main(readcsv=read_csv, method='defaultDense'):
     pred_file = 'data/batch/svm_multi_class_test_dense.csv'
     pred_data = readcsv(pred_file, range(nFeatures))
     pred_labels = readcsv(pred_file, range(nFeatures, nFeatures + 1))
-    
+
     # Create an algorithm object to predict multi-class SVM values
     algorithm = d4p.multi_class_classifier_prediction(nClasses,
                                                       training=d4p.svm_training(method='thunder'),
                                                       prediction=d4p.svm_prediction())
     # Pass data to prediction. Prediction result provides prediction
     pred_result = algorithm.compute(pred_data, train_result.model)
     assert pred_result.prediction.shape == (train_data.shape[0], 1)
-    
+
     return (pred_result, pred_labels)
 
 
 if __name__ == "__main__":
     (pred_res, pred_labels) = main()
-
     print("\nSVM classification results (first 20 observations):\n", pred_res.prediction[0:20])
     print("\nGround truth (first 20 observations):\n", pred_labels[0:20])
     print('All looks good!')

requirements-dev.txt

@@ -0,0 +1,4 @@
+Cython==0.29.21
+Jinja2==2.11.2
+numpy>=1.14
+clang-format==9.0.0

requirements-test.txt

@@ -0,0 +1,5 @@
+pytest==6.2.1
+scikit-learn==0.24.1
+pandas==1.2.1
+lightgbm==3.1.1
+xgboost==1.3.3