diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index f088f6e2e6..a9a82516a9 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -3,26 +3,64 @@ variables: value: https://registrationcenter-download.intel.com/akdlm/irc_nas/17453/w_BaseKit_p_2021.1.0.2664_offline.exe - name: WINDOWS_DPCPP_COMPONENTS value: intel.oneapi.win.dpcpp-compiler + - name: python.version + value: '3.7' + - name: 'PYTHON' + value: python + - name: 'ARGS' + value: '1' jobs: -- job: Linux +- job: Linux_Docs pool: vmImage: 'ubuntu-20.04' steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' - script: | conda update -y -q conda - conda create -q -y -n CB -c conda-forge python=3.7 conda-build=3.21.1 conda-verify - displayName: Create Anaconda environment + conda create -q -y -n CB -c intel python=$(python.version) dal-devel impi-devel + displayName: 'Conda create' - script: | . /usr/share/miniconda/etc/profile.d/conda.sh conda activate CB - conda build --override-channels -c intel -c conda-forge --numpy=1.19 conda-recipe - displayName: conda build + pip install -r requirements-dev.txt + pip install -r requirements-doc.txt + pip install -r requirements-test.txt + displayName: 'Install requirements' + - script: | + . /usr/share/miniconda/etc/profile.d/conda.sh + conda activate CB + export DALROOT=$CONDA_PREFIX + ./conda-recipe/build.sh + displayName: 'Build daal4py' + - script: | + . /usr/share/miniconda/etc/profile.d/conda.sh + conda activate CB + ./conda-recipe/run_test.sh + displayName: Testing + - script: | + . /usr/share/miniconda/etc/profile.d/conda.sh + conda activate CB + cd doc + make html + displayName: 'Build documentation' + - script: | + cp -R doc/_build $(Build.ArtifactStagingDirectory)/html + displayName: 'Copy build' + - task: PublishPipelineArtifact@0 + inputs: + artifactName: 'daal4py-documentation' + targetPath: '$(Build.ArtifactStagingDirectory)/html' - job: Linux_DPCPP pool: vmImage: 'ubuntu-20.04' steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' - script: | wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB @@ -41,7 +79,7 @@ jobs: displayName: 'apt-get' - script: | conda update -y -q conda - conda create -q -y -n CB -c conda-forge python=3.7 conda-build=3.21.1 conda-verify + conda create -q -y -n CB -c conda-forge python=$(python.version) conda-build=3.21.1 conda-verify displayName: Create Anaconda environment - script: | export DPCPPROOT=/opt/intel/oneapi/compiler/latest @@ -59,26 +97,18 @@ jobs: sudo chown -R $USER $CONDA conda config --set always_yes yes --set changeps1 no conda update -q conda - conda create -n CB python=3.7 + conda create -n CB python=$(python.version) conda install -n CB -q cython scipy pytest pandas pyyaml joblib numpydoc jinja2 numpy clang-tools lightgbm scikit-learn mpich conda install -n CB -q --override-channels -c intel dal dal-include displayName: Create Anaconda environment - script: | source activate CB export DALROOT=$CONDA_PREFIX - export MPIROOT=$CONDA_PREFIX - export MACOSX_DEPLOYMENT_TARGET=10.15 - export LDFLAGS="${LDFLAGS//-Wl,-dead_strip_dylibs}" - export LDFLAGS_LD="${LDFLAGS_LD//-dead_strip_dylibs}" - export LDSHARED="-bundle -undefined dynamic_lookup -flto -Wl,-export_dynamic -Wl,-pie -Wl,-headerpad_max_install_names" - python setup.py install --single-version-externally-managed --record=record.txt + ./conda-recipe/build.sh displayName: Conda build - script: | source activate CB - python -c "import daal4py" - mpirun -n 4 python -m unittest discover -v -s tests -p spmd*.py - pytest --pyargs daal4py/sklearn/ - python tests/run_tests.py + ./conda-recipe/run_test.sh displayName: Testing - script: | source activate CB @@ -88,14 +118,15 @@ jobs: - job: Windows_DPCPP pool: vmImage: 'windows-latest' - steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' - script: .ci/scripts/install_windows.bat $(WINDOWS_BASEKIT_URL) $(WINDOWS_DPCPP_COMPONENTS) timeoutInMinutes: 15 displayName: oneAPI environment - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" displayName: Add conda to PATH - - script: conda create -q -y -n CB python=3.7 conda-build conda-verify + - script: conda create -q -y -n CB python=$(python.version) conda-build conda-verify displayName: Create Anaconda environment - script: | call activate CB diff --git a/.ci/pipeline/docs.yml b/.ci/pipeline/docs.yml deleted file mode 100644 index ceac37fb8d..0000000000 --- a/.ci/pipeline/docs.yml +++ /dev/null @@ -1,41 +0,0 @@ -trigger: - branches: - include: - - master - paths: - include: - - doc - - examples - -pr: - branches: - include: - - master - paths: - include: - - doc - - examples - -pool: - vmImage: 'ubuntu-20.04' - -steps: -- script: | - cd doc - sudo apt-get install python3-setuptools - sudo pip3 install -r requirements.txt - displayName: 'Install requirements' - -- script: | - cd doc - make html - displayName: 'Build documentation' - -- script: | - cp -R doc/_build $(Build.ArtifactStagingDirectory)/html - displayName: 'Copy build' - -- task: PublishPipelineArtifact@0 - inputs: - artifactName: 'daal4py-documentation' - targetPath: '$(Build.ArtifactStagingDirectory)/html' diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..d09ca9f093 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Shell scripts require LF +*.sh text eol=lf +# Batch scripts require CRLF +*.bat text eol=crlf diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh old mode 100644 new mode 100755 diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh old mode 100644 new mode 100755 index 3252a466d4..8e6d3385f2 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -1,7 +1,7 @@ #!/bin/bash #******************************************************************************* -# Copyright 2014-2020 Intel Corporation +# Copyright 2014-2021 Intel Corporation # All Rights Reserved. # # This software is licensed under the Apache License, Version 2.0 (the @@ -18,27 +18,7 @@ # limitations under the License. #******************************************************************************* -# if dpc++ vars path is specified -if [ ! -z "${DPCPPROOT}" ]; then - source ${DPCPPROOT}/env/vars.sh -fi - -# if DAALROOT is specified -if [ "${DAALROOT}" != "" ] && [ "${DALROOT}" == "" ] ; then - export DALROOT="${DAALROOT}" -fi - -if [ "${DALROOT}" != "" ]; then - conda remove daal --force -y - source ${DALROOT}/env/vars.sh -fi - -# if TBBROOT is specified -if [ ! -z "${TBBROOT}" ]; then - conda remove tbb --force -y - if [ -e ${TBBROOT}/env/vars.sh ]; then - source ${TBBROOT}/env/vars.sh - else - export LD_LIBRARY_PATH=${TBBROOT}/lib/intel64:${LD_LIBRARY_PATH} - fi -fi +python -c "import daal4py" +mpirun -n 4 python -m unittest discover -v -s tests -p spmd*.py +pytest --pyargs daal4py/sklearn/ +python tests/run_tests.py diff --git a/examples/svm_multiclass_batch.py b/examples/svm_multiclass_batch.py index b26b9cf6ea..dfa3794330 100644 --- a/examples/svm_multiclass_batch.py +++ b/examples/svm_multiclass_batch.py @@ -41,7 +41,7 @@ def main(readcsv=read_csv, method='defaultDense'): algorithm = d4p.multi_class_classifier_training(nClasses=nClasses, training=d4p.svm_training(method='thunder'), prediction=d4p.svm_prediction()) - + # Pass data to training. Training result provides model train_result = algorithm.compute(train_data, train_labels) assert train_result.model.NumberOfFeatures == nFeatures @@ -52,7 +52,7 @@ def main(readcsv=read_csv, method='defaultDense'): pred_file = 'data/batch/svm_multi_class_test_dense.csv' pred_data = readcsv(pred_file, range(nFeatures)) pred_labels = readcsv(pred_file, range(nFeatures, nFeatures + 1)) - + # Create an algorithm object to predict multi-class SVM values algorithm = d4p.multi_class_classifier_prediction(nClasses, training=d4p.svm_training(method='thunder'), @@ -60,13 +60,12 @@ def main(readcsv=read_csv, method='defaultDense'): # Pass data to prediction. Prediction result provides prediction pred_result = algorithm.compute(pred_data, train_result.model) assert pred_result.prediction.shape == (train_data.shape[0], 1) - + return (pred_result, pred_labels) if __name__ == "__main__": (pred_res, pred_labels) = main() - print("\nSVM classification results (first 20 observations):\n", pred_res.prediction[0:20]) print("\nGround truth (first 20 observations):\n", pred_labels[0:20]) print('All looks good!') diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100755 index 0000000000..d4c13c2ed8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +Cython==0.29.21 +Jinja2==2.11.2 +numpy>=1.14 +clang-format==9.0.0 diff --git a/doc/requirements.txt b/requirements-doc.txt similarity index 100% rename from doc/requirements.txt rename to requirements-doc.txt diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000000..0ed611caae --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,5 @@ +pytest==6.2.1 +scikit-learn==0.24.1 +pandas==1.2.1 +lightgbm==3.1.1 +xgboost==1.3.3