diff --git a/1-Training/AzureServiceClassifier_Training.ipynb b/1-Training/AzureServiceClassifier_Training.ipynb index 815fc58..8db0e1e 100644 --- a/1-Training/AzureServiceClassifier_Training.ipynb +++ b/1-Training/AzureServiceClassifier_Training.ipynb @@ -29,7 +29,7 @@ "- Introduction to Transformer and BERT deep learning models\n", "- Introduction to Azure Machine Learning service\n", "- Preparing raw data for training using Apache Spark\n", - "- Registering cleanup training data as a Dataset\n", + "- Registering cleaned up training data as a Dataset\n", "- Debugging the model in Tensorflow 2.0 Eager Mode\n", "- Training the model on GPU cluster\n", "- Monitoring training progress with built-in Tensorboard dashboard \n", @@ -152,7 +152,10 @@ "\"Drawing\"\n", "\n", "_Taken from [5](http://jalammar.github.io/illustrated-bert/)_\n", - " " + "\n", + "The end-to-end training process of the stackoverflow question tagging model looks like this:\n", + "\n", + "![](images/model-training-e2e.png)\n" ] }, { @@ -314,9 +317,9 @@ "from azureml.core import Datastore, Dataset\n", "\n", "datastore_name = 'tfworld'\n", - "container_name = 'azureml-blobstore-7c6bdd88-21fa-453a-9c80-16998f02935f'\n", - "account_name = 'tfworld6818510241'\n", - "sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2019-11-08T05:12:15Z&st=2019-10-23T20:12:15Z&spr=https&sig=eDqnc51TkqiIklpQfloT5vcU70pgzDuKb5PAGTvCdx4%3D'\n", + "container_name = 'azure-service-classifier'\n", + "account_name = 'johndatasets'\n", + "sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2021-06-02T03:40:25Z&st=2020-03-09T19:40:25Z&spr=https&sig=bUwK7AJUj2c%2Fr90Qf8O1sojF0w6wRFgL2c9zMVCWNPA%3D'\n", "\n", "datastore = Datastore.register_azure_blob_container(workspace=workspace, \n", " datastore_name=datastore_name, \n", @@ -404,7 +407,7 @@ "\n", "In addition to UI we can register datasets using SDK. In this workshop we will register second type of Datasets using code - File Dataset. File Dataset allows specific folder in our datastore that contains our data files to be registered as a Dataset.\n", "\n", - "There is a folder within our datastore called **azure-service-data** that contains all our training and testing data. We will register this as a dataset." + "There is a folder within our datastore called **data** that contains all our training and testing data. We will register this as a dataset." ] }, { @@ -415,7 +418,7 @@ }, "outputs": [], "source": [ - "azure_dataset = Dataset.File.from_files(path=(datastore, 'azure-service-classifier/data'))\n", + "azure_dataset = Dataset.File.from_files(path=(datastore, 'data'))\n", "\n", "azure_dataset = azure_dataset.register(workspace=workspace,\n", " name='Azure Services Dataset',\n", @@ -474,7 +477,7 @@ "metadata": {}, "outputs": [], "source": [ - "%%pip install transformers==2.0.0" + "%pip install transformers==2.0.0" ] }, { @@ -534,7 +537,7 @@ "\n", "* **ACTION**: Install [Microsoft VS Code](https://code.visualstudio.com/) on your local machine.\n", "\n", - "* **ACTION**: Follow this [configuration guide](https://github.com/danielsc/azureml-debug-training/blob/master/Setting%20up%20VSCode%20Remote%20on%20an%20AzureML%20Notebook%20VM.md) to setup VS Code Remote connection to Notebook VM.\n", + "* **ACTION**: Follow this [configuration guide](https://github.com/danielsc/azureml-debug-training/blob/master/Setting%20up%20VSCode%20Remote%20on%20an%20AzureML%20Compute%20Instance.md) to setup VS Code Remote connection to Notebook VM.\n", "\n", "#### Debug training code using step-by-step debugger\n", "\n", @@ -610,7 +613,7 @@ " },\n", " framework_version='2.0',\n", " use_gpu=True,\n", - " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.1.22'])" + " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.3.0'])" ] }, { @@ -757,7 +760,7 @@ " },\n", " framework_version='2.0',\n", " use_gpu=True,\n", - " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.1.22'])\n", + " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.3.0'])\n", "\n", "run2 = experiment.submit(estimator2)" ] @@ -865,7 +868,7 @@ "run2.download_files(prefix='outputs/model')\n", "\n", "# If you haven't finished training the model then just download pre-made model from datastore\n", - "datastore.download('./',prefix=\"azure-service-classifier/model\")" + "datastore.download('./',prefix=\"model\")" ] }, { @@ -906,7 +909,7 @@ " \n", "labels = ['azure-web-app-service', 'azure-storage', 'azure-devops', 'azure-virtual-machine', 'azure-functions']\n", "# Load model and tokenizer\n", - "loaded_model = TFBertForMultiClassification.from_pretrained('azure-service-classifier/model', num_labels=len(labels))\n", + "loaded_model = TFBertForMultiClassification.from_pretrained('model', num_labels=len(labels))\n", "tokenizer = BertTokenizer.from_pretrained('bert-base-cased')\n", "print(\"Model loaded from disk.\")" ] @@ -1023,7 +1026,7 @@ " node_count=1,\n", " distributed_training=Mpi(process_count_per_node=2),\n", " use_gpu=True,\n", - " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.1.22'])\n", + " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.3.0'])\n", "\n", "run3 = experiment.submit(estimator3)" ] @@ -1144,7 +1147,7 @@ " },\n", " framework_version='2.0',\n", " use_gpu=True,\n", - " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.1.22'])" + " pip_packages=['transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.3.0'])" ] }, { @@ -1262,9 +1265,9 @@ "metadata": { "file_extension": ".py", "kernelspec": { - "display_name": "Python 3.6 - AzureML", + "display_name": "Python 3", "language": "python", - "name": "python3-azureml" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1276,7 +1279,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.2" + "version": "3.7.3" }, "mimetype": "text/x-python", "name": "python", diff --git a/1-Training/databricks/stackoverflow-data-prep.dbc b/1-Training/databricks/stackoverflow-data-prep.dbc deleted file mode 100644 index ed8d86b..0000000 Binary files a/1-Training/databricks/stackoverflow-data-prep.dbc and /dev/null differ diff --git a/1-Training/databricks/stackoverflow-data-prep.html b/1-Training/databricks/stackoverflow-data-prep.html deleted file mode 100644 index c822ec9..0000000 --- a/1-Training/databricks/stackoverflow-data-prep.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - -stackoverflow-data-prep - Databricks - - - - - - - - - - - - - - - - - - - diff --git a/1-Training/images/model-training-e2e.png b/1-Training/images/model-training-e2e.png new file mode 100644 index 0000000..338aef3 Binary files /dev/null and b/1-Training/images/model-training-e2e.png differ diff --git a/1-Training/spark/stackoverflow-data-prep.dbc b/1-Training/spark/stackoverflow-data-prep.dbc deleted file mode 100644 index ed8d86b..0000000 Binary files a/1-Training/spark/stackoverflow-data-prep.dbc and /dev/null differ diff --git a/1-Training/spark/stackoverflow-data-prep.html b/1-Training/spark/stackoverflow-data-prep.html deleted file mode 100644 index c822ec9..0000000 --- a/1-Training/spark/stackoverflow-data-prep.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - -stackoverflow-data-prep - Databricks - - - - - - - - - - - - - - - - - - - diff --git a/1-Training/train_eager.py b/1-Training/train_eager.py index 7e144df..963b5a8 100644 --- a/1-Training/train_eager.py +++ b/1-Training/train_eager.py @@ -135,8 +135,8 @@ def main(_): optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.learning_rate, epsilon=1e-08, clipnorm=1.0) loss = tf.keras.losses.SparseCategoricalCrossentropy() metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') - #model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + # Train and evaluate model for item, label in train_dataset: with tf.GradientTape() as tape: prediction, = model(item, training=True) diff --git a/2-Inferencing/AzureServiceClassifier_Inferencing.ipynb b/2-Inferencing/AzureServiceClassifier_Inferencing.ipynb index 2ab361d..56d6ca5 100644 --- a/2-Inferencing/AzureServiceClassifier_Inferencing.ipynb +++ b/2-Inferencing/AzureServiceClassifier_Inferencing.ipynb @@ -103,17 +103,9 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\r\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!docker ps" ] @@ -152,17 +144,9 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SDK version: 1.0.69\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Check core SDK version number\n", "import azureml.core\n", @@ -181,20 +165,9 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Workspace name: south-central\n", - "Azure region: southcentralus\n", - "Subscription id: 3210cb2e-d3ad-41cf-a6f1-c5a9c3dc522f\n", - "Resource group: aml-rg-107124\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from azureml.core import Workspace\n", "\n", @@ -227,16 +200,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from azureml.core.datastore import Datastore\n", "\n", "datastore_name = 'tfworld'\n", - "container_name = 'azureml-blobstore-7c6bdd88-21fa-453a-9c80-16998f02935f'\n", - "account_name = 'tfworld6818510241'\n", - "sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2019-11-08T05:12:15Z&st=2019-10-23T20:12:15Z&spr=https&sig=eDqnc51TkqiIklpQfloT5vcU70pgzDuKb5PAGTvCdx4%3D'\n", + "container_name = 'azure-service-classifier'\n", + "account_name = 'johndatasets'\n", + "sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2021-06-02T03:40:25Z&st=2020-03-09T19:40:25Z&spr=https&sig=bUwK7AJUj2c%2Fr90Qf8O1sojF0w6wRFgL2c9zMVCWNPA%3D'\n", "\n", "datastore = Datastore.register_azure_blob_container(workspace=ws, \n", " datastore_name=datastore_name, \n", @@ -254,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -271,36 +244,13 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading azure-service-classifier/model/bert_tf2.onnx\n", - "Downloading azure-service-classifier/model/config.json\n", - "Downloading azure-service-classifier/model/tf_model.h5\n", - "Downloaded azure-service-classifier/model/config.json, 1 files out of an estimated total of 3\n", - "Downloaded azure-service-classifier/model/bert_tf2.onnx, 2 files out of an estimated total of 3\n", - "Downloaded azure-service-classifier/model/tf_model.h5, 3 files out of an estimated total of 3\n" - ] - }, - { - "data": { - "text/plain": [ - "3" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from azureml.core.model import Model\n", "\n", - "datastore.download('./',prefix=\"azure-service-classifier/model\")" + "datastore.download('./',prefix=\"model\")" ] }, { @@ -313,23 +263,15 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Registering model azure-service-classifier\n" - ] - } - ], - "source": [ - "model = Model.register(model_path = \"./azure-service-classifier/model\",\n", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = Model.register(model_path = \"./model\",\n", " model_name = \"azure-service-classifier\", # this is the name the model is registered as\n", " tags = {'pretrained': \"BERT\"},\n", " workspace = ws)\n", - "model_dir = './azure-service-classifier/model'" + "model_dir = './model'" ] }, { @@ -360,25 +302,9 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using TensorFlow backend.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Keras version: 2.3.1\n", - "Tensorflow version: 2.0.0\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import keras\n", "import tensorflow as tf\n", @@ -397,37 +323,9 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: transformers in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (2.0.0)\n", - "Requirement already satisfied: sacremoses in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from transformers) (0.0.35)\n", - "Requirement already satisfied: sentencepiece in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from transformers) (0.1.83)\n", - "Requirement already satisfied: requests in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from transformers) (2.22.0)\n", - "Requirement already satisfied: boto3 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from transformers) (1.9.250)\n", - "Requirement already satisfied: numpy in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from transformers) (1.16.2)\n", - "Requirement already satisfied: tqdm in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from transformers) (4.36.1)\n", - "Requirement already satisfied: regex in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from transformers) (2019.8.19)\n", - "Requirement already satisfied: click in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from sacremoses->transformers) (7.0)\n", - "Requirement already satisfied: six in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from sacremoses->transformers) (1.12.0)\n", - "Requirement already satisfied: joblib in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from sacremoses->transformers) (0.13.2)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests->transformers) (2019.9.11)\n", - "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests->transformers) (1.24.2)\n", - "Requirement already satisfied: idna<2.9,>=2.5 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests->transformers) (2.8)\n", - "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n", - "Requirement already satisfied: botocore<1.13.0,>=1.12.250 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from boto3->transformers) (1.12.250)\n", - "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from boto3->transformers) (0.2.1)\n", - "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from boto3->transformers) (0.9.4)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.250->boto3->transformers) (2.8.0)\n", - "Requirement already satisfied: docutils<0.16,>=0.10 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.250->boto3->transformers) (0.15.2)\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%pip install transformers" ] @@ -442,17 +340,9 @@ }, { "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Model loaded from disk.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from transformers import BertTokenizer, TFBertPreTrainedModel, TFBertMainLayer\n", "from transformers.modeling_tf_utils import get_initializer\n", @@ -490,19 +380,9 @@ }, { "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'prediction': 'azure-virtual-machine', 'probability': '0.98652285'}\n", - "CPU times: user 866 ms, sys: 25.2 ms, total: 891 ms\n", - "Wall time: 277 ms\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%%time\n", "import json \n", @@ -578,29 +458,11 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING - Path already exists. Skipping download for ./azure-service-classifier/model/bert_tf2.onnx\n" - ] - }, - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "datastore.download('.',prefix=\"azure-service-classifier/model/bert_tf2.onnx\")" + "datastore.download('.',prefix=\"model/bert_tf2.onnx\")" ] }, { @@ -612,22 +474,9 @@ }, { "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting onnxruntime\n", - "\u001b[?25l Downloading https://files.pythonhosted.org/packages/a1/a3/7d71f481f3632320210676c19c9eb3ac565cf0e98a299883e0cb9324b3d7/onnxruntime-0.5.0-cp36-cp36m-manylinux2010_x86_64.whl (3.2MB)\n", - "\u001b[K |████████████████████████████████| 3.2MB 3.5MB/s eta 0:00:01\n", - "\u001b[?25hInstalling collected packages: onnxruntime\n", - "Successfully installed onnxruntime-0.5.0\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%pip install onnxruntime" ] @@ -642,17 +491,9 @@ }, { "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ONNX Model loaded from disk.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import numpy as np\n", "import onnxruntime as rt\n", @@ -661,7 +502,7 @@ "labels = ['azure-web-app-service', 'azure-storage', 'azure-devops', 'azure-virtual-machine', 'azure-functions']\n", "tokenizer = BertTokenizer.from_pretrained('bert-base-cased')\n", "\n", - "sess = rt.InferenceSession(\"./azure-service-classifier/model/bert_tf2.onnx\")\n", + "sess = rt.InferenceSession(\"./model/bert_tf2.onnx\")\n", "print(\"ONNX Model loaded from disk.\")" ] }, @@ -674,25 +515,9 @@ }, { "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Input name : token_type_ids:0\n", - "Input shape : ['unk__847', 128]\n", - "Input type : tensor(int32)\n", - "Input name : input_ids:0\n", - "Input shape : ['unk__848', 128]\n", - "Input type : tensor(int32)\n", - "Input name : attention_mask:0\n", - "Input shape : ['unk__849', 128]\n", - "Input type : tensor(int32)\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "for i in range(len(sess.get_inputs())):\n", " input_name = sess.get_inputs()[i].name\n", @@ -705,19 +530,9 @@ }, { "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Output name : tf_bert_for_multi_classification/Identity:0\n", - "Output shape : ['unk__850', 5]\n", - "Output type : tensor(float)\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "for i in range(len(sess.get_outputs())):\n", " output_name = sess.get_outputs()[i].name\n", @@ -737,19 +552,9 @@ }, { "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'prediction': 'azure-virtual-machine', 'probability': '0.98652273'}\n", - "CPU times: user 757 ms, sys: 0 ns, total: 757 ms\n", - "Wall time: 195 ms\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%%time\n", "import json \n", @@ -839,7 +644,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -862,7 +667,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -885,7 +690,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -921,7 +726,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -970,505 +775,9 @@ }, { "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading model azure-service-classifier:4 to /tmp/azureml_e9kvls38/azure-service-classifier/4\n", - "Generating Docker build context.\n", - "2019/10/25 21:14:01 Downloading source code...\n", - "2019/10/25 21:14:02 Finished downloading source code\n", - "2019/10/25 21:14:03 Creating Docker network: acb_default_network, driver: 'bridge'\n", - "2019/10/25 21:14:03 Successfully set up Docker network: acb_default_network\n", - "2019/10/25 21:14:03 Setting up Docker configuration...\n", - "2019/10/25 21:14:05 Successfully set up Docker configuration\n", - "2019/10/25 21:14:05 Logging in to registry: southcentral6a2da4dd.azurecr.io\n", - "2019/10/25 21:14:06 Successfully logged into southcentral6a2da4dd.azurecr.io\n", - "2019/10/25 21:14:06 Executing step ID: acb_step_0. Timeout(sec): 5400, Working directory: '', Network: 'acb_default_network'\n", - "2019/10/25 21:14:06 Scanning for dependencies...\n", - "2019/10/25 21:14:07 Successfully scanned dependencies\n", - "2019/10/25 21:14:07 Launching container with name: acb_step_0\n", - "Sending build context to Docker daemon 59.39kB\n", - "Step 1/15 : FROM mcr.microsoft.com/azureml/base:intelmpi2018.3-ubuntu16.04@sha256:a1b514f3ba884b9a7695cbba5638933ddaf222e8ce3e8c81e8cdf861679abb05\n", - "sha256:a1b514f3ba884b9a7695cbba5638933ddaf222e8ce3e8c81e8cdf861679abb05: Pulling from azureml/base\n", - "a1298f4ce990: Pulling fs layer\n", - "04a3282d9c4b: Pulling fs layer\n", - "9b0d3db6dc03: Pulling fs layer\n", - "8269c605f3f1: Pulling fs layer\n", - "6504d449e70c: Pulling fs layer\n", - "4e38f320d0d4: Pulling fs layer\n", - "b0a763e8ee03: Pulling fs layer\n", - "11917a028ca4: Pulling fs layer\n", - "a6c378d11cbf: Pulling fs layer\n", - "6cc007ad9140: Pulling fs layer\n", - "6c1698a608f3: Pulling fs layer\n", - "8269c605f3f1: Waiting\n", - "6504d449e70c: Waiting\n", - "4e38f320d0d4: Waiting\n", - "b0a763e8ee03: Waiting\n", - "a6c378d11cbf: Waiting\n", - "6cc007ad9140: Waiting\n", - "6c1698a608f3: Waiting\n", - "11917a028ca4: Waiting\n", - "9b0d3db6dc03: Download complete\n", - "04a3282d9c4b: Verifying Checksum\n", - "04a3282d9c4b: Download complete\n", - "8269c605f3f1: Download complete\n", - "a1298f4ce990: Verifying Checksum\n", - "a1298f4ce990: Download complete\n", - "4e38f320d0d4: Verifying Checksum\n", - "4e38f320d0d4: Download complete\n", - "6504d449e70c: Verifying Checksum\n", - "6504d449e70c: Download complete\n", - "b0a763e8ee03: Verifying Checksum\n", - "b0a763e8ee03: Download complete\n", - "11917a028ca4: Verifying Checksum\n", - "11917a028ca4: Download complete\n", - "6cc007ad9140: Verifying Checksum\n", - "6cc007ad9140: Download complete\n", - "6c1698a608f3: Verifying Checksum\n", - "6c1698a608f3: Download complete\n", - "a6c378d11cbf: Verifying Checksum\n", - "a6c378d11cbf: Download complete\n", - "a1298f4ce990: Pull complete\n", - "04a3282d9c4b: Pull complete\n", - "9b0d3db6dc03: Pull complete\n", - "8269c605f3f1: Pull complete\n", - "6504d449e70c: Pull complete\n", - "4e38f320d0d4: Pull complete\n", - "b0a763e8ee03: Pull complete\n", - "11917a028ca4: Pull complete\n", - "a6c378d11cbf: Pull complete\n", - "6cc007ad9140: Pull complete\n", - "6c1698a608f3: Pull complete\n", - "Digest: sha256:a1b514f3ba884b9a7695cbba5638933ddaf222e8ce3e8c81e8cdf861679abb05\n", - "Status: Downloaded newer image for mcr.microsoft.com/azureml/base:intelmpi2018.3-ubuntu16.04@sha256:a1b514f3ba884b9a7695cbba5638933ddaf222e8ce3e8c81e8cdf861679abb05\n", - " ---> 93a72e6bd1ce\n", - "Step 2/15 : USER root\n", - " ---> Running in a62950b011f1\n", - "Removing intermediate container a62950b011f1\n", - " ---> 10b3080924af\n", - "Step 3/15 : RUN mkdir -p $HOME/.cache\n", - " ---> Running in b6796d07d9ec\n", - "Removing intermediate container b6796d07d9ec\n", - " ---> 6e6249a04400\n", - "Step 4/15 : WORKDIR /\n", - " ---> Running in f2ed61f54d49\n", - "Removing intermediate container f2ed61f54d49\n", - " ---> f46021a40f54\n", - "Step 5/15 : COPY azureml-environment-setup/99brokenproxy /etc/apt/apt.conf.d/\n", - " ---> a4f03a32a89a\n", - "Step 6/15 : RUN if dpkg --compare-versions `conda --version | grep -oE '[^ ]+$'` lt 4.4.11; then conda install conda==4.4.11; fi\n", - " ---> Running in 110778210a6c\n", - "Removing intermediate container 110778210a6c\n", - " ---> 12c2eda1897d\n", - "Step 7/15 : COPY azureml-environment-setup/mutated_conda_dependencies.yml azureml-environment-setup/mutated_conda_dependencies.yml\n", - " ---> 54f8ba0a40d5\n", - "Step 8/15 : RUN ldconfig /usr/local/cuda/lib64/stubs && conda env create -p /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe -f azureml-environment-setup/mutated_conda_dependencies.yml && rm -rf \"$HOME/.cache/pip\" && conda clean -aqy && CONDA_ROOT_DIR=$(conda info --root) && rm -rf \"$CONDA_ROOT_DIR/pkgs\" && find \"$CONDA_ROOT_DIR\" -type d -name __pycache__ -exec rm -rf {} + && ldconfig\n", - " ---> Running in 8dd00c605ada\n", - "Solving environment: ...working... done\n", - "\u001b[91m\n", - "\n", - "==> WARNING: A newer version of conda exists. <==\n", - " current version: 4.5.11\n", - " latest version: 4.7.12\n", - "\n", - "Please update conda by running\n", - "\n", - " $ conda update -n base -c defaults conda\n", - "\n", - "\n", - "python-dateutil-2.8. | 219 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "openssl-1.0.2r | 3.1 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "tk-8.5.19 | 1.9 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "pip-19.3.1 | 1.9 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "libgcc-ng-9.1.0 | 8.1 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "numpy-1.17.3 | 5.2 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "wheel-0.33.6 | 35 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "pandas-0.25.2 | 11.4 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "sqlite-3.13.0 | 4.9 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "libcblas-3.8.0 | 10 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "six-1.12.0 | 22 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "libstdcxx-ng-9.1.0 | 4.0 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "setuptools-41.4.0 | 624 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "libopenblas-0.3.7 | 7.6 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "libgfortran-ng-7.3.0 | 1.7 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "readline-6.2 | 713 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "python-3.6.2 | 19.0 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "xz-5.2.4 | 366 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "certifi-2019.9.11 | 147 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "zlib-1.2.11 | 105 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "liblapack-3.8.0 | 10 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "libblas-3.8.0 | 10 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "_libgcc_mutex-0.1 | 3 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "ncurses-5.9 | 1.1 MB | ########## | 100% \u001b[0m\u001b[91m\n", - "pytz-2019.3 | 237 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "ca-certificates-2019 | 144 KB | ########## | 100% \u001b[0m\u001b[91m\n", - "Downloading and Extracting Packages\n", - "Preparing transaction: ...working... done\n", - "Verifying transaction: ...working... done\n", - "Executing transaction: ...working... done\n", - "Requirement already satisfied: numpy in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from -r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 1)) (1.17.3)\n", - "Requirement already satisfied: pandas in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from -r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 2)) (0.25.2)\n", - "Collecting inference-schema[numpy-support]\n", - " Downloading https://files.pythonhosted.org/packages/e5/db/54cc017c207e36555db23ac7944e703e8c7c05d6b2e0503041d503af766d/inference_schema-1.0.0-py3-none-any.whl\n", - "Collecting azureml-defaults==1.0.69.*\n", - " Downloading https://files.pythonhosted.org/packages/1c/cb/5b70897d8c98daebcbba9df6a5fd4a5cf80ef52d97fa7ba7c6be5e6a2d69/azureml_defaults-1.0.69-py2.py3-none-any.whl\n", - "Collecting tensorflow==2.0.0\n", - " Downloading https://files.pythonhosted.org/packages/46/0f/7bd55361168bb32796b360ad15a25de6966c9c1beb58a8e30c01c8279862/tensorflow-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl (86.3MB)\n", - "Collecting transformers==2.0.0\n", - " Downloading https://files.pythonhosted.org/packages/66/99/ca0e4c35ccde7d290de3c9c236d5629d1879b04927e5ace9bd6d9183e236/transformers-2.0.0-py3-none-any.whl (290kB)\n", - "Requirement already satisfied: python-dateutil>=2.6.1 in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from pandas->-r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 2)) (2.8.0)\n", - "Requirement already satisfied: pytz>=2017.2 in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from pandas->-r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 2)) (2019.3)\n", - "Collecting wrapt==1.11.1\n", - " Downloading https://files.pythonhosted.org/packages/67/b2/0f71ca90b0ade7fad27e3d20327c996c6252a2ffe88f50a95bba7434eda9/wrapt-1.11.1.tar.gz\n", - "Collecting configparser==3.7.4\n", - " Downloading https://files.pythonhosted.org/packages/ba/05/6c96328e92e625fc31445d24d75a2c92ef9ba34fc5b037fe69693c362a0d/configparser-3.7.4-py2.py3-none-any.whl\n", - "Collecting applicationinsights>=0.11.7\n", - " Downloading https://files.pythonhosted.org/packages/a1/53/234c53004f71f0717d8acd37876e0b65c121181167057b9ce1b1795f96a0/applicationinsights-0.11.9-py2.py3-none-any.whl (58kB)\n", - "Collecting azureml-model-management-sdk==1.0.1b6.post1\n", - " Downloading https://files.pythonhosted.org/packages/4e/53/9004a1e7d6d4e796abc4bcc8286bfc2a32739c5fbac3859ca7429a228897/azureml_model_management_sdk-1.0.1b6.post1-py2.py3-none-any.whl (130kB)\n", - "Collecting flask==1.0.3\n", - " Downloading https://files.pythonhosted.org/packages/9a/74/670ae9737d14114753b8c8fdf2e8bd212a05d3b361ab15b44937dfd40985/Flask-1.0.3-py2.py3-none-any.whl (92kB)\n", - "Collecting gunicorn==19.9.0\n", - " Downloading https://files.pythonhosted.org/packages/8c/da/b8dd8deb741bff556db53902d4706774c8e1e67265f69528c14c003644e6/gunicorn-19.9.0-py2.py3-none-any.whl (112kB)\n", - "Collecting azureml-core==1.0.69.*\n", - " Downloading https://files.pythonhosted.org/packages/9a/4f/b5c71c45f9aa82aa2636dd5ec7e19c6c11138c8ef127faa5cbbbc1efa0b7/azureml_core-1.0.69-py2.py3-none-any.whl (1.1MB)\n", - "Collecting json-logging-py==0.2\n", - " Downloading https://files.pythonhosted.org/packages/e9/e1/46c70eebf216b830867c4896ee678cb7f1b28bb68a2810c7e9a811cecfbc/json-logging-py-0.2.tar.gz\n", - "Collecting gast==0.2.2\n", - " Downloading https://files.pythonhosted.org/packages/4e/35/11749bf99b2d4e3cceb4d55ca22590b0d7c2c62b9de38ac4a4a7f4687421/gast-0.2.2.tar.gz\n", - "Collecting termcolor>=1.1.0\n", - " Downloading https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz\n", - "Collecting keras-applications>=1.0.8\n", - " Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)\n", - "Collecting astor>=0.6.0\n", - " Downloading https://files.pythonhosted.org/packages/d1/4f/950dfae467b384fc96bc6469de25d832534f6b4441033c39f914efd13418/astor-0.8.0-py2.py3-none-any.whl\n", - "Collecting tensorflow-estimator<2.1.0,>=2.0.0\n", - " Downloading https://files.pythonhosted.org/packages/fc/08/8b927337b7019c374719145d1dceba21a8bb909b93b1ad6f8fb7d22c1ca1/tensorflow_estimator-2.0.1-py2.py3-none-any.whl (449kB)\n", - "Collecting opt-einsum>=2.3.2\n", - " Downloading https://files.pythonhosted.org/packages/b8/83/755bd5324777875e9dff19c2e59daec837d0378c09196634524a3d7269ac/opt_einsum-3.1.0.tar.gz (69kB)\n", - "Requirement already satisfied: wheel>=0.26 in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from tensorflow==2.0.0->-r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 5)) (0.33.6)\n", - "Collecting keras-preprocessing>=1.0.5\n", - " Downloading https://files.pythonhosted.org/packages/28/6a/8c1f62c37212d9fc441a7e26736df51ce6f0e38455816445471f10da4f0a/Keras_Preprocessing-1.1.0-py2.py3-none-any.whl (41kB)\n", - "Collecting protobuf>=3.6.1\n", - " Downloading https://files.pythonhosted.org/packages/a8/52/d8d2dbff74b8bf517c42db8d44c3f9ef6555e6f5d6caddfa3f207b9143df/protobuf-3.10.0-cp36-cp36m-manylinux1_x86_64.whl (1.3MB)\n", - "Collecting absl-py>=0.7.0\n", - " Downloading https://files.pythonhosted.org/packages/3b/72/e6e483e2db953c11efa44ee21c5fdb6505c4dffa447b4263ca8af6676b62/absl-py-0.8.1.tar.gz (103kB)\n", - "Collecting grpcio>=1.8.6\n", - " Downloading https://files.pythonhosted.org/packages/30/54/c9810421e41ec0bca2228c6f06b1b1189b196b69533cbcac9f71b44727f8/grpcio-1.24.3-cp36-cp36m-manylinux2010_x86_64.whl (2.2MB)\n", - "Collecting tensorboard<2.1.0,>=2.0.0\n", - " Downloading https://files.pythonhosted.org/packages/9b/a6/e8ffa4e2ddb216449d34cfcb825ebb38206bee5c4553d69e7bc8bc2c5d64/tensorboard-2.0.0-py3-none-any.whl (3.8MB)\n", - "Collecting google-pasta>=0.1.6\n", - " Downloading https://files.pythonhosted.org/packages/d0/33/376510eb8d6246f3c30545f416b2263eee461e40940c2a4413c711bdf62d/google_pasta-0.1.7-py3-none-any.whl (52kB)\n", - "Requirement already satisfied: six>=1.10.0 in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from tensorflow==2.0.0->-r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 5)) (1.12.0)\n", - "Collecting sentencepiece\n", - " Downloading https://files.pythonhosted.org/packages/14/3d/efb655a670b98f62ec32d66954e1109f403db4d937c50d779a75b9763a29/sentencepiece-0.1.83-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n", - "Collecting sacremoses\n", - " Downloading https://files.pythonhosted.org/packages/1f/8e/ed5364a06a9ba720fddd9820155cc57300d28f5f43a6fd7b7e817177e642/sacremoses-0.0.35.tar.gz (859kB)\n", - "Collecting regex\n", - " Downloading https://files.pythonhosted.org/packages/ff/60/d9782c56ceefa76033a00e1f84cd8c586c75e6e7fea2cd45ee8b46a386c5/regex-2019.08.19-cp36-cp36m-manylinux1_x86_64.whl (643kB)\n", - "Collecting boto3\n", - " Downloading https://files.pythonhosted.org/packages/0e/41/27fb3969a76240d4c42a8f64b9d5ae78c676bab38e980e03b1bbaef279bd/boto3-1.10.2-py2.py3-none-any.whl (128kB)\n", - "Collecting requests\n", - " Downloading https://files.pythonhosted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/requests-2.22.0-py2.py3-none-any.whl (57kB)\n", - "Collecting tqdm\n", - " Downloading https://files.pythonhosted.org/packages/e1/c1/bc1dba38b48f4ae3c4428aea669c5e27bd5a7642a74c8348451e0bd8ff86/tqdm-4.36.1-py2.py3-none-any.whl (52kB)\n", - "Collecting adal>=0.4.5\n", - " Downloading https://files.pythonhosted.org/packages/4f/b5/3ea9ae3d1096b9ff31e8f1846c47d49f3129a12464ac0a73b602de458298/adal-1.2.2-py2.py3-none-any.whl (53kB)\n", - "Collecting liac-arff>=2.1.1\n", - " Downloading https://files.pythonhosted.org/packages/e9/35/fbc9217cfa91d98888b43e1a19c03a50d716108c58494c558c65e308f372/liac-arff-2.4.0.tar.gz\n", - "Collecting dill>=0.2.7.1\n", - " Downloading https://files.pythonhosted.org/packages/c7/11/345f3173809cea7f1a193bfbf02403fff250a3360e0e118a1630985e547d/dill-0.3.1.1.tar.gz (151kB)\n", - "Collecting click>=5.1\n", - " Downloading https://files.pythonhosted.org/packages/fa/37/45185cb5abbc30d7257104c434fe0b07e5a195a6847506c074527aa599ec/Click-7.0-py2.py3-none-any.whl (81kB)\n", - "Collecting itsdangerous>=0.24\n", - " Downloading https://files.pythonhosted.org/packages/76/ae/44b03b253d6fade317f32c24d100b3b35c2239807046a4c953c7b89fa49e/itsdangerous-1.1.0-py2.py3-none-any.whl\n", - "Collecting Jinja2>=2.10\n", - " Downloading https://files.pythonhosted.org/packages/65/e0/eb35e762802015cab1ccee04e8a277b03f1d8e53da3ec3106882ec42558b/Jinja2-2.10.3-py2.py3-none-any.whl (125kB)\n", - "Collecting Werkzeug>=0.14\n", - " Downloading https://files.pythonhosted.org/packages/ce/42/3aeda98f96e85fd26180534d36570e4d18108d62ae36f87694b476b83d6f/Werkzeug-0.16.0-py2.py3-none-any.whl (327kB)\n", - "Collecting docker\n", - " Downloading https://files.pythonhosted.org/packages/cc/ca/699d4754a932787ef353a157ada74efd1ceb6d1fc0bfb7989ae1e7b33111/docker-4.1.0-py2.py3-none-any.whl (139kB)\n", - "Collecting pathspec\n", - " Downloading https://files.pythonhosted.org/packages/7a/68/5902e8cd7f7b17c5879982a3a3ee2ad0c3b92b80c79989a2d3e1ca8d29e1/pathspec-0.6.0.tar.gz\n", - "Collecting cryptography!=1.9,!=2.0.*,!=2.1.*,!=2.2.*\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Downloading https://files.pythonhosted.org/packages/ca/9a/7cece52c46546e214e10811b36b2da52ce1ea7fa203203a629b8dfadad53/cryptography-2.8-cp34-abi3-manylinux2010_x86_64.whl (2.3MB)\n", - "Collecting contextlib2\n", - " Downloading https://files.pythonhosted.org/packages/85/60/370352f7ef6aa96c52fb001831622f50f923c1d575427d021b8ab3311236/contextlib2-0.6.0.post1-py2.py3-none-any.whl\n", - "Collecting msrest>=0.5.1\n", - " Downloading https://files.pythonhosted.org/packages/27/b0/c34b3ea9b2ed74b800520fbefb312cdb7f05c20b8bd42e5e7662a5614f98/msrest-0.6.10-py2.py3-none-any.whl (82kB)\n", - "Collecting PyJWT\n", - " Downloading https://files.pythonhosted.org/packages/87/8b/6a9f14b5f781697e51259d81657e6048fd31a113229cf346880bb7545565/PyJWT-1.7.1-py2.py3-none-any.whl\n", - "Collecting SecretStorage\n", - " Downloading https://files.pythonhosted.org/packages/82/59/cb226752e20d83598d7fdcabd7819570b0329a61db07cfbdd21b2ef546e3/SecretStorage-3.1.1-py3-none-any.whl\n", - "Collecting azure-mgmt-keyvault>=0.40.0\n", - " Downloading https://files.pythonhosted.org/packages/b3/d1/9fed0a3a3b43d0b1ad59599b5c836ccc4cf117e26458075385bafe79575b/azure_mgmt_keyvault-2.0.0-py2.py3-none-any.whl (80kB)\n", - "Collecting urllib3>=1.23\n", - " Downloading https://files.pythonhosted.org/packages/e0/da/55f51ea951e1b7c63a579c09dd7db825bb730ec1fe9c0180fc77bfb31448/urllib3-1.25.6-py2.py3-none-any.whl (125kB)\n", - "Collecting azure-mgmt-authorization>=0.40.0\n", - " Downloading https://files.pythonhosted.org/packages/5e/17/4724694ddb3311955ddc367eddcd0928f8ee2c7b12d5a6f0b12bca0b03db/azure_mgmt_authorization-0.60.0-py2.py3-none-any.whl (82kB)\n", - "Collecting jmespath\n", - " Downloading https://files.pythonhosted.org/packages/83/94/7179c3832a6d45b266ddb2aac329e101367fbdb11f425f13771d27f225bb/jmespath-0.9.4-py2.py3-none-any.whl\n", - "Collecting azure-common>=1.1.12\n", - " Downloading https://files.pythonhosted.org/packages/00/55/a703923c12cd3172d5c007beda0c1a34342a17a6a72779f8a7c269af0cd6/azure_common-1.1.23-py2.py3-none-any.whl\n", - "Collecting backports.tempfile\n", - " Downloading https://files.pythonhosted.org/packages/b4/5c/077f910632476281428fe254807952eb47ca78e720d059a46178c541e669/backports.tempfile-1.0-py2.py3-none-any.whl\n", - "Collecting jsonpickle\n", - " Downloading https://files.pythonhosted.org/packages/07/07/c157520a3ebd166c8c24c6ae0ecae7c3968eb4653ff0e5af369bb82f004d/jsonpickle-1.2-py2.py3-none-any.whl\n", - "Collecting pyopenssl\n", - " Downloading https://files.pythonhosted.org/packages/01/c8/ceb170d81bd3941cbeb9940fc6cc2ef2ca4288d0ca8929ea4db5905d904d/pyOpenSSL-19.0.0-py2.py3-none-any.whl (53kB)\n", - "Collecting azure-mgmt-resource>=1.2.1\n", - " Downloading https://files.pythonhosted.org/packages/7c/0d/80815326fa04f2a73ea94b0f57c29669c89df5aa5f5e285952f6445a91c4/azure_mgmt_resource-5.1.0-py2.py3-none-any.whl (681kB)\n", - "Collecting msrestazure>=0.4.33\n", - " Downloading https://files.pythonhosted.org/packages/68/75/5cb56ca8cbc6c5fe476e4878c73f57a331edcf55e5d3fcb4a7377d7d659d/msrestazure-0.6.2-py2.py3-none-any.whl (40kB)\n", - "Collecting azure-mgmt-containerregistry>=2.0.0\n", - " Downloading https://files.pythonhosted.org/packages/97/70/8c2d0509db466678eba16fa2b0a539499f3b351b1f2993126ad843d5be13/azure_mgmt_containerregistry-2.8.0-py2.py3-none-any.whl (718kB)\n", - "Collecting ndg-httpsclient\n", - " Downloading https://files.pythonhosted.org/packages/fb/67/c2f508c00ed2a6911541494504b7cac16fe0b0473912568df65fd1801132/ndg_httpsclient-0.5.1-py3-none-any.whl\n", - "Collecting ruamel.yaml<=0.15.89,>=0.15.35\r\n", - " Downloading https://files.pythonhosted.org/packages/36/e1/cc2fa400fa5ffde3efa834ceb15c464075586de05ca3c553753dcd6f1d3b/ruamel.yaml-0.15.89-cp36-cp36m-manylinux1_x86_64.whl (651kB)\n", - "Collecting azure-graphrbac>=0.40.0\n", - " Downloading https://files.pythonhosted.org/packages/3e/93/02056aca45162f9fc275d1eaad12a2a07ef92375afb48eabddc4134b8315/azure_graphrbac-0.61.1-py2.py3-none-any.whl (141kB)\n", - "Collecting azure-mgmt-storage>=1.5.0\n", - " Downloading https://files.pythonhosted.org/packages/8a/d9/117216e5f671f6c3238c50cba583924252c5ee08091a7d10fa1d3113faa3/azure_mgmt_storage-6.0.0-py2.py3-none-any.whl (514kB)\n", - "Collecting h5py\n", - " Downloading https://files.pythonhosted.org/packages/60/06/cafdd44889200e5438b897388f3075b52a8ef01f28a17366d91de0fa2d05/h5py-2.10.0-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n", - "Requirement already satisfied: setuptools in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from protobuf>=3.6.1->tensorflow==2.0.0->-r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 5)) (41.4.0)\n", - "Collecting markdown>=2.6.8\n", - " Downloading https://files.pythonhosted.org/packages/c0/4e/fd492e91abdc2d2fcb70ef453064d980688762079397f779758e055f6575/Markdown-3.1.1-py2.py3-none-any.whl (87kB)\n", - "Collecting joblib\n", - " Downloading https://files.pythonhosted.org/packages/8f/42/155696f85f344c066e17af287359c9786b436b1bf86029bb3411283274f3/joblib-0.14.0-py2.py3-none-any.whl (294kB)\n", - "Collecting botocore<1.14.0,>=1.13.2\n", - " Downloading https://files.pythonhosted.org/packages/33/57/6cd269b6c243f5409fd60b480bf7b9c98a10fa9bb083af223189d7617db5/botocore-1.13.2-py2.py3-none-any.whl (5.3MB)\n", - "Collecting s3transfer<0.3.0,>=0.2.0\r\n", - " Downloading https://files.pythonhosted.org/packages/16/8a/1fc3dba0c4923c2a76e1ff0d52b305c44606da63f718d14d3231e21c51b0/s3transfer-0.2.1-py2.py3-none-any.whl (70kB)\n", - "Collecting chardet<3.1.0,>=3.0.2\n", - " Downloading https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl (133kB)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/python3.6/site-packages (from requests->transformers==2.0.0->-r /azureml-environment-setup/condaenv._j7n_dvr.requirements.txt (line 6)) (2019.9.11)\n", - "Collecting idna<2.9,>=2.5\n", - " Downloading https://files.pythonhosted.org/packages/14/2c/cd551d81dbe15200be1cf41cd03869a46fe7226e7450af7a6545bfc474c9/idna-2.8-py2.py3-none-any.whl (58kB)\n", - "Collecting MarkupSafe>=0.23\n", - " Downloading https://files.pythonhosted.org/packages/b2/5f/23e0023be6bb885d00ffbefad2942bc51a620328ee910f64abe5a8d18dd1/MarkupSafe-1.1.1-cp36-cp36m-manylinux1_x86_64.whl\n", - "Collecting websocket-client>=0.32.0\n", - " Downloading https://files.pythonhosted.org/packages/29/19/44753eab1fdb50770ac69605527e8859468f3c0fd7dc5a76dd9c4dbd7906/websocket_client-0.56.0-py2.py3-none-any.whl (200kB)\n", - "Collecting cffi!=1.11.3,>=1.8\r\n", - " Downloading https://files.pythonhosted.org/packages/ab/15/d6bd2c322da944ba74ca545dd5e4af6e1e72339cbbc738e6877e349cdfbe/cffi-1.13.1-cp36-cp36m-manylinux1_x86_64.whl (392kB)\n", - "Collecting requests-oauthlib>=0.5.0\n", - " Downloading https://files.pythonhosted.org/packages/c2/e2/9fd03d55ffb70fe51f587f20bcf407a6927eb121de86928b34d162f0b1ac/requests_oauthlib-1.2.0-py2.py3-none-any.whl\n", - "Collecting isodate>=0.6.0\n", - " Downloading https://files.pythonhosted.org/packages/9b/9f/b36f7774ff5ea8e428fdcfc4bb332c39ee5b9362ddd3d40d9516a55221b2/isodate-0.6.0-py2.py3-none-any.whl (45kB)\n", - "Collecting jeepney\n", - " Downloading https://files.pythonhosted.org/packages/0a/4c/ef880713a6c6d628869596703167eab2edf8e0ec2d870d1089dcb0901b81/jeepney-0.4.1-py3-none-any.whl (60kB)\n", - "Collecting backports.weakref\n", - " Downloading https://files.pythonhosted.org/packages/88/ec/f598b633c3d5ffe267aaada57d961c94fdfa183c5c3ebda2b6d151943db6/backports.weakref-1.0.post1-py2.py3-none-any.whl\n", - "Collecting pyasn1>=0.1.1\n", - " Downloading https://files.pythonhosted.org/packages/a1/71/8f0d444e3a74e5640a3d5d967c1c6b015da9c655f35b2d308a55d907a517/pyasn1-0.4.7-py2.py3-none-any.whl (76kB)\n", - "Collecting docutils<0.16,>=0.10\n", - " Downloading https://files.pythonhosted.org/packages/22/cd/a6aa959dca619918ccb55023b4cb151949c64d4d5d55b3f4ffd7eee0c6e8/docutils-0.15.2-py3-none-any.whl (547kB)\n", - "Collecting pycparser\n", - " Downloading https://files.pythonhosted.org/packages/68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a/pycparser-2.19.tar.gz (158kB)\n", - "Collecting oauthlib>=3.0.0\n", - " Downloading https://files.pythonhosted.org/packages/05/57/ce2e7a8fa7c0afb54a0581b14a65b56e62b5759dbc98e80627142b8a3704/oauthlib-3.1.0-py2.py3-none-any.whl (147kB)\n", - "Building wheels for collected packages: wrapt, json-logging-py, gast, termcolor, opt-einsum, absl-py, sacremoses, liac-arff, dill, pathspec, pycparser\n", - " Building wheel for wrapt (setup.py): started\n", - " Building wheel for wrapt (setup.py): finished with status 'done'\r\n", - " Created wheel for wrapt: filename=wrapt-1.11.1-cp36-cp36m-linux_x86_64.whl size=66699 sha256=5eda38184a5bf03b4e0631279c73f485fc396528cd869ae14050d1d35ba26881\n", - " Stored in directory: /root/.cache/pip/wheels/89/67/41/63cbf0f6ac0a6156588b9587be4db5565f8c6d8ccef98202fc\n", - " Building wheel for json-logging-py (setup.py): started\n", - " Building wheel for json-logging-py (setup.py): finished with status 'done'\n", - " Created wheel for json-logging-py: filename=json_logging_py-0.2-cp36-none-any.whl size=3924 sha256=f302b83624c98861b035505be6a7754bfd6a8885770310af0ad5aedf8bf4f8b6\n", - " Stored in directory: /root/.cache/pip/wheels/0d/2e/1c/c638b7589610d8b9358a6e5eb008edacb8b3e9b6d1edc9479f\n", - " Building wheel for gast (setup.py): started\n", - " Building wheel for gast (setup.py): finished with status 'done'\n", - " Created wheel for gast: filename=gast-0.2.2-cp36-none-any.whl size=7540 sha256=b9e02920b6ee1cc226eea4a6ce4015e0f935a075e93d4518d16196c732c300a2\n", - " Stored in directory: /root/.cache/pip/wheels/5c/2e/7e/a1d4d4fcebe6c381f378ce7743a3ced3699feb89bcfbdadadd\n", - " Building wheel for termcolor (setup.py): started\n", - " Building wheel for termcolor (setup.py): finished with status 'done'\n", - " Created wheel for termcolor: filename=termcolor-1.1.0-cp36-none-any.whl size=4832 sha256=3fa36ba8be185652506439f9b9a379877b013780686902a429b7d4bd04ed6b6a\n", - " Stored in directory: /root/.cache/pip/wheels/7c/06/54/bc84598ba1daf8f970247f550b175aaaee85f68b4b0c5ab2c6\n", - " Building wheel for opt-einsum (setup.py): started\n", - " Building wheel for opt-einsum (setup.py): finished with status 'done'\n", - " Created wheel for opt-einsum: filename=opt_einsum-3.1.0-cp36-none-any.whl size=61682 sha256=6b94d2b0660acbf0f18644e50274f2313fc8f17ca8b7f2dac5e6744754e331d1\n", - " Stored in directory: /root/.cache/pip/wheels/2c/b1/94/43d03e130b929aae7ba3f8d15cbd7bc0d1cb5bb38a5c721833\n", - " Building wheel for absl-py (setup.py): started\n", - " Building wheel for absl-py (setup.py): finished with status 'done'\n", - " Created wheel for absl-py: filename=absl_py-0.8.1-cp36-none-any.whl size=121167 sha256=5ca42adcef5397324f7c243b06f13be0921e19e2638598358500dcc338f24a36\n", - " Stored in directory: /root/.cache/pip/wheels/a7/15/a0/0a0561549ad11cdc1bc8fa1191a353efd30facf6bfb507aefc\n", - " Building wheel for sacremoses (setup.py): started\n", - " Building wheel for sacremoses (setup.py): finished with status 'done'\r\n", - " Created wheel for sacremoses: filename=sacremoses-0.0.35-cp36-none-any.whl size=883999 sha256=c1ba760689b52ad3d18509b289f1f7102fea36c07125f96c99f54dc4bf40326c\n", - " Stored in directory: /root/.cache/pip/wheels/63/2a/db/63e2909042c634ef551d0d9ac825b2b0b32dede4a6d87ddc94\n", - " Building wheel for liac-arff (setup.py): started\n", - " Building wheel for liac-arff (setup.py): finished with status 'done'\n", - " Created wheel for liac-arff: filename=liac_arff-2.4.0-cp36-none-any.whl size=13333 sha256=5dc8278eeb5c1cc6244099ffe5e4727ad4fb59daecb2990c4ec51a0bb989c130\n", - " Stored in directory: /root/.cache/pip/wheels/d1/6a/e7/529dc54d76ecede4346164a09ae3168df358945612710f5203\n", - " Building wheel for dill (setup.py): started\n", - " Building wheel for dill (setup.py): finished with status 'done'\n", - " Created wheel for dill: filename=dill-0.3.1.1-cp36-none-any.whl size=78532 sha256=07b39a543d29f4bdddb27a027c773c8a57a6316fffbb7ec84d6f0d1c91842844\n", - " Stored in directory: /root/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7\n", - " Building wheel for pathspec (setup.py): started\n", - " Building wheel for pathspec (setup.py): finished with status 'done'\n", - " Created wheel for pathspec: filename=pathspec-0.6.0-cp36-none-any.whl size=26671 sha256=5100a6bae5c7f7847c14dfbe9f1ae1f8f35e81552896383772b7f9152cd1a8eb\n", - " Stored in directory: /root/.cache/pip/wheels/62/b8/e1/e2719465b5947c40cd85d613d6cb33449b86a1ca5a6c574269\n", - " Building wheel for pycparser (setup.py): started\n", - " Building wheel for pycparser (setup.py): finished with status 'done'\n", - " Created wheel for pycparser: filename=pycparser-2.19-py2.py3-none-any.whl size=111029 sha256=5cd8c026b03055da267733484bc0da9783594977e4033bc9286d9ae08fbf2a11\n", - " Stored in directory: /root/.cache/pip/wheels/f2/9a/90/de94f8556265ddc9d9c8b271b0f63e57b26fb1d67a45564511\n", - "Successfully built wrapt json-logging-py gast termcolor opt-einsum absl-py sacremoses liac-arff dill pathspec pycparser\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Installing collected packages: wrapt, inference-schema, configparser, applicationinsights, chardet, urllib3, idna, requests, pycparser, cffi, cryptography, PyJWT, adal, liac-arff, dill, azureml-model-management-sdk, click, itsdangerous, MarkupSafe, Jinja2, Werkzeug, flask, gunicorn, websocket-client, docker, pathspec, contextlib2, oauthlib, requests-oauthlib, isodate, msrest, jeepney, SecretStorage, azure-common, msrestazure, azure-mgmt-keyvault, azure-mgmt-authorization, jmespath, backports.weakref, backports.tempfile, jsonpickle, pyopenssl, azure-mgmt-resource, azure-mgmt-containerregistry, pyasn1, ndg-httpsclient, ruamel.yaml, azure-graphrbac, azure-mgmt-storage, azureml-core, json-logging-py, azureml-defaults, gast, termcolor, h5py, keras-applications, astor, tensorflow-estimator, opt-einsum, keras-preprocessing, protobuf, absl-py, grpcio, markdown, tensorboard, google-pasta, tensorflow, sentencepiece, joblib, tqdm, sacremoses, regex, docutils, botocore, s3transfer, boto3, transformers\n", - "Successfully installed Jinja2-2.10.3 MarkupSafe-1.1.1 PyJWT-1.7.1 SecretStorage-3.1.1 Werkzeug-0.16.0 absl-py-0.8.1 adal-1.2.2 applicationinsights-0.11.9 astor-0.8.0 azure-common-1.1.23 azure-graphrbac-0.61.1 azure-mgmt-authorization-0.60.0 azure-mgmt-containerregistry-2.8.0 azure-mgmt-keyvault-2.0.0 azure-mgmt-resource-5.1.0 azure-mgmt-storage-6.0.0 azureml-core-1.0.69 azureml-defaults-1.0.69 azureml-model-management-sdk-1.0.1b6.post1 backports.tempfile-1.0 backports.weakref-1.0.post1 boto3-1.10.2 botocore-1.13.2 cffi-1.13.1 chardet-3.0.4 click-7.0 configparser-3.7.4 contextlib2-0.6.0.post1 cryptography-2.8 dill-0.3.1.1 docker-4.1.0 docutils-0.15.2 flask-1.0.3 gast-0.2.2 google-pasta-0.1.7 grpcio-1.24.3 gunicorn-19.9.0 h5py-2.10.0 idna-2.8 inference-schema-1.0.0 isodate-0.6.0 itsdangerous-1.1.0 jeepney-0.4.1 jmespath-0.9.4 joblib-0.14.0 json-logging-py-0.2 jsonpickle-1.2 keras-applications-1.0.8 keras-preprocessing-1.1.0 liac-arff-2.4.0 markdown-3.1.1 msrest-0.6.10 msrestazure-0.6.2 ndg-httpsclient-0.5.1 oauthlib-3.1.0 opt-einsum-3.1.0 pathspec-0.6.0 protobuf-3.10.0 pyasn1-0.4.7 pycparser-2.19 pyopenssl-19.0.0 regex-2019.8.19 requests-2.22.0 requests-oauthlib-1.2.0 ruamel.yaml-0.15.89 s3transfer-0.2.1 sacremoses-0.0.35 sentencepiece-0.1.83 tensorboard-2.0.0 tensorflow-2.0.0 tensorflow-estimator-2.0.1 termcolor-1.1.0 tqdm-4.36.1 transformers-2.0.0 urllib3-1.25.6 websocket-client-0.56.0 wrapt-1.11.1\n", - "\u001b[91m\n", - "\u001b[0m#\n", - "# To activate this environment, use:\n", - "# > source activate /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe\n", - "#\n", - "# To deactivate an active environment, use:\n", - "# > source deactivate\n", - "#\n", - "\n", - "Removing intermediate container 8dd00c605ada\n", - " ---> 6b82dc96b472\n", - "Step 9/15 : ENV PATH /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/bin:$PATH\n", - " ---> Running in 890b9a637145\n", - "Removing intermediate container 890b9a637145\n", - " ---> c2ff746e57f4\n", - "Step 10/15 : ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe\n", - " ---> Running in dd4ca898cb85\n", - "Removing intermediate container dd4ca898cb85\n", - " ---> f9ad78ea34a4\n", - "Step 11/15 : ENV LD_LIBRARY_PATH /azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib:$LD_LIBRARY_PATH\n", - " ---> Running in 279b2eb0b142\n", - "Removing intermediate container 279b2eb0b142\n", - " ---> 17a717ab31df\n", - "Step 12/15 : COPY azureml-environment-setup/spark_cache.py azureml-environment-setup/log4j.properties /azureml-environment-setup/\n", - " ---> 3d12ea59059c\n", - "Step 13/15 : RUN if [ $SPARK_HOME ]; then /bin/bash -c '$SPARK_HOME/bin/spark-submit /azureml-environment-setup/spark_cache.py'; fi\n", - " ---> Running in 69541a32742d\n", - "Removing intermediate container 69541a32742d\n", - " ---> ab80d0052d97\n", - "Step 14/15 : ENV AZUREML_ENVIRONMENT_IMAGE True\n", - " ---> Running in 8361080e2621\n", - "Removing intermediate container 8361080e2621\n", - " ---> ebe402b017b8\n", - "Step 15/15 : CMD [\"bash\"]\n", - " ---> Running in 7bade2845ffc\n", - "Removing intermediate container 7bade2845ffc\n", - " ---> 92c56a134548\n", - "Successfully built 92c56a134548\n", - "Successfully tagged southcentral6a2da4dd.azurecr.io/azureml/azureml_c1f241d01b5197c539f2c3744f68f5e3:latest\n", - "2019/10/25 21:17:39 Successfully executed container: acb_step_0\n", - "2019/10/25 21:17:39 Executing step ID: acb_step_1. Timeout(sec): 5400, Working directory: '', Network: 'acb_default_network'\n", - "2019/10/25 21:17:39 Pushing image: southcentral6a2da4dd.azurecr.io/azureml/azureml_c1f241d01b5197c539f2c3744f68f5e3:latest, attempt 1\n", - "The push refers to repository [southcentral6a2da4dd.azurecr.io/azureml/azureml_c1f241d01b5197c539f2c3744f68f5e3]\n", - "2c26408fe379: Preparing\n", - "eada16cd347c: Preparing\n", - "a5b8f4eed6a4: Preparing\n", - "72feb27092b1: Preparing\n", - "13d944a70c3a: Preparing\n", - "564508f84629: Preparing\n", - "e1171d4d60ca: Preparing\n", - "6ef1a8ae63b7: Preparing\n", - "85389f9ead9e: Preparing\n", - "f2608f66a0e3: Preparing\n", - "0e259b09e5f4: Preparing\n", - "340dc32eb998: Preparing\n", - "df18b66efaa6: Preparing\n", - "ccdb13a20bf2: Preparing\n", - "9513cdf4e497: Preparing\n", - "7f083f9454c0: Preparing\n", - "29f36b5893dc: Preparing\n", - "564508f84629: Waiting\n", - "e1171d4d60ca: Waiting\n", - "6ef1a8ae63b7: Waiting\n", - "85389f9ead9e: Waiting\n", - "f2608f66a0e3: Waiting\n", - "0e259b09e5f4: Waiting\n", - "340dc32eb998: Waiting\n", - "df18b66efaa6: Waiting\n", - "ccdb13a20bf2: Waiting\n", - "9513cdf4e497: Waiting\n", - "7f083f9454c0: Waiting\n", - "29f36b5893dc: Waiting\n", - "a5b8f4eed6a4: Pushed\n", - "13d944a70c3a: Pushed\n", - "2c26408fe379: Pushed\n", - "72feb27092b1: Pushed\n", - "564508f84629: Pushed\n", - "e1171d4d60ca: Pushed\n", - "6ef1a8ae63b7: Pushed\n", - "85389f9ead9e: Pushed\n", - "340dc32eb998: Pushed\n", - "ccdb13a20bf2: Pushed\n", - "9513cdf4e497: Pushed\n", - "7f083f9454c0: Pushed\n", - "0e259b09e5f4: Pushed\n", - "f2608f66a0e3: Pushed\n", - "29f36b5893dc: Pushed\n", - "df18b66efaa6: Pushed\n", - "eada16cd347c: Pushed\n", - "latest: digest: sha256:9833c5a34dda5cf97659790ed7064d32e8af5a1b0b7625b0b2bee00933dd4648 size: 3883\n", - "2019/10/25 21:19:13 Successfully pushed image: southcentral6a2da4dd.azurecr.io/azureml/azureml_c1f241d01b5197c539f2c3744f68f5e3:latest\n", - "2019/10/25 21:19:13 Step ID: acb_step_0 marked as successful (elapsed time in seconds: 212.385464)\n", - "2019/10/25 21:19:13 Populating digests for step ID: acb_step_0...\n", - "2019/10/25 21:19:15 Successfully populated digests for step ID: acb_step_0\n", - "2019/10/25 21:19:15 Step ID: acb_step_1 marked as successful (elapsed time in seconds: 93.687260)\n", - "2019/10/25 21:19:15 The following dependencies were found:\n", - "2019/10/25 21:19:15 \n", - "- image:\n", - " registry: southcentral6a2da4dd.azurecr.io\n", - " repository: azureml/azureml_c1f241d01b5197c539f2c3744f68f5e3\n", - " tag: latest\n", - " digest: sha256:9833c5a34dda5cf97659790ed7064d32e8af5a1b0b7625b0b2bee00933dd4648\n", - " runtime-dependency:\n", - " registry: mcr.microsoft.com\n", - " repository: azureml/base\n", - " tag: intelmpi2018.3-ubuntu16.04\n", - " digest: sha256:a1b514f3ba884b9a7695cbba5638933ddaf222e8ce3e8c81e8cdf861679abb05\n", - " git: {}\n", - "\n", - "Run ID: cd3 was successful after 5m16s\n", - "Package creation Succeeded\n", - "Logging into Docker registry southcentral6a2da4dd.azurecr.io\n", - "Logging into Docker registry southcentral6a2da4dd.azurecr.io\n", - "Building Docker image from Dockerfile...\n", - "Step 1/5 : FROM southcentral6a2da4dd.azurecr.io/azureml/azureml_c1f241d01b5197c539f2c3744f68f5e3\n", - " ---> 92c56a134548\n", - "Step 2/5 : COPY azureml-app /var/azureml-app\n", - " ---> 1556d972c537\n", - "Step 3/5 : COPY model_config_map.json /var/azureml-app/model_config_map.json\n", - " ---> e20dddab2b05\n", - "Step 4/5 : RUN mv '/var/azureml-app/tmpu3ib6cdy.py' /var/azureml-app/main.py\n", - " ---> Running in cfaf1cc075a8\n", - " ---> ca239867b574\n", - "Step 5/5 : CMD [\"runsvdir\",\"/var/runit\"]\n", - " ---> Running in f2efbdcb7eb5\n", - " ---> 1e46724d90fb\n", - "Successfully built 1e46724d90fb\n", - "Successfully tagged mymodel:latest\n", - "Starting Docker container...\n", - "Docker container running.\n", - "Checking container health...\n", - "Local webservice is running at http://localhost:32770\n", - "32770\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "from azureml.core.model import InferenceConfig, Model\n", "from azureml.core.webservice import LocalWebservice\n", @@ -1493,17 +802,9 @@ }, { "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "http://localhost:32770/score\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "print(local_service.scoring_uri)" ] @@ -1526,21 +827,9 @@ }, { "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Making a scoring call...\n", - "Scoring result:\n", - "{'prediction': 'azure-virtual-machine', 'probability': '0.98652285'}\n", - "CPU times: user 15.9 ms, sys: 660 µs, total: 16.5 ms\n", - "Wall time: 9.9 s\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%%time\n", "import json\n", @@ -1561,17 +850,9 @@ }, { "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting score.py\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%%writefile score.py\n", "import os\n", @@ -1676,19 +957,9 @@ }, { "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Container has been successfully cleaned up.\n", - "Starting Docker container...\n", - "Docker container running.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "local_service.reload()" ] @@ -1720,168 +991,9 @@ }, { "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('/bin/bash: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libtinfo.so.5: no '\n", - " 'version information available (required by /bin/bash)\\n'\n", - " '/bin/bash: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libtinfo.so.5: no '\n", - " 'version information available (required by /bin/bash)\\n'\n", - " '/bin/bash: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libtinfo.so.5: no '\n", - " 'version information available (required by /bin/bash)\\n'\n", - " '/bin/bash: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libtinfo.so.5: no '\n", - " 'version information available (required by /bin/bash)\\n'\n", - " '2019-10-25T21:26:50,050750475+00:00 - rsyslog/run \\n'\n", - " '2019-10-25T21:26:50,051500275+00:00 - iot-server/run \\n'\n", - " 'bash: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libtinfo.so.5: no '\n", - " 'version information available (required by bash)\\n'\n", - " '2019-10-25T21:26:50,053720575+00:00 - gunicorn/run \\n'\n", - " '2019-10-25T21:26:50,061494276+00:00 - nginx/run \\n'\n", - " '/usr/sbin/nginx: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libcrypto.so.1.0.0: '\n", - " 'no version information available (required by /usr/sbin/nginx)\\n'\n", - " '/usr/sbin/nginx: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libcrypto.so.1.0.0: '\n", - " 'no version information available (required by /usr/sbin/nginx)\\n'\n", - " '/usr/sbin/nginx: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libssl.so.1.0.0: '\n", - " 'no version information available (required by /usr/sbin/nginx)\\n'\n", - " '/usr/sbin/nginx: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libssl.so.1.0.0: '\n", - " 'no version information available (required by /usr/sbin/nginx)\\n'\n", - " '/usr/sbin/nginx: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libssl.so.1.0.0: '\n", - " 'no version information available (required by /usr/sbin/nginx)\\n'\n", - " 'EdgeHubConnectionString and IOTEDGE_IOTHUBHOSTNAME are not set. Exiting...\\n'\n", - " '/bin/bash: '\n", - " '/azureml-envs/azureml_acb89c589d9589e8a1eaf57a0759fafe/lib/libtinfo.so.5: no '\n", - " 'version information available (required by /bin/bash)\\n'\n", - " '2019-10-25T21:26:50,136271379+00:00 - iot-server/finish 1 0\\n'\n", - " '2019-10-25T21:26:50,137414879+00:00 - Exit code 1 is normal. Not restarting '\n", - " 'iot-server.\\n'\n", - " 'Starting gunicorn 19.9.0\\n'\n", - " 'Listening at: http://127.0.0.1:31311 (12)\\n'\n", - " 'Using worker: sync\\n'\n", - " 'worker timeout is set to 300\\n'\n", - " 'Booting worker with pid: 43\\n'\n", - " 'Initialized PySpark session.\\n'\n", - " 'TensorFlow version 2.0.0 available.\\n'\n", - " 'To use data.metrics please install scikit-learn. See '\n", - " 'https://scikit-learn.org/stable/index.html\\n'\n", - " 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt '\n", - " 'not found in cache or force_download set to True, downloading to '\n", - " '/tmp/tmp7n3s05bn\\n'\n", - " 'copying /tmp/tmp7n3s05bn to cache at '\n", - " '/root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\\n'\n", - " 'creating metadata file for '\n", - " '/root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\\n'\n", - " 'removing temp file /tmp/tmp7n3s05bn\\n'\n", - " 'loading configuration file '\n", - " 'azureml-models/azure-service-classifier/4/model/config.json\\n'\n", - " 'Model config {\\n'\n", - " ' \"attention_probs_dropout_prob\": 0.1,\\n'\n", - " ' \"finetuning_task\": null,\\n'\n", - " ' \"hidden_act\": \"gelu\",\\n'\n", - " ' \"hidden_dropout_prob\": 0.1,\\n'\n", - " ' \"hidden_size\": 768,\\n'\n", - " ' \"initializer_range\": 0.02,\\n'\n", - " ' \"intermediate_size\": 3072,\\n'\n", - " ' \"layer_norm_eps\": 1e-12,\\n'\n", - " ' \"max_position_embeddings\": 512,\\n'\n", - " ' \"num_attention_heads\": 12,\\n'\n", - " ' \"num_hidden_layers\": 12,\\n'\n", - " ' \"num_labels\": 5,\\n'\n", - " ' \"output_attentions\": false,\\n'\n", - " ' \"output_hidden_states\": false,\\n'\n", - " ' \"pruned_heads\": {},\\n'\n", - " ' \"torchscript\": false,\\n'\n", - " ' \"type_vocab_size\": 2,\\n'\n", - " ' \"use_bfloat16\": false,\\n'\n", - " ' \"vocab_size\": 28996\\n'\n", - " '}\\n'\n", - " '\\n'\n", - " 'loading weights file '\n", - " 'azureml-models/azure-service-classifier/4/model/tf_model.h5\\n'\n", - " '\\r'\n", - " ' 0%| | 0/213450 [00:00 "Build" on the left-hand side, then "New pipeline" to cr ![build connnect step](./images/build-connect.png) -Refer to an **Existing Azure Pipelines YAML file**: - -![configure step](./images/select-iac-pipeline.png) +Refer to an **Existing Azure Pipelines YAML file**, then choose the one corresponding to **iac-create-environment.yml**. -Having done that, run the pipeline: - -![iac run](./images/run-iac-pipeline.png) +Having done that, run the pipeline. Check out created resources in the [Azure Portal](portal.azure.com): @@ -169,9 +165,7 @@ The YAML file includes the following steps: Now that you understand the steps in your pipeline, let's see what it actually does! -In your [Azure DevOps](https://dev.azure.com) project, use the left-hand menu to navigate to "Pipelines"->"Build." Select "New pipeline," and then select "GitHub." If you are already authenticated into GitHub, you should see the repository you forked earlier. Select "Existing Azure Pipelines YAML File." In the pop-up blade, select the correct branch of your GitHub repo and select the path referring to [publish-training-pipeline.yml] (./yml/publish-training-pipeline.yml) in your forked **GitHub** repository: - -![configure ci build pipeline](./images/ci-build-pipeline-configure.png) +In your [Azure DevOps](https://dev.azure.com) project, use the left-hand menu to navigate to "Pipelines"->"Build." Select "New pipeline," and then select "GitHub." If you are already authenticated into GitHub, you should see the repository you forked earlier. Select "Existing Azure Pipelines YAML File." In the pop-up blade, select the correct branch of your GitHub repo and select the path referring to [publish-training-pipeline.yml](./yml/publish-training-pipeline.yml) in your forked **GitHub** repository. You will now be redirected to a review page. Check to make sure you still understand what this pipeline is doing. If everything looks good, click "Run." @@ -193,12 +187,10 @@ Even though we created a service connection earlier in order to create resources ![workspace connection](./images/workspace-connection.png) -### 11. Deploy the Model +### 11. Create the release pipeline. The final step is to deploy your model with a release pipeline. -#### Create the release pipeline - Go to "Pipelines" -> "Releases." In the top right of the second navigation bar from the left, select "New" -> "New release pipeline." Select "Empty job" under "Select a template" on the blade that pops up. ![Selecting empty job](./images/empty-job.png) @@ -207,7 +199,7 @@ Call this stage "Prod," by editing the value of "Stage name" in the blade on the ![Rename prod](./images/prod.png) -#### Add artifacts +### 12. Add artifacts to your pipeline In order for this Release pipeline to work, it needs access to the trained model we produced in the build pipeline. The release pipeline accesses the trained model as part of something called an Artifact. To give this release pipeline access to the relevant artifacts, click on "Add an artifact" in the "Artifacts" box. @@ -217,110 +209,66 @@ Next, select "AzureML Model Artifact" (you may need to click "Show more"). Selec Let's also give the release pipeline access to the build artifact, which contains some of the files that the release pipeline needs in order to run. Click on "Add" in the "Artifacts" box, select "Build," and ensure that the source alias is set to "_ci-build". This naming is necessary for the next step to work properly. -#### Add tasks +### 13. Add QA stage -Great, so your release pipeline has access to your artifacts, but it doesn't actually _do_ anything. Let's give it some work. +Great, so your release pipeline has access to your artifacts, but it doesn't actually _do_ anything. Let's give it some work. First, let's have it deploy to a quality assurance (QA) instance hosted with Azure Container Instances (ACI). Click on the hyperlinked text that says "1 job, 0 task" in the name of the stage. Click on the plus icon on the right hand side of the cell which says "Agent job." On the menu which appears, search for "Azure ML Model Deploy," and click "Add." -![create release example](./images/deploy-task.png) - Click on the red text which says "Some settings need attention" and fill in the values shown in the table below: | Parameter | Value | | --------------------------------- | ---------------------------------------------------------------------------------------------------- | | Display Name | Azure ML Model Deploy | -| Azure ML Workspace | | +| Azure ML Workspace | Fill in the name of your Azure ML service connection | | Inference config Path | `$(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/scripts/scoring/inference_config.yml` | -| Model Deployment Target | Azure Kubernetes Service | -| Select AKS Cluster for Deployment | myaks (**This value is specified in the .env file, and you should have an existing cluster with this name**) | -| Deployment Name | bert-stack-overflow-aks | -| Deployment Configuration file | `$(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/scripts/scoring/deployment_config_aks.yml` | +| Model Deployment Target | Azure Container Instance | +| Deployment Name | bert-stack-overflow-aci | +| Deployment Configuration file | `$(System.DefaultWorkingDirectory)/_ci-build/mlops-pipelines/scripts/scoring/deployment_config_aci.yml` | | Overwrite existing deployment | X | - Then click "Save." -#### Enable continuous integration - -Go to "Pipelines" -> "Releases" and then click on your new pipeline. In the top right of each artifact you specified, you should see a lightning bolt. Click on this lightning bolt and then toggle the trigger for "Continuous deployment." This will ensure that the deployment is released every time one of these artifacts changes. Make sure to save your changes. - - - - - - - - +To kick off your first deployment, click "Create release." +![create-release-button](./images/create-release-button.png) -### 10. Test your deployed model +### 15. Test your deployed model Open your machine learning workspace in the [Azure portal](portal.azure.com), and click on "Deployments" on the lefthand side. Open up your AKS cluster, and use the Scoring URI and Primary Key for this step. @@ -332,19 +280,22 @@ Let's see if we can submit a query to our deployed model! Open up a Python inter import json import requests - url = '' api_key = '' -payload = {'text': 'I am trying to release a website'} -headers = {'content-type': 'application/json', 'Authorization':('Bearer '+ api_key)} -response = requests.post(url, data=json.dumps(payload), headers=headers) -response_body = json.loads(response.content) # convert to dict for next step -print("Given your question of \"{}\", we predict the tag is {} with probability {}" - .format(payload.get("text"), response_body.get("prediction"), response_body.get("probability"))) + +def predict_tags(question_body): + payload = {'text': question_body} + headers = {'content-type': 'application/json', 'Authorization':('Bearer '+ api_key)} + response = requests.post(url, data=json.dumps(payload), headers=headers) + response_body = json.loads(response.content) # convert to dict for next step + print("Given your question of \"{}\", we predict the tag is {} with probability {}" + .format(payload.get("text"), response_body.get("prediction"), response_body.get("probability"))) + +predict_tags('How can I specify Service Principal in devops pipeline when deploying virtual machine?') ``` Congratulations! You have two pipelines set up end to end: - Build pipeline: triggered on code change to master branch on GitHub, performs linting, unit testing and publishing a training pipeline. Also train, evaluate and register a model - - Release Deployment pipeline: deploys a model to a Prod (AKS) environment + - Release Deployment pipeline: triggered when build artifacts change or registered model changes, deploys a model to a Prod (AKS) environment diff --git a/3-ML-Ops/create-resources/requirements.txt b/3-ML-Ops/create-resources/requirements.txt index 63168aa..6a21fa8 100755 --- a/3-ML-Ops/create-resources/requirements.txt +++ b/3-ML-Ops/create-resources/requirements.txt @@ -6,4 +6,4 @@ python-dotenv>=0.10.3 flake8 flake8_formatter_junit_xml azure-cli==2.0.71 -azureml-dataprep[pandas] \ No newline at end of file +azureml-dataprep[pandas]==1.3.0 \ No newline at end of file diff --git a/3-ML-Ops/images/create-release-button.png b/3-ML-Ops/images/create-release-button.png new file mode 100755 index 0000000..9ab7b7c Binary files /dev/null and b/3-ML-Ops/images/create-release-button.png differ diff --git a/3-ML-Ops/images/deploy-task.png b/3-ML-Ops/images/deploy-task.png index c513d9b..ebec321 100755 Binary files a/3-ML-Ops/images/deploy-task.png and b/3-ML-Ops/images/deploy-task.png differ diff --git a/3-ML-Ops/images/final-release.png b/3-ML-Ops/images/final-release.png new file mode 100755 index 0000000..6674375 Binary files /dev/null and b/3-ML-Ops/images/final-release.png differ diff --git a/3-ML-Ops/scripts/evaluate/evaluate_model.py b/3-ML-Ops/scripts/evaluate/evaluate_model.py index 859d8dd..c42e1ab 100755 --- a/3-ML-Ops/scripts/evaluate/evaluate_model.py +++ b/3-ML-Ops/scripts/evaluate/evaluate_model.py @@ -54,41 +54,39 @@ all_runs = exp.get_runs(include_children=True) -# get the last completed run -while True: - new_model_run = next(all_runs) - if (new_model_run.get_status() == 'Finished'): +new_model_run = None + +# get the last completed run with metrics +new_model_run = None +for run in all_runs: + acc = run.get_metrics().get("val_accuracy") + print(f'run is {run}, acc is {acc}') + if run.get_status() == 'Finished' and acc is not None: + new_model_run = run + print('found a valid new model with acc {}'.format(acc)) break - new_model_run_id = new_model_run.id - -# print(f'New Run found with Run ID of: {new_model_run_id}') - -model_list = Model.list(ws) - -if len(model_list) != 0: # there are some models - # # Get most recently registered model, we assume that - # is the model in production. - # Download this model and compare it with the recently - # trained model by running test with same data set. - production_model = next( - filter( - lambda x: x.created_time == max( - model.created_time for model in model_list), - model_list, - ) - ) - production_model_run_id = production_model.id - run_list = exp.get_runs() - # Get the run history for both production model and - # newly trained model and compare mse - production_model_run = production_model.run - # new_model_run = Run(exp, run_id=new_model_run_id) +if new_model_run is None: + raise Exception('new model must log a val_accuracy metric, please check' + ' your train.py file') +model_generator = Model.list(ws) + +# Check that there are models +if len(model_generator) > 0: + + # Get the model with best val accuracy, assume this is best + cur_max = None + production_model = None - production_model_acc = production_model_run.get_metrics().get( - "val_accuracy") + for model in model_generator: + cur_acc = model.run.get_metrics().get("val_accuracy")[-1] + if cur_max is None or cur_acc > cur_max: + cur_max = cur_acc + production_model = model + + production_model_acc = cur_max new_model_acc = new_model_run.get_metrics().get( - "val_accuracy") + "val_accuracy")[-1] print( "Current Production model acc: {}, New trained model acc: {}".format( production_model_acc, new_model_acc @@ -96,16 +94,15 @@ ) promote_new_model = False - if new_model_acc > production_model_acc: + if new_model_acc > production_model_acc or production_model_acc is None: promote_new_model = True print("New trained model performs better, will be registered") - promote_new_model = True + else: promote_new_model = True print("This is the first model to be trained, \ thus nothing to evaluate for now") - # Writing the run id to /aml_config/run_id.json if promote_new_model: model_path = './outputs/exports' diff --git a/3-ML-Ops/scripts/scoring/deployment_config_aci.yml b/3-ML-Ops/scripts/scoring/deployment_config_aci.yml new file mode 100644 index 0000000..0990abb --- /dev/null +++ b/3-ML-Ops/scripts/scoring/deployment_config_aci.yml @@ -0,0 +1,4 @@ +containerResourceRequirements: + cpu: 1 + memoryInGB: 4 +computeType: ACI \ No newline at end of file diff --git a/3-ML-Ops/train-and-register-model.py b/3-ML-Ops/train-and-register-model.py index afae53b..4ea18b0 100755 --- a/3-ML-Ops/train-and-register-model.py +++ b/3-ML-Ops/train-and-register-model.py @@ -50,14 +50,6 @@ def main(): if aml_compute is not None: print(aml_compute) - # Get AKS cluster for deployment - aks_compute = get_aks( - aml_workspace, - aks_name - ) - if aks_compute is not None: - print(aks_compute) - run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=['numpy', 'pandas', 'scikit-learn', 'keras'], @@ -69,12 +61,11 @@ def main(): 'tensorflow-gpu>=2.0.0']) ) run_config.environment.docker.enabled = True - + datastore_name = 'tfworld' - container_name = 'azureml-blobstore-7c6bdd88-21fa-453a-9c80-16998f02935f' - account_name = 'tfworld6818510241' - # sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2019-11-08T05:12:15Z&st=2019-10-23T20:12:15Z&spr=https&sig=eDqnc51TkqiIklpQfloT5vcU70pgzDuKb5PAGTvCdx4%3D' # noqa: E501 - account_key = os.environ.get("DS_KEY") + container_name = 'azure-service-classifier' + account_name = 'johndatasets' + sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2021-06-02T03:40:25Z&st=2020-03-09T19:40:25Z&spr=https&sig=bUwK7AJUj2c%2Fr90Qf8O1sojF0w6wRFgL2c9zMVCWNPA%3D' try: existing_datastore = Datastore.get(aml_workspace, datastore_name) @@ -84,15 +75,17 @@ def main(): datastore_name=datastore_name, container_name=container_name, account_name=account_name, - account_key=account_key + sas_token=sas_token ) azure_dataset = Dataset.File.from_files( - path=(existing_datastore, 'azure-service-classifier/data')) + path=(existing_datastore, 'data')) azure_dataset = azure_dataset.register( workspace=aml_workspace, name='Azure Services Dataset', - description='Dataset containing azure related posts on Stackoverflow') + description='Dataset containing azure related posts on Stackoverflow', + create_new_version=True) + azure_dataset.to_path() input_data = azure_dataset.as_named_input('input_data1').as_mount( '/tmp/data') @@ -121,7 +114,7 @@ def main(): use_gpu=True, pip_packages=[ 'transformers==2.0.0', - 'azureml-dataprep[fuse,pandas]==1.1.22']) + 'azureml-dataprep[fuse,pandas]==1.3.0']) train_step = EstimatorStep( name="Train Model", @@ -176,6 +169,14 @@ def main(): workspace=aml_workspace, experiment_name=experiment_name) + # Get AKS cluster for deployment + aks_compute = get_aks( + aml_workspace, + aks_name + ) + if aks_compute is not None: + print(aks_compute) + if __name__ == '__main__': main() diff --git a/3-ML-Ops/util/attach_aks.py b/3-ML-Ops/util/attach_aks.py index 59ed627..e97c0d9 100755 --- a/3-ML-Ops/util/attach_aks.py +++ b/3-ML-Ops/util/attach_aks.py @@ -9,11 +9,10 @@ def get_aks( ): # Verify that cluster does not exist already try: - if compute_name in workspace.compute_targets: - aks_target = workspace.compute_targets[compute_name] - if aks_target and type(aks_target) is AksCompute: - print('Found existing compute target ' + compute_name - + ' so using it.') + aks_target = workspace.compute_targets.get(compute_name) + if aks_target is not None and type(aks_target) is AksCompute: + print('Found existing compute target ' + compute_name + + ' so using it.') # noqa: E127 else: prov_config = AksCompute.provisioning_configuration( cluster_purpose=AksCompute.ClusterPurpose.DEV_TEST) @@ -34,4 +33,4 @@ def get_aks( except ComputeTargetException as e: print(e) print('An error occurred trying to provision compute.') - exit() + raise diff --git a/4-Interpretibility/EmployeeAttritionClassifier_Interpretability_Local.ipynb b/4-Interpretibility/EmployeeAttritionClassifier_Interpretability_Local.ipynb new file mode 100644 index 0000000..833755c --- /dev/null +++ b/4-Interpretibility/EmployeeAttritionClassifier_Interpretability_Local.ipynb @@ -0,0 +1,478 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Interpretability With Tensorflow On Azure Machine Learning Service (Local)\n", + "\n", + "## Overview of Tutorial\n", + "This notebook is Part 4 (Explaining Your Model Using Interpretability) of a four part workshop that demonstrates an end-to-end workflow for using Tensorflow on Azure Machine Learning Service. The different components of the workshop are as follows:\n", + "\n", + "- Part 1: [Preparing Data and Model Training](https://github.com/microsoft/bert-stack-overflow/blob/master/1-Training/AzureServiceClassifier_Training.ipynb)\n", + "- Part 2: [Inferencing and Deploying a Model](https://github.com/microsoft/bert-stack-overflow/blob/master/2-Inferencing/AzureServiceClassifier_Inferencing.ipynb)\n", + "- Part 3: [Setting Up a Pipeline Using MLOps](https://github.com/microsoft/bert-stack-overflow/tree/master/3-ML-Ops)\n", + "- Part 4: [Explaining Your Model Interpretability](https://github.com/microsoft/bert-stack-overflow/blob/master/4-Interpretibility/IBMEmployeeAttritionClassifier_Interpretability.ipynb)\n", + "\n", + "_**This notebook showcases how to use the Azure Machine Learning Interpretability SDK to train and explain a binary classification model locally.**_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What is Azure Machine Learning Service?\n", + "Azure Machine Learning service is a cloud service that you can use to develop and deploy machine learning models. Using Azure Machine Learning service, you can track your models as you build, train, deploy, and manage them, all at the broad scale that the cloud provides.\n", + "![](./images/aml-overview.png)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What Is Machine Learning Interpretability?\n", + "Interpretability is the ability to explain why your model made the predictions it did. The Azure Machine Learning service offers various interpretability features to help accomplish this task. These features include:\n", + "\n", + "- Feature importance values for both raw and engineered features.\n", + "- Interpretability on real-world datasets at scale, during training and inference.\n", + "- Interactive visualizations to aid you in the discovery of patterns in data and explanations at training time.\n", + "\n", + "By accurately interpretabiliting your model, it allows you to:\n", + "\n", + "- Use the insights for debugging your model.\n", + "- Validate model behavior matches their objectives.\n", + "- Check for for bias in the model.\n", + "- Build trust in your customers and stakeholders.\n", + "\n", + "![](./images/interpretability-architecture.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Change Tensorflow and Interpret Library Versions\n", + "\n", + "We will be using an older version (1.14) for this particular tutorial in the series as Tensorflow 2.0 is not yet supported for Interpretibility on Azure Machine Learning service. We will also be using version 0.1.0.4 of the interpret library. \n", + "\n", + "If haven't already done so, please update your library versions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Uninstalling tensorflow-gpu-1.14.0:\n", + " Successfully uninstalled tensorflow-gpu-1.14.0\n", + "\u001b[33mWARNING: Skipping keras as it is not installed.\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n", + "Collecting tensorflow-gpu==1.14\n", + " Using cached https://files.pythonhosted.org/packages/76/04/43153bfdfcf6c9a4c38ecdb971ca9a75b9a791bb69a764d652c359aca504/tensorflow_gpu-1.14.0-cp36-cp36m-manylinux1_x86_64.whl\n" + ] + } + ], + "source": [ + "%pip uninstall tensorflow-gpu keras --yes\n", + "%pip install tensorflow-gpu==1.14 interpret-community==0.1.0.4" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After installing packages, you must close and reopen the notebook as well as restarting the kernel.\n", + "\n", + "Let's make sure we have the right verisons" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.14.0\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "import interpret_community\n", + "\n", + "print(tf.version.VERSION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train Model\n", + "For this tutorial, we will be using the *tf.keras module* to train a basic feed forward neural network on the IBM Employee Attrition Dataset. \n", + "\n", + "**We will start by writing the training script to train our model**" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train on 1176 samples, validate on 294 samples\n", + "Epoch 1/20\n", + "1176/1176 [==============================] - 0s 167us/sample - loss: 0.7689 - acc: 0.3903 - val_loss: 0.6767 - val_acc: 0.5306\n", + "Epoch 2/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.6253 - acc: 0.6105 - val_loss: 0.5807 - val_acc: 0.6939\n", + "Epoch 3/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.5428 - acc: 0.7594 - val_loss: 0.5112 - val_acc: 0.8061\n", + "Epoch 4/20\n", + "1176/1176 [==============================] - 0s 14us/sample - loss: 0.4828 - acc: 0.8231 - val_loss: 0.4612 - val_acc: 0.8265\n", + "Epoch 5/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.4437 - acc: 0.8359 - val_loss: 0.4345 - val_acc: 0.8333\n", + "Epoch 6/20\n", + "1176/1176 [==============================] - 0s 14us/sample - loss: 0.4219 - acc: 0.8401 - val_loss: 0.4150 - val_acc: 0.8435\n", + "Epoch 7/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.4060 - acc: 0.8401 - val_loss: 0.4043 - val_acc: 0.8401\n", + "Epoch 8/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3947 - acc: 0.8410 - val_loss: 0.3952 - val_acc: 0.8435\n", + "Epoch 9/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3842 - acc: 0.8410 - val_loss: 0.3868 - val_acc: 0.8469\n", + "Epoch 10/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3736 - acc: 0.8444 - val_loss: 0.3795 - val_acc: 0.8469\n", + "Epoch 11/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3640 - acc: 0.8452 - val_loss: 0.3713 - val_acc: 0.8469\n", + "Epoch 12/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3543 - acc: 0.8520 - val_loss: 0.3663 - val_acc: 0.8469\n", + "Epoch 13/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3448 - acc: 0.8537 - val_loss: 0.3611 - val_acc: 0.8537\n", + "Epoch 14/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3362 - acc: 0.8580 - val_loss: 0.3576 - val_acc: 0.8537\n", + "Epoch 15/20\n", + "1176/1176 [==============================] - 0s 16us/sample - loss: 0.3287 - acc: 0.8614 - val_loss: 0.3526 - val_acc: 0.8571\n", + "Epoch 16/20\n", + "1176/1176 [==============================] - 0s 16us/sample - loss: 0.3206 - acc: 0.8707 - val_loss: 0.3479 - val_acc: 0.8503\n", + "Epoch 17/20\n", + "1176/1176 [==============================] - 0s 19us/sample - loss: 0.3134 - acc: 0.8776 - val_loss: 0.3450 - val_acc: 0.8639\n", + "Epoch 18/20\n", + "1176/1176 [==============================] - 0s 17us/sample - loss: 0.3061 - acc: 0.8801 - val_loss: 0.3438 - val_acc: 0.8639\n", + "Epoch 19/20\n", + "1176/1176 [==============================] - 0s 15us/sample - loss: 0.3005 - acc: 0.8861 - val_loss: 0.3422 - val_acc: 0.8673\n", + "Epoch 20/20\n", + "1176/1176 [==============================] - 0s 16us/sample - loss: 0.2950 - acc: 0.8903 - val_loss: 0.3402 - val_acc: 0.8741\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd \n", + "import numpy as np\n", + "import tensorflow as tf\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.compose import ColumnTransformer\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.impute import SimpleImputer\n", + "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "def preprocess_data(data):\n", + " '''\n", + " \n", + " '''\n", + " # Dropping Employee count as all values are 1 and hence attrition is independent of this feature\n", + " data = data.drop(['EmployeeCount'], axis=1)\n", + " \n", + " # Dropping Employee Number since it is merely an identifier\n", + " data = data.drop(['EmployeeNumber'], axis=1)\n", + " data = data.drop(['Over18'], axis=1)\n", + "\n", + " # Since all values are 80\n", + " data = data.drop(['StandardHours'], axis=1)\n", + "\n", + " # Converting target variables from string to numerical values\n", + " target_map = {'Yes': 1, 'No': 0}\n", + " data[\"Attrition_numerical\"] = data[\"Attrition\"].apply(lambda x: target_map[x])\n", + " target = data[\"Attrition_numerical\"]\n", + "\n", + " data.drop(['Attrition_numerical', 'Attrition'], axis=1, inplace=True)\n", + " \n", + " # Creating dummy columns for each categorical feature\n", + " categorical = []\n", + " for col, value in data.iteritems():\n", + " if value.dtype == 'object':\n", + " categorical.append(col)\n", + "\n", + " # Store the numerical columns in a list numerical\n", + " numerical = data.columns.difference(categorical) \n", + "\n", + " # We create the preprocessing pipelines for both numeric and categorical data.\n", + " numeric_transformer = Pipeline(steps=[\n", + " ('imputer', SimpleImputer(strategy='median')),\n", + " ('scaler', StandardScaler())])\n", + "\n", + " categorical_transformer = Pipeline(steps=[\n", + " ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),\n", + " ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n", + "\n", + " preprocess = ColumnTransformer(\n", + " transformers=[\n", + " ('num', numeric_transformer, numerical),\n", + " ('cat', categorical_transformer, categorical)])\n", + " \n", + " pipeline = make_pipeline(preprocess)\n", + "\n", + " # Split data into train and test sets\n", + " x_train, x_test, y_train, y_test = train_test_split(data, \n", + " target, \n", + " test_size=0.2,\n", + " random_state=0,\n", + " stratify=target)\n", + " \n", + " return x_train, x_test, y_train, y_test, pipeline, preprocess\n", + " \n", + "# Load and preprocess data\n", + "attrition_data = pd.read_csv('./data/data.csv')\n", + "x_train, x_test, y_train, y_test, pipeline, preprocess = preprocess_data(attrition_data)\n", + "\n", + "# Transform data\n", + "x_train_t = pipeline.fit_transform(x_train)\n", + "x_test_t = pipeline.transform(x_test)\n", + "\n", + "# Create model\n", + "model = tf.keras.models.Sequential()\n", + "model.add(tf.keras.layers.Dense(units=16, activation='relu', input_shape=(x_train_t.shape[1],)))\n", + "model.add(tf.keras.layers.Dense(units=16, activation='relu'))\n", + "model.add(tf.keras.layers.Dense(units=1, activation='sigmoid'))\n", + "\n", + "# Compile model\n", + "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) \n", + "\n", + "# Fit model\n", + "model.fit(x_train_t, y_train, epochs=20, verbose=1, batch_size=128, validation_data=(x_test_t, y_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Explain Model Locally\n", + "\n", + "We will start by explaining the trained model locally." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Instantiate the explainer object using trained model.**" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "from interpret.ext.greybox import DeepExplainer\n", + "\n", + "explainer = DeepExplainer(model,\n", + " x_train,\n", + " features=x_train.columns,\n", + " classes=[\"STAYING\", \"LEAVING\"], \n", + " transformations = preprocess,\n", + " model_task=\"classification\",\n", + " is_classifier=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Generate global explanations**" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# Passing in test dataset for evaluation examples - note it must be a representative sample of the original data\n", + "# x_train can be passed as well, but with more examples explanations will take longer although they may be more accurate\n", + "global_explanation = explainer.explain_global(x_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "global importance rank: {'EducationField': 0.053873279205873495, 'JobSatisfaction': 0.03953571012067353, 'OverTime': 0.038583560133207156, 'NumCompaniesWorked': 0.03640758117990181, 'MaritalStatus': 0.029883175044257942, 'YearsSinceLastPromotion': 0.02740686255297018, 'WorkLifeBalance': 0.025936746100254413, 'Age': 0.025243059923775336, 'EnvironmentSatisfaction': 0.024909378453501194, 'JobRole': 0.024056074550579537, 'JobInvolvement': 0.023541541358297214, 'YearsInCurrentRole': 0.020543295211741034, 'MonthlyIncome': 0.02048903332178757, 'RelationshipSatisfaction': 0.017865455491885895, 'Department': 0.016978380925069186, 'DistanceFromHome': 0.016530930528834187, 'BusinessTravel': 0.014322739471855634, 'TotalWorkingYears': 0.012453615317841563, 'DailyRate': 0.011737043367600794, 'StockOptionLevel': 0.011188063218993892, 'YearsWithCurrManager': 0.01045222903210198, 'YearsAtCompany': 0.00971274163942977, 'HourlyRate': 0.00966349132097822, 'PercentSalaryHike': 0.008843251050813376, 'JobLevel': 0.007890152720531868, 'Gender': 0.007496610022988447, 'MonthlyRate': 0.005103606391774755, 'TrainingTimesLastYear': 0.004966644990881875, 'Education': 0.0046528384198318215, 'PerformanceRating': 0.004132269554156894}\n" + ] + } + ], + "source": [ + "# Print out a dictionary that holds the sorted feature importance names and values\n", + "print('global importance rank: {}'.format(global_explanation.get_feature_importance_dict()))" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ranked per class feature names: [['EducationField', 'JobSatisfaction', 'OverTime', 'NumCompaniesWorked', 'MaritalStatus', 'YearsSinceLastPromotion', 'WorkLifeBalance', 'Age', 'EnvironmentSatisfaction', 'JobRole', 'JobInvolvement', 'YearsInCurrentRole', 'MonthlyIncome', 'RelationshipSatisfaction', 'Department', 'DistanceFromHome', 'BusinessTravel', 'TotalWorkingYears', 'DailyRate', 'StockOptionLevel', 'YearsWithCurrManager', 'YearsAtCompany', 'HourlyRate', 'PercentSalaryHike', 'JobLevel', 'Gender', 'MonthlyRate', 'TrainingTimesLastYear', 'Education', 'PerformanceRating'], ['EducationField', 'JobSatisfaction', 'OverTime', 'NumCompaniesWorked', 'MaritalStatus', 'YearsSinceLastPromotion', 'WorkLifeBalance', 'Age', 'EnvironmentSatisfaction', 'JobRole', 'JobInvolvement', 'YearsInCurrentRole', 'MonthlyIncome', 'RelationshipSatisfaction', 'Department', 'DistanceFromHome', 'BusinessTravel', 'TotalWorkingYears', 'DailyRate', 'StockOptionLevel', 'YearsWithCurrManager', 'YearsAtCompany', 'HourlyRate', 'PercentSalaryHike', 'JobLevel', 'Gender', 'MonthlyRate', 'TrainingTimesLastYear', 'Education', 'PerformanceRating']]\n", + "ranked per class feature values: [[0.053873279205873495, 0.03953571012067353, 0.038583560133207156, 0.03640758117990181, 0.029883175044257942, 0.02740686255297018, 0.025936746100254413, 0.025243059923775336, 0.024909378453501194, 0.024056074550579537, 0.023541541358297214, 0.020543295211741034, 0.02048903332178757, 0.017865455491885895, 0.016978380925069186, 0.016530930528834187, 0.014322739471855634, 0.012453615317841563, 0.011737043367600794, 0.011188063218993892, 0.01045222903210198, 0.00971274163942977, 0.00966349132097822, 0.008843251050813376, 0.007890152720531868, 0.007496610022988447, 0.005103606391774755, 0.004966644990881875, 0.0046528384198318215, 0.004132269554156894], [0.053873279205873495, 0.03953571012067353, 0.038583560133207156, 0.03640758117990181, 0.029883175044257942, 0.02740686255297018, 0.025936746100254413, 0.025243059923775336, 0.024909378453501194, 0.024056074550579537, 0.023541541358297214, 0.020543295211741034, 0.02048903332178757, 0.017865455491885895, 0.016978380925069186, 0.016530930528834187, 0.014322739471855634, 0.012453615317841563, 0.011737043367600794, 0.011188063218993892, 0.01045222903210198, 0.00971274163942977, 0.00966349132097822, 0.008843251050813376, 0.007890152720531868, 0.007496610022988447, 0.005103606391774755, 0.004966644990881875, 0.0046528384198318215, 0.004132269554156894]]\n" + ] + } + ], + "source": [ + "# Per class feature names\n", + "print('ranked per class feature names: {}'.format(global_explanation.get_ranked_per_class_names()))\n", + "\n", + "# Per class feature importance values\n", + "print('ranked per class feature values: {}'.format(global_explanation.get_ranked_per_class_values()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Generate local explanations**" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "local importance values: [[[0.04342116583138704, 0.03674833151006784, 0.022893913795502326, 0.016427155338850478, 0.014972139429301024, 0.011560527652549094, 0.009173502097810305, 0.008018425133635074, 0.005188390924937305, 0.0039744281295333705, 0.0039462783451609, 0.003842337903712896, 0.002779007529655213, 0.0023964009358080716, 0.0018396283919989028, 0.0012093021415323102, 0.0009178496431559324, 0.00018800738107529468, 0.0, 0.0, 0.0, -4.9312862829227565e-05, -0.0008976143394395346, -0.0034775503413386846, -0.005650149990708967, -0.00817697712711508, -0.015045928765388503, -0.015105825511097477, -0.018208197918336413, -0.0426478537150794]], [[0.0426478537150794, 0.018208197918336413, 0.015105825511097477, 0.015045928765388503, 0.00817697712711508, 0.005650149990708967, 0.0034775503413386846, 0.0008976143394395346, 4.9312862829227565e-05, 0.0, 0.0, 0.0, -0.00018800738107529468, -0.0009178496431559324, -0.0012093021415323102, -0.0018396283919989028, -0.0023964009358080716, -0.002779007529655213, -0.003842337903712896, -0.0039462783451609, -0.0039744281295333705, -0.005188390924937305, -0.008018425133635074, -0.009173502097810305, -0.011560527652549094, -0.014972139429301024, -0.016427155338850478, -0.022893913795502326, -0.03674833151006784, -0.04342116583138704]]]\n", + "local importance names: [[['EducationField', 'MonthlyIncome', 'Age', 'WorkLifeBalance', 'MaritalStatus', 'DailyRate', 'YearsSinceLastPromotion', 'JobSatisfaction', 'NumCompaniesWorked', 'PerformanceRating', 'YearsAtCompany', 'RelationshipSatisfaction', 'StockOptionLevel', 'DistanceFromHome', 'MonthlyRate', 'TotalWorkingYears', 'Gender', 'JobRole', 'OverTime', 'BusinessTravel', 'Department', 'JobLevel', 'Education', 'TrainingTimesLastYear', 'PercentSalaryHike', 'HourlyRate', 'YearsInCurrentRole', 'JobInvolvement', 'YearsWithCurrManager', 'EnvironmentSatisfaction']], [['EnvironmentSatisfaction', 'YearsWithCurrManager', 'JobInvolvement', 'YearsInCurrentRole', 'HourlyRate', 'PercentSalaryHike', 'TrainingTimesLastYear', 'Education', 'JobLevel', 'OverTime', 'Department', 'BusinessTravel', 'JobRole', 'Gender', 'TotalWorkingYears', 'MonthlyRate', 'DistanceFromHome', 'StockOptionLevel', 'RelationshipSatisfaction', 'YearsAtCompany', 'PerformanceRating', 'NumCompaniesWorked', 'JobSatisfaction', 'YearsSinceLastPromotion', 'DailyRate', 'MaritalStatus', 'WorkLifeBalance', 'Age', 'MonthlyIncome', 'EducationField']]]\n" + ] + } + ], + "source": [ + "# You can pass a specific data point or a group of data points to the explain_local function\n", + "# E.g., Explain the first data point in the test set\n", + "instance_num = 1\n", + "local_explanation = explainer.explain_local(x_test[:instance_num])\n", + "\n", + "sorted_local_importance_values = local_explanation.get_ranked_local_values()\n", + "sorted_local_importance_names = local_explanation.get_ranked_local_names()\n", + "\n", + "print('local importance values: {}'.format(sorted_local_importance_values))\n", + "print('local importance names: {}'.format(sorted_local_importance_names))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Visualize our explanations**" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "75493807b6d64b06b6aa59e2cf5fa88f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "ExplanationWidget(value={'predictedY': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from interpret_community.widget import ExplanationDashboard\n", + "from interpret_community.common.model_wrapper import wrap_model\n", + "from interpret_community.dataset.dataset_wrapper import DatasetWrapper\n", + "from sklearn.pipeline import Pipeline\n", + "\n", + "wrapped_model, ml_domain = wrap_model(model, DatasetWrapper(x_test_t), \"classification\")\n", + "wrapped_model.fit = model.fit\n", + "dashboard_pipeline = Pipeline(steps=[('preprocess', preprocess), ('network', wrapped_model)])\n", + "ExplanationDashboard(global_explanation, dashboard_pipeline, datasetX=x_test)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/4-Interpretibility/EmployeeAttritionClassifier_Interpretability_Remote.ipynb b/4-Interpretibility/EmployeeAttritionClassifier_Interpretability_Remote.ipynb new file mode 100644 index 0000000..eccd001 --- /dev/null +++ b/4-Interpretibility/EmployeeAttritionClassifier_Interpretability_Remote.ipynb @@ -0,0 +1,681 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![Impressions](https://PixelServer20190423114238.azurewebsites.net/api/impressions/MachineLearningNotebooks/how-to-use-azureml/explain-model/explain-on-amlcompute/regression-sklearn-on-amlcompute.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Interpretability With Tensorflow On Azure Machine Learning Service (Remote)\n", + "\n", + "\n", + "## Overview of Tutorial\n", + "This notebook is Part 4 (Explaining Your Model Using Interpretability) of a four part workshop that demonstrates an end-to-end workflow for using Tensorflow on Azure Machine Learning Service. The different components of the workshop are as follows:\n", + "\n", + "- Part 1: [Preparing Data and Model Training](https://github.com/microsoft/bert-stack-overflow/blob/master/1-Training/AzureServiceClassifier_Training.ipynb)\n", + "- Part 2: [Inferencing and Deploying a Model](https://github.com/microsoft/bert-stack-overflow/blob/master/2-Inferencing/AzureServiceClassifier_Inferencing.ipynb)\n", + "- Part 3: [Setting Up a Pipeline Using MLOps](https://github.com/microsoft/bert-stack-overflow/tree/master/3-ML-Ops)\n", + "- Part 4: [Explaining Your Model Interpretability](https://github.com/microsoft/bert-stack-overflow/blob/master/4-Interpretibility/IBMEmployeeAttritionClassifier_Interpretability.ipynb)\n", + "\n", + "_**This notebook showcases how to use the Azure Machine Learning Interpretability SDK to train and explain a binary classification model remotely on an Azure Machine Leanrning Compute Target (AMLCompute).**_\n", + "\n", + "## Table of Contents\n", + "\n", + "1. [Introduction](#Introduction)\n", + "1. [Setup](#Setup)\n", + " 1. Initialize a Workspace\n", + " 1. Create an Experiment\n", + " 1. Introduction to AmlCompute\n", + " 1. Submit an AmlCompute run \n", + "1. Additional operations to perform on AmlCompute\n", + "1. [Download model explanations from Azure Machine Learning Run History](#Download)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "This notebook showcases how to train and explain a binary classification model remotely via Azure Machine Learning Compute (AMLCompute), and download the calculated explanations locally on your personal machine.\n", + "It demonstrates the API calls that you need to make to submit a run for training and explaining a model to AMLCompute, and download the compute explanations remotely.\n", + "\n", + "We will showcase one of the tabular data explainers: TabularExplainer (SHAP).\n", + "\n", + "Problem: Employee Attrition Classification Problem\n", + "\n", + "\n", + "\n", + "![](./images/interpretability-architecture.png)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Change Tensorflow and Interpret Library Versions\n", + "\n", + "We will be using an older version (1.14) for this particular tutorial in the series as Tensorflow 2.0 is not yet supported for Interpretibility on Azure Machine Learning service. We will also be using version 0.1.0.4 of the interpret library. \n", + "\n", + "If haven't already done so, please update your library versions." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting azureml.contrib.interpret\n", + " Using cached https://files.pythonhosted.org/packages/05/cf/05ff8cc39de0c97bc1fd564dc618a8256ff5b2f08446556fc73435e69652/azureml_contrib_interpret-1.0.69-py3-none-any.whl\n", + "Requirement already satisfied, skipping upgrade: azureml-interpret==1.0.69.* in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from azureml.contrib.interpret) (1.0.69)\n", + "Collecting interpret-community==0.1.0.2 (from azureml-interpret==1.0.69.*->azureml.contrib.interpret)\n", + " Using cached https://files.pythonhosted.org/packages/8b/3b/a7eb6beac2d8b21ea442ffe73d90b236e89c97bc6e2c805bcb96ed2c0bdf/interpret_community-0.1.0.2-py3-none-any.whl\n", + "Requirement already satisfied, skipping upgrade: shap<=0.29.3,>=0.20.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.29.3)\n", + "Requirement already satisfied, skipping upgrade: interpret>=0.1.17 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.1.18)\n", + "Requirement already satisfied, skipping upgrade: pandas in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.23.4)\n", + "Requirement already satisfied, skipping upgrade: packaging in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (19.2)\n", + "Requirement already satisfied, skipping upgrade: scipy in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.1.0)\n", + "Requirement already satisfied, skipping upgrade: scikit-learn in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.20.3)\n", + "Requirement already satisfied, skipping upgrade: numpy in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.16.2)\n", + "Requirement already satisfied, skipping upgrade: matplotlib in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (3.1.1)\n", + "Requirement already satisfied, skipping upgrade: tqdm>4.25.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (4.36.1)\n", + "Requirement already satisfied, skipping upgrade: scikit-image in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.16.1)\n", + "Requirement already satisfied, skipping upgrade: ipython in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (7.8.0)\n", + "Requirement already satisfied, skipping upgrade: interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.1.18)\n", + "Requirement already satisfied, skipping upgrade: python-dateutil>=2.5.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from pandas->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.8.0)\n", + "Requirement already satisfied, skipping upgrade: pytz>=2011k in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from pandas->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2019.3)\n", + "Requirement already satisfied, skipping upgrade: six in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from packaging->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.12.0)\n", + "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from packaging->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.4.2)\n", + "Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from matplotlib->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.1.0)\n", + "Requirement already satisfied, skipping upgrade: cycler>=0.10 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from matplotlib->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.10.0)\n", + "Requirement already satisfied, skipping upgrade: imageio>=2.3.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from scikit-image->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.6.0)\n", + "Requirement already satisfied, skipping upgrade: pillow>=4.3.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from scikit-image->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (6.2.0)\n", + "Requirement already satisfied, skipping upgrade: PyWavelets>=0.4.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from scikit-image->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.0.3)\n", + "Requirement already satisfied, skipping upgrade: networkx>=2.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from scikit-image->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.3)\n", + "Requirement already satisfied, skipping upgrade: pygments in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.4.2)\n", + "Requirement already satisfied, skipping upgrade: pexpect; sys_platform != \"win32\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (4.7.0)\n", + "Requirement already satisfied, skipping upgrade: prompt-toolkit<2.1.0,>=2.0.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.0.10)\n", + "Requirement already satisfied, skipping upgrade: backcall in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.1.0)\n", + "Requirement already satisfied, skipping upgrade: decorator in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (4.4.0)\n", + "Requirement already satisfied, skipping upgrade: setuptools>=18.5 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (41.4.0)\n", + "Requirement already satisfied, skipping upgrade: jedi>=0.10 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.15.1)\n", + "Requirement already satisfied, skipping upgrade: traitlets>=4.2 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (4.3.3)\n", + "Requirement already satisfied, skipping upgrade: pickleshare in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.7.5)\n", + "Requirement already satisfied, skipping upgrade: dash>=1.0.0; extra == \"dash\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.4.0)\n", + "Requirement already satisfied, skipping upgrade: dash-table>=4.1.0; extra == \"dash\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (4.4.0)\n", + "Requirement already satisfied, skipping upgrade: gevent>=1.3.6; extra == \"dash\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.4.0)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied, skipping upgrade: dash-cytoscape>=0.1.1; extra == \"dash\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.1.1)\n", + "Requirement already satisfied, skipping upgrade: requests>=2.19.0; extra == \"dash\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.22.0)\n", + "Requirement already satisfied, skipping upgrade: psutil>=5.6.2; extra == \"debug\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (5.6.3)\n", + "Requirement already satisfied, skipping upgrade: joblib>=0.11; extra == \"decisiontree\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.13.2)\n", + "Requirement already satisfied, skipping upgrade: lime>=0.1.1.33; extra == \"lime\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.1.1.36)\n", + "Requirement already satisfied, skipping upgrade: ipykernel>=5.1.0; extra == \"notebook\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (5.1.2)\n", + "Requirement already satisfied, skipping upgrade: plotly>=3.8.1; extra == \"plotly\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (4.1.1)\n", + "Requirement already satisfied, skipping upgrade: SALib>=1.3.3; extra == \"sensitivity\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.3.8)\n", + "Requirement already satisfied, skipping upgrade: dill>=0.2.5; extra == \"shap\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.3.1.1)\n", + "Requirement already satisfied, skipping upgrade: treeinterpreter>=0.2.2; extra == \"treeinterpreter\" in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.2.2)\n", + "Requirement already satisfied, skipping upgrade: ptyprocess>=0.5 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from pexpect; sys_platform != \"win32\"->ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.6.0)\n", + "Requirement already satisfied, skipping upgrade: wcwidth in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.1.7)\n", + "Requirement already satisfied, skipping upgrade: parso>=0.5.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from jedi>=0.10->ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.5.1)\n", + "Requirement already satisfied, skipping upgrade: ipython-genutils in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from traitlets>=4.2->ipython->shap<=0.29.3,>=0.20.0->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.2.0)\n", + "Requirement already satisfied, skipping upgrade: dash-core-components==1.3.0 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.3.0)\n", + "Requirement already satisfied, skipping upgrade: dash-renderer==1.1.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.1.1)\n", + "Requirement already satisfied, skipping upgrade: flask-compress in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.4.0)\n", + "Requirement already satisfied, skipping upgrade: Flask>=1.0.2 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.1.1)\n", + "Requirement already satisfied, skipping upgrade: future in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.18.0)\n", + "Requirement already satisfied, skipping upgrade: dash-html-components==1.0.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.0.1)\n", + "Requirement already satisfied, skipping upgrade: greenlet>=0.4.14 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from gevent>=1.3.6; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.4.15)\n", + "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests>=2.19.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.24.2)\n", + "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests>=2.19.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (3.0.4)\n", + "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests>=2.19.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.8)\n", + "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from requests>=2.19.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2019.9.11)\n", + "Requirement already satisfied, skipping upgrade: tornado>=4.2 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipykernel>=5.1.0; extra == \"notebook\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (6.0.3)\n", + "Requirement already satisfied, skipping upgrade: jupyter-client in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from ipykernel>=5.1.0; extra == \"notebook\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (5.3.3)\n", + "Requirement already satisfied, skipping upgrade: retrying>=1.3.3 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from plotly>=3.8.1; extra == \"plotly\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.3.3)\n", + "Requirement already satisfied, skipping upgrade: itsdangerous>=0.24 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from Flask>=1.0.2->dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.1.0)\n", + "Requirement already satisfied, skipping upgrade: Werkzeug>=0.15 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from Flask>=1.0.2->dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (0.16.0)\n", + "Requirement already satisfied, skipping upgrade: click>=5.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from Flask>=1.0.2->dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (7.0)\n", + "Requirement already satisfied, skipping upgrade: Jinja2>=2.10.1 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from Flask>=1.0.2->dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (2.10.3)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied, skipping upgrade: jupyter-core in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from jupyter-client->ipykernel>=5.1.0; extra == \"notebook\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (4.5.0)\n", + "Requirement already satisfied, skipping upgrade: pyzmq>=13 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from jupyter-client->ipykernel>=5.1.0; extra == \"notebook\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (18.1.0)\n", + "Requirement already satisfied, skipping upgrade: MarkupSafe>=0.23 in /anaconda/envs/azureml_py36/lib/python3.6/site-packages (from Jinja2>=2.10.1->Flask>=1.0.2->dash>=1.0.0; extra == \"dash\"->interpret-core[dash,debug,decisiontree,ebm,lime,linear,notebook,plotly,required,sensitivity,shap,treeinterpreter]>=0.1.18->interpret>=0.1.17->interpret-community==0.1.0.2->azureml-interpret==1.0.69.*->azureml.contrib.interpret) (1.1.1)\n", + "Installing collected packages: azureml.contrib.interpret, interpret-community\n", + " Found existing installation: interpret-community 0.1.0.4\n", + " Uninstalling interpret-community-0.1.0.4:\n", + " Successfully uninstalled interpret-community-0.1.0.4\n", + "Successfully installed azureml.contrib.interpret interpret-community-0.1.0.2\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip uninstall tensorflow-gpu keras --yes\n", + "%pip install tensorflow-gpu==1.14 interpret-community==0.1.0.4" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After installing packages, you must close and reopen the notebook as well as restarting the kernel.\n", + "\n", + "Let's make sure we have the right verisons" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.14.0\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "import interpret_community\n", + "\n", + "print(tf.version.VERSION)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SDK version: 1.0.69\n" + ] + } + ], + "source": [ + "# Check core SDK version number\n", + "import azureml.core\n", + "\n", + "print(\"SDK version:\", azureml.core.VERSION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connect To Workspace\n", + "\n", + "Just like in the previous tutorials, we will need to connect to a [workspace](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.workspace(class)?view=azure-ml-py).\n", + "\n", + "The following code will allow you to create a workspace if you don't already have one created. You must have an Azure subscription to create a workspace:\n", + "\n", + "```python\n", + "from azureml.core import Workspace\n", + "ws = Workspace.create(name='myworkspace',\n", + " subscription_id='',\n", + " resource_group='myresourcegroup',\n", + " create_resource_group=True,\n", + " location='eastus2')\n", + "```\n", + "\n", + "**If you are running this on a Notebook VM, you can import the existing workspace.**" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [ + "create workspace" + ] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "107835-aml-ws\n", + "aml-rg-107835\n", + "southcentralus\n", + "07a3b836-0813-4c05-afd4-3a7ab00358d9\n" + ] + } + ], + "source": [ + "from azureml.core import Workspace\n", + "\n", + "ws = Workspace.from_config()\n", + "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep='\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> **Note:** that the above commands reads a config.json file that exists by default within the Notebook VM. If you are running this locally or want to use a different workspace, you must add a config file to your project directory. The config file should have the following schema:\n", + "\n", + "```\n", + " {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group\": \"\",\n", + " \"workspace_name\": \"\"\n", + " }\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create An Experiment\n", + "\n", + "**Experiment** is a logical container in an Azure ML Workspace. It hosts run records which can include run metrics and output artifacts from your experiments." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from azureml.core import Experiment\n", + "experiment_name = 'explainer-remote-run-tfworld19'\n", + "experiment = Experiment(workspace=ws, name=experiment_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create Compute Target\n", + "\n", + "A [compute target](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.computetarget?view=azure-ml-py) is a designated compute resource/environment where you run your training script or host your service deployment. This location may be your local machine or a cloud-based compute resource. Compute targets can be reused across the workspace for different runs and experiments. \n", + "\n", + "**If you completed tutorial 1 of this series, then you should have already created a compute target and can skip this step**\n", + "\n", + "Otherwise, run the cell below to create an auto-scaling [Azure Machine Learning Compute](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.compute.amlcompute?view=azure-ml-py) cluster, which is a managed-compute infrastructure that allows the user to easily create a single or multi-node compute. To create the cluster, we need to specify the following parameters:\n", + "\n", + "- `vm_size`: The is the type of GPUs that we want to use in our cluster. For this tutorial, we will use **Standard_NC12s_v3 (NVIDIA V100) GPU Machines** .\n", + "- `idle_seconds_before_scaledown`: This is the number of seconds before a node will scale down in our auto-scaling cluster. We will set this to **6000** seconds. \n", + "- `min_nodes`: This is the minimum numbers of nodes that the cluster will have. To avoid paying for compute while they are not being used, we will set this to **0** nodes.\n", + "- `max_modes`: This is the maximum number of nodes that the cluster will scale up to. Will will set this to **2** nodes.\n", + "\n", + "**When jobs are submitted to the cluster it takes approximately 5 minutes to allocate new nodes** " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Succeeded\n", + "AmlCompute wait for completion finished\n", + "Minimum number of nodes requested have been provisioned\n" + ] + } + ], + "source": [ + "from azureml.core.compute import AmlCompute, ComputeTarget\n", + "\n", + "cluster_name = 'v100cluster'\n", + "compute_config = AmlCompute.provisioning_configuration(vm_size='Standard_NC12s_v3', \n", + " idle_seconds_before_scaledown=6000,\n", + " min_nodes=0, \n", + " max_nodes=2)\n", + "\n", + "compute_target = ComputeTarget.create(ws, cluster_name, compute_config)\n", + "compute_target.wait_for_completion(show_output=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**If you already have the compute target created, then you can directly run this cell.**" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "compute_target = ws.compute_targets['v100cluster']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Submit Experiment Run\n", + "\n", + "Now that our compute is ready, we can begin to submit our run." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create project directory\n", + "\n", + "Create a directory that will contain all the necessary code from your local machine that you will need access to on the remote resource. This includes the training script, and any additional files your training script depends on" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'./TFworld-explainer-remote-run-on-amlcompute/train_explain-model.py'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "import shutil\n", + "\n", + "project_folder = './TFworld-explainer-remote-run-on-amlcompute'\n", + "os.makedirs(project_folder, exist_ok=True)\n", + "shutil.copy('train_explain-model.py', project_folder)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Submit job to cluster" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
ExperimentIdTypeStatusDetails PageDocs Page
explainer-remote-run-tfworld19explainer-remote-run-tfworld19_1572371711_23a86907azureml.scriptrunStartingLink to Azure PortalLink to Documentation
" + ], + "text/plain": [ + "Run(Experiment: explainer-remote-run-tfworld19,\n", + "Id: explainer-remote-run-tfworld19_1572371711_23a86907,\n", + "Type: azureml.scriptrun,\n", + "Status: Starting)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from azureml.core.runconfig import RunConfiguration\n", + "from azureml.core.conda_dependencies import CondaDependencies\n", + "from azureml.core.runconfig import DEFAULT_CPU_IMAGE\n", + "\n", + "# create a new runconfig object\n", + "run_config = RunConfiguration()\n", + "\n", + "# signal that you want to use our cluster to execute script.\n", + "run_config.target = compute_target\n", + "\n", + "# enable Docker \n", + "run_config.environment.docker.enabled = True\n", + "\n", + "# set Docker base image to the default CPU-based image\n", + "run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE\n", + "\n", + "# use conda_dependencies.yml to create a conda environment in the Docker image for execution\n", + "run_config.environment.python.user_managed_dependencies = False\n", + "\n", + "\n", + "pip_packages = [\n", + " 'azureml-defaults', 'azureml-core', 'azureml-telemetry',\n", + " 'azureml-dataprep', 'sklearn', 'sklearn-pandas', 'tensorflow==1.14.0',\n", + " 'azureml-contrib-interpret', 'azureml-interpret'\n", + "]\n", + "\n", + "\n", + "index_url = 'https://azuremlsdktestpypi.azureedge.net/sdk-release/Candidate/604C89A437BA41BD942B4F46D9A3591D/'\n", + "run_config.environment.python.conda_dependencies = CondaDependencies.create(pip_packages=pip_packages,\n", + " pin_sdk_version=False, \n", + " pip_indexurl=index_url)\n", + "\n", + "\n", + "\n", + "# Now submit a run on AmlCompute\n", + "from azureml.core.script_run_config import ScriptRunConfig\n", + "\n", + "script_run_config = ScriptRunConfig(source_directory=project_folder,\n", + " script='train_explain-model.py',\n", + " run_config=run_config)\n", + "\n", + "run = experiment.submit(script_run_config)\n", + "\n", + "# Show run details\n", + "run" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: if you need to cancel a run, you can follow [these instructions](https://aka.ms/aml-docs-cancel-run)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0138bca23c5a4ba69d43ee60245bd61a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "_UserRunWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', '…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from azureml.widgets import RunDetails\n", + "RunDetails(run).show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download \n", + "Download model explanation data." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'NoneType' object has no attribute 'local_importance_values'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mExplanationClient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mglobal_explanation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload_model_explanation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mlocal_importance_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mglobal_explanation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocal_importance_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0mexpected_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mglobal_explanation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpected_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'local_importance_values'" + ] + } + ], + "source": [ + "from azureml.contrib.interpret.explanation.explanation_client import ExplanationClient\n", + "\n", + "# Get model explanation data\n", + "client = ExplanationClient.from_run(run)\n", + "global_explanation = client.download_model_explanation()\n", + "local_importance_values = global_explanation.local_importance_values\n", + "expected_values = global_explanation.expected_values\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Or you can use the saved run.id to retrive the feature importance values\n", + "client = ExplanationClient.from_run_id(ws, experiment_name, run.id)\n", + "global_explanation = client.download_model_explanation()\n", + "local_importance_values = global_explanation.local_importance_values\n", + "expected_values = global_explanation.expected_values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get the top k (e.g., 4) most important features with their importance values\n", + "global_explanation_topk = client.download_model_explanation(top_k=4)\n", + "global_importance_values = global_explanation_topk.get_ranked_global_values()\n", + "global_importance_names = global_explanation_topk.get_ranked_global_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('global importance values: {}'.format(global_importance_values))\n", + "print('global importance names: {}'.format(global_importance_names))" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [], + "source": [ + "# Get the top k (e.g., 4) most important features with their importance values\n", + "global_explanation = client.download_model_explanation()\n", + "global_importance_values = global_explanation.get_ranked_global_values()\n", + "global_importance_names = global_explanation.get_ranked_global_names()" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "global importance values: [0.03999193825435882, 0.031464853252919395, 0.02760060189002182, 0.02456048385916402, 0.02258847868616334, 0.022035631877649273, 0.020207199700294994, 0.020114264275080666, 0.019977203398518766, 0.019378221757655807, 0.0170920545860935, 0.015884334486342123, 0.01586617894132825, 0.013408480313220787, 0.012007829060718718, 0.011751867982559418, 0.011463914137255388, 0.011372582275944364, 0.01032799605886111, 0.010208142977892548, 0.010004060575232718, 0.009948627988387974, 0.008879074315566815, 0.008483158706590357, 0.007304193859418363, 0.0056123067810683784, 0.005469643877212432, 0.005419917839260414, 0.004600788428833444, 0.0044333516100935]\n", + "global importance names: ['OverTime', 'MaritalStatus', 'EducationField', 'JobRole', 'YearsInCurrentRole', 'Age', 'JobSatisfaction', 'EnvironmentSatisfaction', 'RelationshipSatisfaction', 'TrainingTimesLastYear', 'JobInvolvement', 'WorkLifeBalance', 'DistanceFromHome', 'Department', 'NumCompaniesWorked', 'JobLevel', 'StockOptionLevel', 'YearsSinceLastPromotion', 'YearsWithCurrManager', 'MonthlyIncome', 'TotalWorkingYears', 'DailyRate', 'YearsAtCompany', 'BusinessTravel', 'Gender', 'MonthlyRate', 'PerformanceRating', 'PercentSalaryHike', 'HourlyRate', 'Education']\n" + ] + } + ], + "source": [ + "print('global importance values: {}'.format(global_importance_values))\n", + "print('global importance names: {}'.format(global_importance_names))" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'OverTime': 0.03999193825435882, 'MaritalStatus': 0.031464853252919395, 'EducationField': 0.02760060189002182, 'JobRole': 0.02456048385916402, 'YearsInCurrentRole': 0.02258847868616334, 'Age': 0.022035631877649273, 'JobSatisfaction': 0.020207199700294994, 'EnvironmentSatisfaction': 0.020114264275080666, 'RelationshipSatisfaction': 0.019977203398518766, 'TrainingTimesLastYear': 0.019378221757655807, 'JobInvolvement': 0.0170920545860935, 'WorkLifeBalance': 0.015884334486342123, 'DistanceFromHome': 0.01586617894132825, 'Department': 0.013408480313220787, 'NumCompaniesWorked': 0.012007829060718718, 'JobLevel': 0.011751867982559418, 'StockOptionLevel': 0.011463914137255388, 'YearsSinceLastPromotion': 0.011372582275944364, 'YearsWithCurrManager': 0.01032799605886111, 'MonthlyIncome': 0.010208142977892548, 'TotalWorkingYears': 0.010004060575232718, 'DailyRate': 0.009948627988387974, 'YearsAtCompany': 0.008879074315566815, 'BusinessTravel': 0.008483158706590357, 'Gender': 0.007304193859418363, 'MonthlyRate': 0.0056123067810683784, 'PerformanceRating': 0.005469643877212432, 'PercentSalaryHike': 0.005419917839260414, 'HourlyRate': 0.004600788428833444, 'Education': 0.0044333516100935}\n" + ] + } + ], + "source": [ + "print(global_explanation.get_feature_importance_dict())" + ] + } + ], + "metadata": { + "authors": [ + { + "name": "mesameki" + } + ], + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/4-Interpretibility/IBMEmployeeAttritionClassifier_Interpretability.ipynb b/4-Interpretibility/IBMEmployeeAttritionClassifier_Interpretability.ipynb deleted file mode 100644 index 33c3843..0000000 --- a/4-Interpretibility/IBMEmployeeAttritionClassifier_Interpretability.ipynb +++ /dev/null @@ -1,366 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Interpretability With Tensorflow 2.0 On Azure Machine Learning Service\n", - "\n", - "## Overview of Tutorial\n", - "This notebook is Part 4 (Explaining Your Model Using Interpretability) of a four part workshop that demonstrates an end-to-end workflow for using Tensorflow 2.0 on Azure Machine Learning Service. The different components of the workshop are as follows:\n", - "\n", - "- Part 1: [Preparing Data and Model Training](https://github.com/microsoft/bert-stack-overflow/blob/master/1-Training/AzureServiceClassifier_Training.ipynb)\n", - "- Part 2: [Inferencing and Deploying a Model](https://github.com/microsoft/bert-stack-overflow/blob/master/2-Inferencing/AzureServiceClassifier_Inferencing.ipynb)\n", - "- Part 3: [Setting Up a Pipeline Using MLOps](https://github.com/microsoft/bert-stack-overflow/tree/master/3-ML-Ops)\n", - "- Part 4: [Explaining Your Model Interpretability](https://github.com/microsoft/bert-stack-overflow/blob/master/4-Interpretibility/IBMEmployeeAttritionClassifier_Interpretability.ipynb)\n", - "\n", - "**In this specific tutorial, we will cover the following topics:**\n", - "\n", - "- TODO\n", - "- TODO" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## What is Azure Machine Learning Service?\n", - "Azure Machine Learning service is a cloud service that you can use to develop and deploy machine learning models. Using Azure Machine Learning service, you can track your models as you build, train, deploy, and manage them, all at the broad scale that the cloud provides.\n", - "![](./images/aml-overview.png)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## What Is Machine Learning Interpretability?\n", - "Interpretability is the ability to explain why your model made the predictions it did. The Azure Machine Learning service offers various interpretability features to help accomplish this task. These features include:\n", - "\n", - "- Feature importance values for both raw and engineered features.\n", - "- Interpretability on real-world datasets at scale, during training and inference.\n", - "- Interactive visualizations to aid you in the discovery of patterns in data and explanations at training time.\n", - "\n", - "By accurately interpretabiliting your model, it allows you to:\n", - "\n", - "- Use the insights for debugging your model.\n", - "- Validate model behavior matches their objectives.\n", - "- Check for for bias in the model.\n", - "- Build trust in your customers and stakeholders.\n", - "\n", - "![](./images/interpretability-architecture.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Install Azure Machine Learning Python SDK\n", - "\n", - "If you are running this on a Notebook VM, the Azure Machine Learning Python SDK is installed by default. If you are running this locally, you can follow these [instructions](https://docs.microsoft.com/en-us/python/api/overview/azure/ml/install?view=azure-ml-py) to install it using pip.\n", - "\n", - "This tutorial series requires version 1.0.69 or higher. We can import the Python SDK to ensure it has been properly installed:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import azureml.core\n", - "\n", - "print(\"Azure Machine Learning Python SDK version:\", azureml.core.VERSION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Connect To Workspace\n", - "\n", - "Just like in the previous tutorials, we will need to connect to a [workspace](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.workspace(class)?view=azure-ml-py).\n", - "\n", - "The following code will allow you to create a workspace if you don't already have one created. You must have an Azure subscription to create a workspace:\n", - "\n", - "```python\n", - "from azureml.core import Workspace\n", - "ws = Workspace.create(name='myworkspace',\n", - " subscription_id='',\n", - " resource_group='myresourcegroup',\n", - " create_resource_group=True,\n", - " location='eastus2')\n", - "```\n", - "\n", - "**If you are running this on a Notebook VM, you can import the existing workspace.**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azureml.core import Workspace\n", - "\n", - "workspace = Workspace.from_config()\n", - "print('Workspace name: ' + workspace.name, \n", - " 'Azure region: ' + workspace.location, \n", - " 'Subscription id: ' + workspace.subscription_id, \n", - " 'Resource group: ' + workspace.resource_group, sep = '\\n')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> **Note:** that the above commands reads a config.json file that exists by default within the Notebook VM. If you are running this locally or want to use a different workspace, you must add a config file to your project directory. The config file should have the following schema:\n", - "\n", - "```\n", - " {\n", - " \"subscription_id\": \"\",\n", - " \"resource_group\": \"\",\n", - " \"workspace_name\": \"\"\n", - " }\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Interpretability In Training\n", - "We will start by showing how we can interpret our model during training. For this tutorial, we will be using Tensorflow 2.0 to train a basic feed forward neural network on the IBM Employee Attrition Dataset. \n", - "\n", - "**Write this script into a project directory**" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "project_folder = 'ibm-attrition-classifier'" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Train on 1323 samples, validate on 147 samples\n", - "Epoch 1/3\n", - "1323/1323 [==============================] - 0s 132us/sample - loss: 297.3814 - acc: 0.8360 - val_loss: 165.1564 - val_acc: 0.8639\n", - "Epoch 2/3\n", - "1323/1323 [==============================] - 0s 20us/sample - loss: 187.7943 - acc: 0.8360 - val_loss: 88.4600 - val_acc: 0.8639\n", - "Epoch 3/3\n", - "1323/1323 [==============================] - 0s 21us/sample - loss: 96.9134 - acc: 0.7944 - val_loss: 69.0247 - val_acc: 0.7551\n" - ] - } - ], - "source": [ - "%%writefile $project_folder/train.py\n", - "import logging\n", - "import pandas as pd\n", - "import tensorflow as tf\n", - "from absl import flags\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "# Ignore warnings in logs\n", - "logging.getLogger(\"transformers.tokenization_utils\").setLevel(logging.ERROR)\n", - "\n", - "def preprocess_data(data):\n", - " data = pd.read_csv(\"data/emp_attrition.csv\")\n", - "\n", - " # replace binary labels with 1's and 0's\n", - " binary_data = {\n", - " 'Gender': ['Male', 'Female'],\n", - " 'Over18': ['N', 'Y'],\n", - " 'OverTime': ['No', 'Yes'],\n", - " 'Attrition': ['No', 'Yes']\n", - " }\n", - " for k, v in binary_data.items():\n", - " data[k].replace(v, [0, 1], inplace=True)\n", - "\n", - " # Make column labeling consistent, so that 1 indicates True\n", - " data.rename(columns={'Gender': 'IsFemale'}, inplace = True)\n", - "\n", - " # one-hot encode categorical data\n", - " one_hot_cols = ['BusinessTravel', 'Department', 'EducationField', 'JobRole', 'MaritalStatus']\n", - " for col_name in one_hot_cols:\n", - " data = pd.concat([data, pd.get_dummies(data[col_name], drop_first=True)], axis=1)\n", - " data.drop([col_name], axis=1, inplace=True)\n", - " \n", - " # Split data\n", - " train, test = train_test_split(data, test_size=0.1)\n", - " train_y = train.pop('Attrition')\n", - " test_y = test.pop('Attrition')\n", - " \n", - " return train, test, train_y, test_y\n", - "\n", - "# Load data\n", - "raw_data = pd.read_csv(\"data/emp_attrition.csv\")\n", - "train_x, test_x, train_y, test_y = preprocess_data(raw_data)\n", - "\n", - "# Train model\n", - "model = tf.keras.models.Sequential()\n", - "model.add(tf.keras.layers.Dense(units=16, activation='relu', input_shape=(len(train_x.columns),)))\n", - "model.add(tf.keras.layers.Dense(units=1, activation='sigmoid'))\n", - "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n", - "\n", - "# Train neural network\n", - "model.fit(train_x, train_y, epochs=3, verbose=1, batch_size=128, validation_data=(test_x, test_y))\n", - "\n", - "# Save model\n", - "model.save('ibm-attrition-classifier/model.h5')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Run training script**" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Traceback (most recent call last):\n", - " File \"ibm-attrition-classifier/train.py\", line 2, in \n", - " import pandas as pd\n", - "ImportError: No module named pandas\n" - ] - } - ], - "source": [ - "!python $project_folder/train.py" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Load model and perform interpretability**" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING - From /anaconda/envs/azureml_py36/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n", - "WARNING - From /anaconda/envs/azureml_py36/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n" - ] - } - ], - "source": [ - "# TODO: LOAD MODEL AND EXPLAIN IT\n", - "import tensorflow as tf\n", - "\n", - "model = tf.keras.models.load_model('ibm-attrition-classifier/model.h5')\n", - "\n", - "# from azureml.explain.model.tabular_explainer import TabularExplainer\n", - "# # \"features\" and \"classes\" fields are optional\n", - "# explainer = TabularExplainer(network, \n", - "# train)\n", - "\n", - "# # you can use the training data or the test data here\n", - "# global_explanation = explainer.explain_global(x_train)\n", - "\n", - "# # if you used the PFIExplainer in the previous step, use the next line of code instead\n", - "# # global_explanation = explainer.explain_global(x_train, true_labels=y_test)\n", - "\n", - "# # sorted feature importance values and feature names\n", - "# sorted_global_importance_values = global_explanation.get_ranked_global_values()\n", - "# sorted_global_importance_names = global_explanation.get_ranked_global_names()\n", - "# dict(zip(sorted_global_importance_names, sorted_global_importance_values))\n", - "\n", - "# # alternatively, you can print out a dictionary that holds the top K feature names and values\n", - "# global_explanation.get_feature_importance_dict()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Train and Explain Locally\n", - "We will start by training our model locally in the Jupyter Notebook." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Train and Explain Remotely\n", - "Now we will train our model on the compute target created back in the [first tutorial]()." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Interpretability In Inferencing" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Raw Feature Transformations" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Visualizations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.6 - AzureML", - "language": "python", - "name": "python3-azureml" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/4-Interpretibility/train_explain-model.py b/4-Interpretibility/train_explain-model.py new file mode 100644 index 0000000..735479e --- /dev/null +++ b/4-Interpretibility/train_explain-model.py @@ -0,0 +1,146 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +import os +import pandas as pd +import zipfile +from sklearn.model_selection import train_test_split +from sklearn.externals import joblib +from sklearn.preprocessing import StandardScaler, OneHotEncoder +from sklearn.impute import SimpleImputer +from sklearn.pipeline import Pipeline +from sklearn.pipeline import make_pipeline + +from azureml.core.run import Run + +OUTPUT_DIR = './outputs/' +os.makedirs(OUTPUT_DIR, exist_ok=True) + +# get the IBM employee attrition dataset +outdirname = 'dataset.6.21.19' +try: + from urllib import urlretrieve +except ImportError: + from urllib.request import urlretrieve +zipfilename = outdirname + '.zip' +urlretrieve('https://publictestdatasets.blob.core.windows.net/data/' + zipfilename, zipfilename) +with zipfile.ZipFile(zipfilename, 'r') as unzip: + unzip.extractall('.') +attritionData = pd.read_csv('./WA_Fn-UseC_-HR-Employee-Attrition.csv') + +# dropping Employee count as all values are 1 and hence attrition is independent of this feature +attritionData = attritionData.drop(['EmployeeCount'], axis=1) +# dropping Employee Number since it is merely an identifier +attritionData = attritionData.drop(['EmployeeNumber'], axis=1) +attritionData = attritionData.drop(['Over18'], axis=1) +# since all values are 80 +attritionData = attritionData.drop(['StandardHours'], axis=1) + +# converting target variables from string to numerical values +target_map = {'Yes': 1, 'No': 0} +attritionData["Attrition_numerical"] = attritionData["Attrition"].apply(lambda x: target_map[x]) +target = attritionData["Attrition_numerical"] + +attritionXData = attritionData.drop(['Attrition_numerical', 'Attrition'], axis=1) + +# Creating dummy columns for each categorical feature +categorical = [] +for col, value in attritionXData.iteritems(): + if value.dtype == 'object': + categorical.append(col) + +# Store the numerical columns in a list numerical +numerical = attritionXData.columns.difference(categorical) + + +from sklearn.compose import ColumnTransformer + +# We create the preprocessing pipelines for both numeric and categorical data. +numeric_transformer = Pipeline(steps=[ + ('imputer', SimpleImputer(strategy='median')), + ('scaler', StandardScaler())]) + +categorical_transformer = Pipeline(steps=[ + ('imputer', SimpleImputer(strategy='constant', fill_value='missing')), + ('onehot', OneHotEncoder(handle_unknown='ignore'))]) + +preprocess = ColumnTransformer( + transformers=[ + ('num', numeric_transformer, numerical), + ('cat', categorical_transformer, categorical)]) + +pipeline = make_pipeline(preprocess) + + +X_train, X_test, y_train, y_test = train_test_split(attritionXData, + target, + test_size=0.2, + random_state=0, + stratify=target) + +X_train_t = pipeline.fit_transform(X_train) +X_test_t = pipeline.transform(X_test) + +# check tensorflow version +import tensorflow as tf +from distutils.version import StrictVersion + +print(tf.__version__) +# Append classifier to preprocessing pipeline. +# Now we have a full prediction pipeline. + + +network = tf.keras.models.Sequential() +network.add(tf.keras.layers.Dense(units=16, activation='relu', input_shape=(X_train_t.shape[1],))) +network.add(tf.keras.layers.Dense(units=16, activation='relu')) +network.add(tf.keras.layers.Dense(units=1, activation='sigmoid')) + +# Compile neural network +network.compile(loss='binary_crossentropy', # Cross-entropy + optimizer='rmsprop', # Root Mean Square Propagation + metrics=['accuracy']) # Accuracy performance metric + +# Train neural network +history = network.fit(X_train_t, # Features + y_train, # Target vector + epochs=20, # Number of epochs + verbose=1, # Print description after each epoch + batch_size=100, # Number of observations per batch + validation_data=(X_test_t, y_test)) # Data for evaluation + + + + +# You can run the DeepExplainer directly, or run the TabularExplainer which will choose the most appropriate explainer +from interpret.ext.greybox import DeepExplainer +explainer = DeepExplainer(network, + X_train, + features=X_train.columns, + classes=["STAYING", "LEAVING"], + transformations=preprocess, + model_task="classification", + is_classifier=True + ) + +# you can use the training data or the test data here +global_explanation = explainer.explain_global(X_test) +# You can pass a specific data point or a group of data points to the explain_local function +# E.g., Explain the first data point in the test set +instance_num = 1 +local_explanation = explainer.explain_local(X_test[:instance_num]) + + +# get the run this was submitted from to interact with run history +run = Run.get_context() + +from azureml.contrib.interpret.explanation.explanation_client import ExplanationClient + +# create an explanation client to store the explanation (contrib API) +client = ExplanationClient.from_run(run) + +# uploading model explanation data for storage or visualization +comment = 'Global explanation on classification model trained on IBM employee attrition dataset' +client.upload_model_explanation(global_explanation, comment=comment) + +network.save('./outputs/model.h5')