400_deepracer_rl.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section: Workshop Setup\n",
    "\n",
    "### Untar the AWS RoboMaker Simulation Application to the working folder "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Clean the build directory if present\n",
    "!python3 sim_app_bundler.py --clean\n",
    "\n",
    "# # Untar the simapp bundle\n",
    "!python3 sim_app_bundler.py --untar ./deepracer-simapp.tar.gz\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Global Variables and Notebook Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create unique job name, lowercase only.\n",
    "job_name_prefix = 'deepracer-notebook'\n",
    "\n",
    "# Duration of job in seconds (1 hours)\n",
    "job_duration_in_seconds = 3600\n",
    "\n",
    "# Select the Amazon SageMaker Docker instance type\n",
    "#instance_type = \"ml.c4.2xlarge\"\n",
    "instance_type = \"ml.p2.xlarge\" \n",
    "#instance_type = \"ml.c5.4xlarge\"\n",
    "\n",
    "#Output from the CloudFormation stack\n",
    "#Insert output S3BucketName\n",
    "s3_bucket = 'INSERT YOUR S3BucketName HERE'\n",
    "\n",
    "\n",
    "# Change this for multiple rollouts. This will invoke the specified number of robomaker simulation jobs.\n",
    "num_simulation_workers = 3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "import sys\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import subprocess\n",
    "import yaml\n",
    "sys.path.append(\"common\")\n",
    "sys.path.append(\"./src\")\n",
    "from misc import get_execution_role, wait_for_s3_object\n",
    "from docker_utils import build_and_push_docker_image\n",
    "from sagemaker.rl import RLEstimator, RLToolkit, RLFramework\n",
    "from time import gmtime, strftime\n",
    "import time\n",
    "from IPython.display import Markdown\n",
    "from markdown_helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Starting SageMaker session\n",
    "sage_session = sagemaker.session.Session()\n",
    "\n",
    "# AWS Region\n",
    "aws_region = sage_session.boto_region_name\n",
    "if aws_region not in [\"us-west-2\", \"us-east-1\", \"eu-west-1\"]:\n",
    "    raise Exception(\"This notebook uses RoboMaker which is available only in US East (N. Virginia),\"\n",
    "                    \"US West (Oregon) and EU (Ireland). Please switch to one of these regions.\")# S3 bucket\n",
    "\n",
    "\n",
    "\n",
    "# SDK appends the job name and output folder\n",
    "s3_output_path = 's3://{}/'.format(s3_bucket)\n",
    "\n",
    "# Ensure that the S3 prefix contains the keyword 'sagemaker'\n",
    "s3_prefix = job_name_prefix + \"-sagemaker-\" + strftime(\"%y%m%d-%H%M%S\", gmtime())\n",
    "\n",
    "# Get the AWS account id of this account\n",
    "sts = boto3.client(\"sts\")\n",
    "account_id = sts.get_caller_identity()['Account']\n",
    "\n",
    "print(\"Using s3 bucket {}\".format(s3_bucket))\n",
    "print(\"Model checkpoints and other metadata will be stored at: \\ns3://{}/{}\".format(s3_bucket, s3_prefix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ec2 = boto3.client('ec2')\n",
    "\n",
    "deepracer_security_groups = [group[\"GroupId\"] for group in ec2.describe_security_groups()['SecurityGroups']\\\n",
    "                             if group['GroupName'].startswith(\"deepracer-vpc-4323-341\")]\n",
    "\n",
    "deepracer_vpc = [vpc['VpcId'] for vpc in ec2.describe_vpcs()['Vpcs'] \\\n",
    "                if \"Tags\" in vpc for val in vpc['Tags'] \\\n",
    "                if val['Value'] == 'deepracer-vpc-notebook'][0]\n",
    "deepracer_subnets = [subnet[\"SubnetId\"] for subnet in ec2.describe_subnets()[\"Subnets\"] \\\n",
    "                    if subnet[\"VpcId\"] == deepracer_vpc]\n",
    "\n",
    "print(\"Using VPC:\", deepracer_vpc)\n",
    "print(\"Using security group:\", deepracer_security_groups)\n",
    "print(\"Using subnets:\", deepracer_subnets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "source": [
    "![stop](stop.png)  **This Section is complete, please return to the workshop to continue.**\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section: Upload the actions and rewards artifact files\n",
    "\n",
    "### Copy custom files to Amazon S3 bucket so that Amazon SageMaker and AWS RoboMaker can pick them up\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_location = \"s3://%s/%s\" % (s3_bucket, s3_prefix)\n",
    "print(s3_location)\n",
    "\n",
    "# Clean up the previously uploaded files\n",
    "!aws s3 rm --recursive {s3_location}\n",
    "\n",
    "!aws s3 cp ./src/artifacts/rewards/object_avoidance_head_to_head.py {s3_location}/customer_reward_function.py\n",
    "\n",
    "!aws s3 cp ./src/artifacts/actions/stereo_shallow_two_speed_5steering.json {s3_location}/model/model_metadata.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![stop](stop.png) **This Section is complete, please return to the workshop to continue.**\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section: Train the RL Model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build and push Docker image\n",
    "\n",
    "The file ./Dockerfile contains all the packages that are installed into the docker. Instead of using the default sagemaker container. We will be using this docker container. \n",
    "\n",
    "If the docker file is not yet present, this takes about 8 minutes to complete. It takes a few seconds on subsequent runs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!docker rm -f $(docker ps -a -q);\n",
    "!docker rmi -f $(docker images -q);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "from copy_to_sagemaker_container import get_sagemaker_docker, copy_to_sagemaker_container, get_custom_image_name\n",
    "cpu_or_gpu = 'gpu' if instance_type.startswith('ml.p') else 'cpu'\n",
    "# repo name\n",
    "repository_short_name = job_name_prefix + \"-%s\" % cpu_or_gpu\n",
    "custom_image_name = get_custom_image_name(repository_short_name)\n",
    "\n",
    "try:\n",
    "    print(\"Copying files from your notebook to existing sagemaker container\")\n",
    "    sagemaker_docker_id = get_sagemaker_docker(repository_short_name)\n",
    "    copy_to_sagemaker_container(sagemaker_docker_id, repository_short_name)\n",
    "except Exception as e:\n",
    "    print(\"Creating sagemaker container\")\n",
    "    docker_build_args = {\n",
    "        'CPU_OR_GPU': cpu_or_gpu, \n",
    "        'AWS_REGION': boto3.Session().region_name,\n",
    "    }\n",
    "    custom_image_name = build_and_push_docker_image(repository_short_name, build_args=docker_build_args)\n",
    "    print(\"Using ECR image %s\" % custom_image_name)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set the Amazon SageMaker Execution role"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sagemaker_role = sagemaker.get_execution_role()\n",
    "except:\n",
    "    sagemaker_role = get_execution_role('sagemaker')\n",
    "\n",
    "print(\"Using Sagemaker IAM role arn: \\n{}\".format(sagemaker_role))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set algorithm metrics for CloudWatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_definitions = [\n",
    "    # Training> Name=main_level/agent, Worker=0, Episode=19, Total reward=-102.88, Steps=19019, Training iteration=1\n",
    "    {'Name': 'reward-training',\n",
    "     'Regex': '^Training>.*Total reward=(.*?),'},\n",
    "    \n",
    "    # Policy training> Surrogate loss=-0.32664725184440613, KL divergence=7.255815035023261e-06, Entropy=2.83156156539917, training epoch=0, learning_rate=0.00025\n",
    "    {'Name': 'ppo-surrogate-loss',\n",
    "     'Regex': '^Policy training>.*Surrogate loss=(.*?),'},\n",
    "     {'Name': 'ppo-entropy',\n",
    "     'Regex': '^Policy training>.*Entropy=(.*?),'},\n",
    "   \n",
    "    # Testing> Name=main_level/agent, Worker=0, Episode=19, Total reward=1359.12, Steps=20015, Training iteration=2\n",
    "    {'Name': 'reward-testing',\n",
    "     'Regex': '^Testing>.*Total reward=(.*?),'},\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the RL model using the Python SDK Script mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set the RLEstimator parameters\n",
    "estimator = RLEstimator(entry_point=\"training_worker.py\",\n",
    "                        source_dir='src',\n",
    "                        image_name=custom_image_name,\n",
    "                        dependencies=[\"common/\"],\n",
    "                        role=sagemaker_role,\n",
    "                        train_instance_type=instance_type,\n",
    "                        train_instance_count=1,\n",
    "                        output_path=s3_output_path,\n",
    "                        base_job_name=job_name_prefix,\n",
    "                        metric_definitions=metric_definitions,\n",
    "                        train_max_run=job_duration_in_seconds,\n",
    "                        hyperparameters={\n",
    "                            \"s3_bucket\": s3_bucket,\n",
    "                            \"s3_prefix\": s3_prefix,\n",
    "                            \"aws_region\": aws_region,\n",
    "                            \"model_metadata_s3_key\": \"%s/model/model_metadata.json\" % s3_prefix,\n",
    "                            \"reward_function_s3_source\": \"%s/customer_reward_function.py\" % s3_prefix,\n",
    "                            \"batch_size\": \"64\",\n",
    "                            \"num_epochs\": \"10\",\n",
    "                            \"stack_size\": \"1\",\n",
    "                            \"lr\": \"0.0003\",\n",
    "                            \"exploration_type\": \"Categorical\",\n",
    "                            \"e_greedy_value\": \"1\",\n",
    "                            \"epsilon_steps\": \"10000\",\n",
    "                            \"beta_entropy\": \"0.01\",\n",
    "                            \"discount_factor\": \"0.999\",\n",
    "                            \"loss_type\": \"Huber\",\n",
    "                            \"num_episodes_between_training\": \"20\",\n",
    "                            \"max_sample_count\": \"0\",\n",
    "                            \"sampling_frequency\": \"1\"\n",
    "                        },\n",
    "                        subnets=deepracer_subnets,\n",
    "                        security_group_ids=deepracer_security_groups,\n",
    "                    )\n",
    "\n",
    "estimator.fit(wait=False)\n",
    "job_name = estimator.latest_training_job.job_name\n",
    "training_job_arn = estimator.latest_training_job.describe()['TrainingJobArn']\n",
    "print(\"Training job: %s\" % job_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![stop](stop.png) **This Section is complete, please return to the workshop to continue.**\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section: Start the AWS Robomaker Simulation for Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the Amazon Kinesis video stream (optional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kvs_stream_name = \"dr-kvs-{}\".format(job_name)\n",
    "\n",
    "!aws --region {aws_region} kinesisvideo create-stream --stream-name {kvs_stream_name} --media-type video/h264 --data-retention-in-hours 24\n",
    "print (\"Created kinesis video stream {}\".format(kvs_stream_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the AWS RoboMaker Simulation Application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "robomaker = boto3.client(\"robomaker\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "robomaker_s3_key = 'robomaker/simulation_ws.tar.gz'\n",
    "robomaker_source = {'s3Bucket': s3_bucket,\n",
    "                    's3Key': robomaker_s3_key,\n",
    "                    'architecture': \"X86_64\"}\n",
    "simulation_software_suite={'name': 'Gazebo',\n",
    "                           'version': '7'}\n",
    "robot_software_suite={'name': 'ROS',\n",
    "                      'version': 'Kinetic'}\n",
    "rendering_engine={'name': 'OGRE',\n",
    "                  'version': '1.x'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bundle the AWS RoboMaker Simulation Application\n",
    "\n",
    "If you ***have*** made changes to the simulation application assets, re-bundle it using the Python file sim_app_bundler.py. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rsync -av ./src/markov/ ./build/simapp/bundle/opt/install/sagemaker_rl_agent/lib/python3.5/site-packages/markov\n",
    "#!python3 sim_app_bundler.py --tar /bundle/opt/install/sagemaker_rl_agent/lib/python3.5/site-packages/markov"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you ***have not*** made any changes to files in the markov folder that need to run in the simapp, run the below cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "!python3 sim_app_bundler.py --tar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Upload the AWS RoboMaker Simulation Application to an Amazon S3 bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('./build/output.tar.gz'):\n",
    "    print(\"Using the latest simapp from public s3 bucket\")\n",
    "    # Download Robomaker simApp for the deepracer public s3 bucket\n",
    "    simulation_application_bundle_location = \"s3://deepracer-managed-resources-us-east-1/deepracer-simapp.tar.gz\"\n",
    "    !aws s3 cp {simulation_application_bundle_location} ./\n",
    "\n",
    "    # Remove if the Robomaker sim-app is present in s3 bucket\n",
    "    !aws s3 rm s3://{s3_bucket}/{robomaker_s3_key}\n",
    "\n",
    "    # Uploading the Robomaker SimApp to your S3 bucket\n",
    "    !aws s3 cp ./deepracer-simapp.tar.gz s3://{s3_bucket}/{robomaker_s3_key}\n",
    "\n",
    "    # Cleanup the locally downloaded version of SimApp\n",
    "    !rm deepracer-simapp.tar.gz\n",
    "else:\n",
    "    print(\"Using the simapp from build directory\")\n",
    "    !aws s3 cp ./build/output.tar.gz s3://{s3_bucket}/{robomaker_s3_key}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create ARN for the AWS RoboMaker Simulation Application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "app_name = \"deepracer-notebook-application\" + strftime(\"%y%m%d-%H%M%S\", gmtime())\n",
    "\n",
    "print(app_name)\n",
    "try:\n",
    "    response = robomaker.create_simulation_application(name=app_name,\n",
    "                                                       sources=[robomaker_source],\n",
    "                                                       simulationSoftwareSuite=simulation_software_suite,\n",
    "                                                       robotSoftwareSuite=robot_software_suite,\n",
    "                                                       renderingEngine=rendering_engine)\n",
    "    simulation_app_arn = response[\"arn\"]\n",
    "    print(\"Created a new simulation app with ARN:\", simulation_app_arn)\n",
    "except Exception as e:\n",
    "    if \"AccessDeniedException\" in str(e):\n",
    "        display(Markdown(generate_help_for_robomaker_all_permissions(role)))\n",
    "        raise e\n",
    "    else:\n",
    "        raise e"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set the AWS RoboMaker Simulation Job parameters and start the training Simulation Jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./src/artifacts/yaml/training_yaml_template.yaml\", \"r\") as filepointer:\n",
    "    yaml_config = yaml.load(filepointer)\n",
    "\n",
    "#world_name is the track pattern you wish to use\n",
    "\n",
    "world_name = \"Spain_track\"\n",
    " \n",
    "# Race-type supported for training are TIME_TRIAL, OBJECT_AVOIDANCE, HEAD_TO_BOT\n",
    "# If you need to modify more attributes look at the template yaml file\n",
    "# Default is TIME_TRIAL\n",
    "\n",
    "race_type = \"OBJECT_AVOIDANCE\"\n",
    "\n",
    "if race_type == \"OBJECT_AVOIDANCE\":\n",
    "    yaml_config['NUMBER_OF_OBSTACLES']     = \"6\"\n",
    "    yaml_config['RACE_TYPE']               = \"OBJECT_AVOIDANCE\"\n",
    "\n",
    "elif race_type == \"HEAD_TO_BOT\":\n",
    "    yaml_config['NUMBER_OF_BOT_CARS']      = \"6\"\n",
    "    yaml_config['RACE_TYPE']               = \"HEAD_TO_BOT\"\n",
    "\n",
    "\n",
    "yaml_config['WORLD_NAME']                  = world_name\n",
    "yaml_config['SAGEMAKER_SHARED_S3_BUCKET']  = s3_bucket\n",
    "yaml_config['SAGEMAKER_SHARED_S3_PREFIX']  = s3_prefix\n",
    "yaml_config['TRAINING_JOB_ARN']            = training_job_arn\n",
    "yaml_config['METRICS_S3_BUCKET']           = s3_bucket\n",
    "yaml_config['METRICS_S3_OBJECT_KEY']       = \"{}/training_metrics.json\".format(s3_prefix)\n",
    "yaml_config['SIMTRACE_S3_BUCKET']          = s3_bucket\n",
    "yaml_config['SIMTRACE_S3_PREFIX']          = \"{}/iteration-data/training\".format(s3_prefix)\n",
    "yaml_config['AWS_REGION']                  = aws_region\n",
    "yaml_config['ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID'] = account_id\n",
    "yaml_config['KINESIS_VIDEO_STREAM_NAME']   = kvs_stream_name\n",
    "yaml_config['REWARD_FILE_S3_KEY']          = \"{}/customer_reward_function.py\".format(s3_prefix)\n",
    "yaml_config['MODEL_METADATA_FILE_S3_KEY']  = \"{}/model/model_metadata.json\".format(s3_prefix)\n",
    "yaml_config['NUM_WORKERS']                 = num_simulation_workers\n",
    "yaml_config['MP4_S3_BUCKET']               = s3_bucket\n",
    "yaml_config['MP4_S3_OBJECT_PREFIX']        = \"{}/iteration-data/training\".format(s3_prefix)\n",
    "\n",
    "\n",
    "# Printing the modified yaml parameter\n",
    "for key, value in yaml_config.items():\n",
    "    print(\"{}: {}\".format(key.ljust(40, ' '), value))\n",
    "\n",
    "# Uploading the modified yaml parameter\n",
    "with open(\"./training_params.yaml\", \"w\") as filepointer:\n",
    "    yaml.dump(yaml_config, filepointer)\n",
    "    \n",
    "s3_yaml_name=\"training_params.yaml\"\n",
    "!aws s3 cp ./training_params.yaml {s3_location}/training_params.yaml\n",
    "!rm training_params.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vpcConfig = {\"subnets\": deepracer_subnets,\n",
    "             \"securityGroups\": deepracer_security_groups,\n",
    "             \"assignPublicIp\": True}\n",
    "\n",
    "responses = []\n",
    "for job_no in range(num_simulation_workers):\n",
    "    client_request_token = strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n",
    "    envriron_vars = {\n",
    "        \"S3_YAML_NAME\": s3_yaml_name,\n",
    "        \"SAGEMAKER_SHARED_S3_PREFIX\": s3_prefix,\n",
    "        \"SAGEMAKER_SHARED_S3_BUCKET\": s3_bucket,\n",
    "        \"WORLD_NAME\": world_name,\n",
    "        \"KINESIS_VIDEO_STREAM_NAME\": kvs_stream_name,\n",
    "        \"APP_REGION\": aws_region,\n",
    "        \"MODEL_METADATA_FILE_S3_KEY\": \"%s/model/model_metadata.json\" % s3_prefix,\n",
    "        \"ROLLOUT_IDX\": str(job_no)\n",
    "    }\n",
    "\n",
    "    simulation_application = {\"application\":simulation_app_arn,\n",
    "                              \"launchConfig\": {\"packageName\": \"deepracer_simulation_environment\",\n",
    "                                               \"launchFile\": \"distributed_training.launch\",\n",
    "                                               \"environmentVariables\": envriron_vars}\n",
    "                             }\n",
    "    response =  robomaker.create_simulation_job(iamRole=sagemaker_role,\n",
    "                                            clientRequestToken=client_request_token,\n",
    "                                            maxJobDurationInSeconds=job_duration_in_seconds,\n",
    "                                            failureBehavior=\"Fail\",\n",
    "                                            simulationApplications=[simulation_application],\n",
    "                                            vpcConfig=vpcConfig\n",
    "                                            )\n",
    "    responses.append(response)\n",
    "    time.sleep(5)\n",
    "    \n",
    "\n",
    "print(\"Created the following jobs:\")\n",
    "job_arns = [response[\"arn\"] for response in responses]\n",
    "for job_arn in job_arns:\n",
    "    print(\"Job ARN\", job_arn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot metrics for training job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_dir = \"/tmp/{}\".format(job_name)\n",
    "os.system(\"mkdir {}\".format(tmp_dir))\n",
    "print(\"Create local folder {}\".format(tmp_dir))\n",
    "\n",
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "training_metrics_file = \"training_metrics.json\"\n",
    "training_metrics_path = \"{}/{}\".format(s3_prefix, training_metrics_file)\n",
    "wait_for_s3_object(s3_bucket, training_metrics_path, tmp_dir)\n",
    "\n",
    "json_file = \"{}/{}\".format(tmp_dir, training_metrics_file)\n",
    "with open(json_file) as fp:  \n",
    "    data = json.load(fp)\n",
    "\n",
    "df = pd.DataFrame(data['metrics'])\n",
    "x_axis = 'episode'\n",
    "y_axis = 'reward_score'\n",
    "\n",
    "plt = df.plot(x=x_axis,y=y_axis, figsize=(12,5), legend=True, style='b-')\n",
    "plt.set_ylabel(y_axis);\n",
    "plt.set_xlabel(x_axis);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![stop](stop.png) **This Section is complete, please return to the workshop to continue.**\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section: Start the AWS Robomaker Simulation for Model Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set the AWS RoboMaker Simulation Job parameters and start the evaluation Simulation Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./src/artifacts/yaml/evaluation_yaml_template.yaml\", \"r\") as filepointer:\n",
    "    yaml_config = yaml.load(filepointer)\n",
    "\n",
    "#world_name is the track pattern you wish to use\n",
    "\n",
    "world_name = \"Spain_track\"\n",
    "\n",
    "# Race-type supported for evaluation are TIME_TRIAL, OBJECT_AVOIDANCE, HEAD_TO_BOT\n",
    "# If you need to modify more attributes look at the template yaml file\n",
    "race_type = \"OBJECT_AVOIDANCE\"\n",
    "\n",
    "if race_type == \"OBJECT_AVOIDANCE\":\n",
    "    yaml_config['NUMBER_OF_OBSTACLES']     = \"6\"\n",
    "    yaml_config['RACE_TYPE']               = \"OBJECT_AVOIDANCE\"\n",
    "\n",
    "elif race_type == \"HEAD_TO_BOT\":\n",
    "    yaml_config['NUMBER_OF_BOT_CARS']      = \"6\"\n",
    "    yaml_config['RACE_TYPE']               = \"HEAD_TO_BOT\"\n",
    "\n",
    "yaml_config['WORLD_NAME']                  = world_name\n",
    "yaml_config['MODEL_S3_BUCKET']             = s3_bucket\n",
    "yaml_config['MODEL_S3_PREFIX']             = s3_prefix\n",
    "yaml_config['AWS_REGION']                  = aws_region\n",
    "yaml_config['METRICS_S3_BUCKET']           = s3_bucket\n",
    "yaml_config['METRICS_S3_OBJECT_KEY']       = \"{}/evaluation_metrics.json\".format(s3_prefix)\n",
    "yaml_config['SIMTRACE_S3_BUCKET']          = s3_bucket\n",
    "yaml_config['SIMTRACE_S3_PREFIX']          = \"{}/iteration-data/evaluation\".format(s3_prefix)\n",
    "yaml_config['ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID'] = account_id\n",
    "yaml_config['NUMBER_OF_TRIALS']            = \"5\"\n",
    "yaml_config['MP4_S3_BUCKET']               = s3_bucket\n",
    "yaml_config['MP4_S3_OBJECT_PREFIX']        = \"{}/iteration-data/evaluation\".format(s3_prefix)\n",
    "yaml_config['DISPLAY_NAME']                = \"Racer Display Name\"\n",
    "yaml_config['RACER_NAME']                  = \"Racer Name\"\n",
    "yaml_config['MODEL_NAME']                  = \"Model Name\"\n",
    "yaml_config['LEADERBOARD_TYPE']            = \"LEAGUE\"\n",
    "yaml_config['LEADERBOARD_NAME']            = \"Leaderboard Name\"\n",
    "yaml_config['CAR_COLOR']                   = \"Grey\"\n",
    "yaml_config['NUMBER_OF_RESETS']            = \"10000\"\n",
    "yaml_config['PENALTY_SECONDS']             = \"5.0\"\n",
    "yaml_config['OFF_TRACK_PENALTY']           = \"5.0\"\n",
    "yaml_config['COLLISION_PENALTY']           = \"5.0\"\n",
    "\n",
    "\n",
    "# Printing the modified yaml parameter\n",
    "for key, value in yaml_config.items():\n",
    "    print(\"{}: {}\".format(key.ljust(40, ' '), value))\n",
    "\n",
    "# Uploading the modified yaml parameter\n",
    "with open(\"./evaluation_params.yaml\", \"w\") as filepointer:\n",
    "    yaml.dump(yaml_config, filepointer)\n",
    "\n",
    "s3_yaml_name=\"evaluation_params.yaml\"\n",
    "!aws s3 cp ./evaluation_params.yaml {s3_location}/evaluation_params.yaml\n",
    "!rm evaluation_params.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_simulation_workers = 1\n",
    "\n",
    "envriron_vars = {\n",
    "    \"S3_YAML_NAME\": s3_yaml_name,\n",
    "    \"MODEL_S3_PREFIX\": s3_prefix,\n",
    "    \"MODEL_S3_BUCKET\": s3_bucket,\n",
    "    \"WORLD_NAME\": world_name,\n",
    "    \"KINESIS_VIDEO_STREAM_NAME\": kvs_stream_name,\n",
    "    \"APP_REGION\": aws_region,\n",
    "    \"MODEL_METADATA_FILE_S3_KEY\": \"%s/model/model_metadata.json\" % s3_prefix\n",
    "}\n",
    "\n",
    "simulation_application = {\n",
    "    \"application\":simulation_app_arn,\n",
    "    \"launchConfig\": {\n",
    "         \"packageName\": \"deepracer_simulation_environment\",\n",
    "         \"launchFile\": \"evaluation.launch\",\n",
    "         \"environmentVariables\": envriron_vars\n",
    "    }\n",
    "}\n",
    "                            \n",
    "vpcConfig = {\"subnets\": deepracer_subnets,\n",
    "             \"securityGroups\": deepracer_security_groups,\n",
    "             \"assignPublicIp\": True}\n",
    "\n",
    "responses = []\n",
    "for job_no in range(num_simulation_workers):\n",
    "    response =  robomaker.create_simulation_job(clientRequestToken=strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime()),\n",
    "                                                outputLocation={ \n",
    "                                                  \"s3Bucket\": s3_bucket,\n",
    "                                                  \"s3Prefix\": s3_prefix\n",
    "                                                },\n",
    "                                                maxJobDurationInSeconds=job_duration_in_seconds,\n",
    "                                                iamRole=sagemaker_role,\n",
    "                                                failureBehavior=\"Fail\",\n",
    "                                                simulationApplications=[simulation_application],\n",
    "                                                vpcConfig=vpcConfig)\n",
    "    responses.append(response)\n",
    "\n",
    "print(\"Created the following jobs:\")\n",
    "job_arns = [response[\"arn\"] for response in responses]\n",
    "for job_arn in job_arns:\n",
    "    print(\"Job ARN\", job_arn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot metrics for evaluation job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluation_metrics_file = \"evaluation_metrics.json\"\n",
    "evaluation_metrics_path = \"{}/{}\".format(s3_prefix, evaluation_metrics_file)\n",
    "wait_for_s3_object(s3_bucket, evaluation_metrics_path, tmp_dir)\n",
    "\n",
    "json_file = \"{}/{}\".format(tmp_dir, evaluation_metrics_file)\n",
    "with open(json_file) as fp:  \n",
    "    data = json.load(fp)\n",
    "\n",
    "df_1 = pd.DataFrame(data['metrics'])\n",
    "# Converting milliseconds to seconds\n",
    "df_1['elapsed_time'] = df_1['elapsed_time_in_milliseconds']/1000\n",
    "df_1 = df_1[['trial', 'completion_percentage', 'elapsed_time']]\n",
    "\n",
    "display(df_1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![stop](stop.png) **This Section is complete, please return to the workshop to continue.**\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2.7.16 64-bit",
   "language": "python",
   "name": "python271664bitd6cc66dd4c1845728b82216980b36753"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "2.7.16-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}