deepracer-lidar-tf12-adv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prequisites"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create new sim app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cp ./src/lib/level_manager.py ./build/simapp/bundle/usr/local/lib/python3.5/dist-packages/rl_coach/level_manager.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Only when you have changes to SimApp\n",
    "# !python3 sim_app_bundler.py --tar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Customize environment, reward, preset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#experiment_name = 'lidar-simple-5CNN-2Dense-Narrow_3Cars'\n",
    "#experiment_name = 'lidar-advanced-3CNN-1Dense-Wide_3Cars'\n",
    "#experiment_name = 'lidar-advanced-3CNN-2Dense-Narrow_3Cars'\n",
    "#experiment_name = 'lidar-advanced-5CNN-1Dense-Wide_3Cars'\n",
    "experiment_name = 'lidar-advanced-5CNN-2Dense-Narrow_5Cars_run4'\n",
    "\n",
    "\n",
    "#reward\n",
    "if 'advanced' in experiment_name:\n",
    "    !cp customize/reward_advanced.py src/markov/rewards/reward_function.py\n",
    "if 'simple' in experiment_name:\n",
    "    !cp customize/reward_simple.py src/markov/rewards/reward_function.py\n",
    "\n",
    "# preset\n",
    "if '5CNN' in experiment_name and 'Narrow' in experiment_name:\n",
    "    !cp customize/preset_5CNN_2Dense_narrow.py src/markov/presets/preset.py\n",
    "if '5CNN' in experiment_name and 'Wide' in experiment_name:\n",
    "    !cp customize/preset_5CNN_1Dense_wide.py src/markov/presets/preset.py\n",
    "\n",
    "if '3CNN' in experiment_name and 'Narrow' in experiment_name:\n",
    "    !cp customize/preset_3CNN_2Dense_narrow.py src/markov/presets/preset.py\n",
    "if '3CNN' in experiment_name and 'Wide' in experiment_name:\n",
    "    !cp customize/preset_3CNN_1Dense_wide.py src/markov/presets/preset.py\n",
    "\n",
    "# the environment file\n",
    "if '5Cars' in experiment_name:\n",
    "    !cp customize/deepracer_racetrack_env_lidar_5cars.py src/markov/environments/deepracer_racetrack_env.py\n",
    "if '3Cars' in experiment_name:\n",
    "    !cp customize/deepracer_racetrack_env_lidar_3cars.py src/markov/environments/deepracer_racetrack_env.py\n",
    "    \n",
    "#!cp customize/deepracer_racetrack_env_shallow_advanced-lidar-turns.py src/markov/environments/deepracer_racetrack_env.py\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "import sys\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import subprocess\n",
    "sys.path.append(\"common\")\n",
    "from misc import get_execution_role, wait_for_s3_object\n",
    "from docker_utils import build_and_push_docker_image\n",
    "from sagemaker.rl import RLEstimator, RLToolkit, RLFramework\n",
    "from time import gmtime, strftime\n",
    "import time\n",
    "from IPython.display import Markdown\n",
    "from markdown_helper import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initializing basic parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select the instance type\n",
    "instance_type = \"ml.p3.16xlarge\"\n",
    "#instance_type = \"ml.p2.xlarge\"\n",
    "# instance_type = \"ml.c5.4xlarge\"\n",
    "\n",
    "# Starting SageMaker session\n",
    "sage_session = sagemaker.session.Session()\n",
    "\n",
    "# stick to 1,2,4,8\n",
    "# num_simulation_workers = 1\n",
    "\n",
    "# Create unique job name.\n",
    "job_name_prefix = 'sahika-lidar-oldcar-tf12-wide-ic'\n",
    "\n",
    "# Duration of job in seconds (1 hours)\n",
    "job_duration_in_seconds = 60 * 60 * 20\n",
    "\n",
    "# AWS Region\n",
    "aws_region = sage_session.boto_region_name\n",
    "if aws_region not in [\"us-west-2\", \"us-east-1\", \"eu-west-1\"]:\n",
    "    raise Exception(\"This notebook uses RoboMaker which is available only in US East (N. Virginia),\"\n",
    "                    \"US West (Oregon) and EU (Ireland). Please switch to one of these regions.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup S3 bucket\n",
    "Set up the linkage and authentication to the S3 bucket that we want to use for checkpoint and metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using s3 bucket sahika-lidar-oldcar-tf12-wide-ic\n",
      "Model checkpoints and other metadata will be stored at: \n",
      "s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456\n"
     ]
    }
   ],
   "source": [
    "# S3 bucket\n",
    "s3_bucket = job_name_prefix #\"sahika-multicar-stereo-camera\"\n",
    "#s3_bucket = \"sahika-multicar-lidar-small-angle\"\n",
    "\n",
    "# SDK appends the job name and output folder\n",
    "s3_output_path = 's3://{}/'.format(s3_bucket)\n",
    "\n",
    "#Ensure that the S3 prefix contains the keyword 'sagemaker'\n",
    "s3_prefix = job_name_prefix + \"-sagemaker-\" + strftime(\"%y%m%d-%H%M%S\", gmtime())\n",
    "\n",
    "# Get the AWS account id of this account\n",
    "sts = boto3.client(\"sts\")\n",
    "account_id = sts.get_caller_identity()['Account']\n",
    "\n",
    "print(\"Using s3 bucket {}\".format(s3_bucket))\n",
    "print(\"Model checkpoints and other metadata will be stored at: \\ns3://{}/{}\".format(s3_bucket, s3_prefix))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create an IAM role\n",
    "Either get the execution role when running from a SageMaker notebook `role = sagemaker.get_execution_role()` or, when running from local machine, use utils method `role = get_execution_role('role_name')` to create an execution role."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using Sagemaker IAM role arn: \n",
      "arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sagemaker_role = sagemaker.get_execution_role()\n",
    "except:\n",
    "    sagemaker_role = get_execution_role('sagemaker')\n",
    "\n",
    "print(\"Using Sagemaker IAM role arn: \\n{}\".format(sagemaker_role))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Please note that this notebook cannot be run in `SageMaker local mode` as the simulator is based on AWS RoboMaker service."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Permission setup for invoking AWS RoboMaker from this notebook\n",
    "In order to enable this notebook to be able to execute AWS RoboMaker jobs, we need to add one trust relationship to the default execution role of this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# display(Markdown(generate_help_for_robomaker_trust_relationship(sagemaker_role)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Permission setup for Sagemaker to S3 bucket\n",
    "\n",
    "The sagemaker writes the Redis IP address, models to the S3 bucket. This requires PutObject permission on the bucket. Make sure the sagemaker role you are using as this permissions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# display(Markdown(generate_s3_write_permission_for_sagemaker_role(sagemaker_role)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build and push docker image\n",
    "\n",
    "The file ./Dockerfile contains all the packages that are installed into the docker. Instead of using the default sagemaker container. We will be using this docker container."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying files from your notebook to existing sagemaker container\n",
      "docker images sahika-lidar-oldcar-tf12-wide-ic-gpu | sed -n 2,2p\n",
      "Sagemaker docker id : 7acfdc6f0522\n",
      "docker run -d -t 7acfdc6f0522\n",
      "docker exec -d 88c4762bce9659b776568fc282c11b42551f655c1420e2e28c858216f2619a5f rm -rf /opt/amazon/markov\n",
      "docker cp ./src/markov 88c4762bce9659b776568fc282c11b42551f655c1420e2e28c858216f2619a5f:/opt/amazon/markov\n",
      "============ Copied Markov scripts to sagemaker docker ============ \n",
      " \n",
      "docker ps -l|sed -n 2,2p\n",
      "docker commit 88c4762bce96 sahika-lidar-oldcar-tf12-wide-ic-gpu\n",
      "============ Commited all the changes to docker ============ \n",
      " \n",
      "CPU times: user 29.5 ms, sys: 27.9 ms, total: 57.4 ms\n",
      "Wall time: 1.41 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from copy_to_sagemaker_container import get_sagemaker_docker, copy_to_sagemaker_container, get_custom_image_name\n",
    "cpu_or_gpu = 'gpu' if instance_type.startswith('ml.p') else 'cpu'\n",
    "# repo name\n",
    "repository_short_name = job_name_prefix + \"-%s\" % cpu_or_gpu\n",
    "custom_image_name = get_custom_image_name(repository_short_name)\n",
    "\n",
    "try:\n",
    "    print(\"Copying files from your notebook to existing sagemaker container\")\n",
    "    sagemaker_docker_id = get_sagemaker_docker(repository_short_name)\n",
    "    copy_to_sagemaker_container(sagemaker_docker_id, repository_short_name)\n",
    "except Exception as e:\n",
    "    print(\"Creating sagemaker container\")\n",
    "    docker_build_args = {\n",
    "        'CPU_OR_GPU': cpu_or_gpu, \n",
    "        'AWS_REGION': boto3.Session().region_name,\n",
    "    }\n",
    "    custom_image_name = build_and_push_docker_image(repository_short_name, build_args=docker_build_args)\n",
    "    print(\"Using ECR image %s\" % custom_image_name)\n",
    "    \n",
    "# print(\"Creating sagemaker container\")\n",
    "# docker_build_args = {\n",
    "#     'CPU_OR_GPU': cpu_or_gpu, \n",
    "#     'AWS_REGION': boto3.Session().region_name,\n",
    "# }\n",
    "# custom_image_name = build_and_push_docker_image(repository_short_name, build_args=docker_build_args)\n",
    "# print(\"Using ECR image %s\" % custom_image_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using ECR image 828753602420.dkr.ecr.us-west-2.amazonaws.com/sahika-lidar-oldcar-tf12-wide-ic-gpu\n"
     ]
    }
   ],
   "source": [
    "print(\"Using ECR image %s\" % custom_image_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configure VPC\n",
    "\n",
    "Since SageMaker and RoboMaker have to communicate with each other over the network, both of these services need to run in VPC mode. This can be done by supplying subnets and security groups to the job launching scripts.  \n",
    "We will check if the deepracer-vpc stack is created and use it if present (This is present if the AWS Deepracer console is used atleast once to create a model). Else we will use the default VPC stack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using the default VPC stacks\n",
      "Using VPC: vpc-b418facc\n",
      "Using security group: ['sg-fca8088c']\n",
      "Using subnets: ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd']\n"
     ]
    }
   ],
   "source": [
    "ec2 = boto3.client('ec2')\n",
    "\n",
    "#\n",
    "# Check if the user has Deepracer-VPC and use that if its present. This will have all permission.\n",
    "# This VPC will be created when you have used the Deepracer console and created one model atleast\n",
    "# If this is not present. Use the default VPC connnection\n",
    "#\n",
    "deepracer_security_groups = [group[\"GroupId\"] for group in ec2.describe_security_groups()['SecurityGroups']\\\n",
    "                             if group['GroupName'].startswith(\"deepracer-vpc\")]\n",
    "if(deepracer_security_groups):\n",
    "    print(\"Using the DeepRacer VPC stacks\")\n",
    "    deepracer_vpc = [vpc['VpcId'] for vpc in ec2.describe_vpcs()['Vpcs'] \\\n",
    "                     if \"Tags\" in vpc for val in vpc['Tags'] \\\n",
    "                     if val['Value'] == 'deepracer-vpc'][0]\n",
    "    deepracer_subnets = [subnet[\"SubnetId\"] for subnet in ec2.describe_subnets()[\"Subnets\"] \\\n",
    "                         if subnet[\"VpcId\"] == deepracer_vpc]\n",
    "else:\n",
    "    print(\"Using the default VPC stacks\")\n",
    "    deepracer_vpc = [vpc['VpcId'] for vpc in ec2.describe_vpcs()['Vpcs'] if vpc[\"IsDefault\"] == True][0]\n",
    "\n",
    "    deepracer_security_groups = [group[\"GroupId\"] for group in ec2.describe_security_groups()['SecurityGroups'] \\\n",
    "                                 if 'VpcId' in group and group[\"GroupName\"] == \"default\" and group[\"VpcId\"] == deepracer_vpc]\n",
    "\n",
    "    deepracer_subnets = [subnet[\"SubnetId\"] for subnet in ec2.describe_subnets()[\"Subnets\"] \\\n",
    "                         if subnet[\"VpcId\"] == deepracer_vpc and subnet['DefaultForAz']==True]\n",
    "\n",
    "print(\"Using VPC:\", deepracer_vpc)\n",
    "print(\"Using security group:\", deepracer_security_groups)\n",
    "print(\"Using subnets:\", deepracer_subnets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Route Table\n",
    "A SageMaker job running in VPC mode cannot access S3 resourcs. So, we need to create a VPC S3 endpoint to allow S3 access from SageMaker container. To learn more about the VPC mode, please visit [this link.](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating \n",
      "Trying to attach S3 endpoints to the following route tables: ['rtb-0d549376']\n",
      "S3 endpoint already exists.\n"
     ]
    }
   ],
   "source": [
    "#TODO: Explain to customer what CREATE_ROUTE_TABLE is doing\n",
    "CREATE_ROUTE_TABLE = True\n",
    "\n",
    "def create_vpc_endpoint_table():\n",
    "    print(\"Creating \")\n",
    "    try:\n",
    "        route_tables = [route_table[\"RouteTableId\"] for route_table in ec2.describe_route_tables()['RouteTables']\\\n",
    "                        if route_table['VpcId'] == deepracer_vpc]\n",
    "    except Exception as e:\n",
    "        if \"UnauthorizedOperation\" in str(e):\n",
    "            display(Markdown(generate_help_for_s3_endpoint_permissions(sagemaker_role)))\n",
    "        else:\n",
    "            display(Markdown(create_s3_endpoint_manually(aws_region, deepracer_vpc)))\n",
    "        raise e\n",
    "\n",
    "    print(\"Trying to attach S3 endpoints to the following route tables:\", route_tables)\n",
    "    \n",
    "    if not route_tables:\n",
    "        raise Exception((\"No route tables were found. Please follow the VPC S3 endpoint creation \"\n",
    "                         \"guide by clicking the above link.\"))\n",
    "    try:\n",
    "        ec2.create_vpc_endpoint(DryRun=False,\n",
    "                                VpcEndpointType=\"Gateway\",\n",
    "                                VpcId=deepracer_vpc,\n",
    "                                ServiceName=\"com.amazonaws.{}.s3\".format(aws_region),\n",
    "                                RouteTableIds=route_tables)\n",
    "        print(\"S3 endpoint created successfully!\")\n",
    "    except Exception as e:\n",
    "        if \"RouteAlreadyExists\" in str(e):\n",
    "            print(\"S3 endpoint already exists.\")\n",
    "        elif \"UnauthorizedOperation\" in str(e):\n",
    "            display(Markdown(generate_help_for_s3_endpoint_permissions(role)))\n",
    "            raise e\n",
    "        else:\n",
    "            display(Markdown(create_s3_endpoint_manually(aws_region, default_vpc)))\n",
    "            raise e\n",
    "\n",
    "if CREATE_ROUTE_TABLE:\n",
    "    create_vpc_endpoint_table()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup the environment\n",
    "\n",
    "The environment is defined in a Python file called “deepracer_racetrack_env.py” and the file can be found at `src/markov/environments/`. This file implements the gym interface for our Gazebo based RoboMakersimulator. This is a common environment file used by both SageMaker and RoboMaker. The environment variable - `NODE_TYPE` defines which node the code is running on. So, the expressions that have `rospy` dependencies are executed on RoboMaker only.  \n",
    "\n",
    "We can experiment with different reward functions by modifying `reward_function` in `src/markov/rewards/`. Action space and steering angles can be changed by modifying `src/markov/actions/`.json file\n",
    "\n",
    "### Configure the preset for RL algorithm\n",
    "\n",
    "The parameters that configure the RL training job are defined in `src/markov/presets/`. Using the preset file, you can define agent parameters to select the specific agent algorithm. We suggest using Clipped PPO for this example.  \n",
    "You can edit this file to modify algorithm parameters like learning_rate, neural network structure, batch_size, discount factor etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment the pygmentize code lines to see the code\n",
    "\n",
    "# Environmental File\n",
    "#!pygmentize src/markov/environments/deepracer_racetrack_env.py\n",
    "\n",
    "# Reward function\n",
    "#!pygmentize src/markov/rewards/default.py\n",
    "\n",
    "# Action space\n",
    "#!pygmentize src/markov/actions/model_metadata_10_state.json\n",
    "\n",
    "# Preset File\n",
    "#!pygmentize src/markov/presets/default.py\n",
    "#!pygmentize src/markov/presets/preset_attention_layer.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Copy custom files to S3 bucket so that sagemaker & robomaker can pick it up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456\n"
     ]
    }
   ],
   "source": [
    "s3_location = \"s3://%s/%s\" % (s3_bucket, s3_prefix)\n",
    "print(s3_location)\n",
    "\n",
    "# Clean up the previously uploaded files\n",
    "!aws s3 rm --recursive {s3_location}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Copy the environment file to the s3 bucket\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: src/markov/.ipynb_checkpoints/s3_client-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/.ipynb_checkpoints/s3_client-checkpoint.py\n",
      "upload: src/markov/.ipynb_checkpoints/evaluation_worker-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/.ipynb_checkpoints/evaluation_worker-checkpoint.py\n",
      "upload: src/markov/__init__.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/__init__.py\n",
      "upload: src/markov/defaults.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/defaults.py\n",
      "upload: src/markov/environments/__init__.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/environments/__init__.py\n",
      "upload: src/markov/environments/.ipynb_checkpoints/dump2s3_deepracer_racetrack_env_advanced_sahika-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/environments/.ipynb_checkpoints/dump2s3_deepracer_racetrack_env_advanced_sahika-checkpoint.py\n",
      "upload: src/markov/deepracer_memory_multi.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/deepracer_memory_multi.py\n",
      "upload: src/markov/.ipynb_checkpoints/rollout_worker-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/.ipynb_checkpoints/rollout_worker-checkpoint.py\n",
      "upload: src/markov/presets/.ipynb_checkpoints/default-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/presets/.ipynb_checkpoints/default-checkpoint.py\n",
      "upload: src/markov/actions/model_metadata.json to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/actions/model_metadata.json\n",
      "upload: src/markov/deepracer_memory.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/deepracer_memory.py\n",
      "upload: src/markov/evaluation_worker.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/evaluation_worker.py\n",
      "upload: src/markov/environments/.ipynb_checkpoints/deepracer_racetrack_env-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/environments/.ipynb_checkpoints/deepracer_racetrack_env-checkpoint.py\n",
      "upload: src/markov/presets/.ipynb_checkpoints/preset_vgg_attention_bnorm-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/presets/.ipynb_checkpoints/preset_vgg_attention_bnorm-checkpoint.py\n",
      "upload: src/markov/environments/deepracer_racetrack_env.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/environments/deepracer_racetrack_env.py\n",
      "upload: src/markov/presets/__init__.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/presets/__init__.py\n",
      "upload: src/markov/presets/.ipynb_checkpoints/preset_self_attention-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/presets/.ipynb_checkpoints/preset_self_attention-checkpoint.py\n",
      "upload: src/markov/presets/preset.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/presets/preset.py\n",
      "upload: src/markov/presets/.ipynb_checkpoints/preset_shallow_attention-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/presets/.ipynb_checkpoints/preset_shallow_attention-checkpoint.py\n",
      "upload: src/markov/environments/.ipynb_checkpoints/deepracer_racetrack_env_advanced_sahika-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/environments/.ipynb_checkpoints/deepracer_racetrack_env_advanced_sahika-checkpoint.py\n",
      "upload: src/markov/rewards/__init__.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/rewards/__init__.py\n",
      "upload: src/markov/environments/.ipynb_checkpoints/deepracer_racetrack_env_backup-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/environments/.ipynb_checkpoints/deepracer_racetrack_env_backup-checkpoint.py\n",
      "upload: src/markov/presets/.ipynb_checkpoints/preset_vgg_attention-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/presets/.ipynb_checkpoints/preset_vgg_attention-checkpoint.py\n",
      "upload: src/markov/s3_client.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/s3_client.py\n",
      "upload: src/markov/utils.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/utils.py\n",
      "upload: src/markov/rollout_worker.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/rollout_worker.py\n",
      "upload: src/markov/rewards/reward_function.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/rewards/reward_function.py\n",
      "upload: src/markov/sagemaker_graph_manager.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/sagemaker_graph_manager.py\n",
      "upload: src/markov/rewards/.ipynb_checkpoints/advanced_wedge_reward-checkpoint.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/rewards/.ipynb_checkpoints/advanced_wedge_reward-checkpoint.py\n",
      "upload: src/markov/s3_boto_data_store.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/s3_boto_data_store.py\n",
      "upload: src/markov/s3_simdata_upload.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/markov/s3_simdata_upload.py\n",
      "upload: src/markov/environments/deepracer_racetrack_env.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/environments/deepracer_racetrack_env.py\n"
     ]
    }
   ],
   "source": [
    "# recursive copy\n",
    "!aws s3 cp src/markov {s3_location}/markov/ --recursive\n",
    "\n",
    "# Make any changes to the environment and preset files below and upload these files\n",
    "!aws s3 cp src/markov/environments/deepracer_racetrack_env.py {s3_location}/environments/deepracer_racetrack_env.py\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### change the reward function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: src/markov/rewards/reward_function.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py\r\n"
     ]
    }
   ],
   "source": [
    "#!aws s3 cp src/markov/rewards/default.py {s3_location}/rewards/reward_function.py\n",
    "#!aws s3 cp src/markov/rewards/wide_throttlepenalty.py {s3_location}/rewards/reward_function.py\n",
    "#!aws s3 cp src/markov/rewards/wedge_reward.py {s3_location}/rewards/reward_function.py\n",
    "#!aws s3 cp src/markov/rewards/advanced_change_lane.py {s3_location}/rewards/reward_function.py\n",
    "#!aws s3 cp src/markov/rewards/basic_lane_only.py {s3_location}/rewards/reward_function.py\n",
    "#!aws s3 cp src/markov/rewards/reward_function_shallow_advanced.py {s3_location}/rewards/reward_function.py\n",
    "#!aws s3 cp src/markov/rewards/simple_V2.py {s3_location}/rewards/reward_function.py\n",
    "\n",
    "!aws s3 cp src/markov/rewards/reward_function.py {s3_location}/rewards/reward_function.py\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### change the action space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: src/markov/actions/model_metadata.json to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json\r\n"
     ]
    }
   ],
   "source": [
    "#!aws s3 cp src/markov/actions/model_metadata_10_state-throtte_1.json {s3_location}/model_metadata.json\n",
    "#!aws s3 cp src/markov/actions/model_metadata_10_state-throtte_3.json {s3_location}/model_metadata.json\n",
    "# !aws s3 cp src/markov/actions/model_metadata_10_state-throtte_3-slow_1.json {s3_location}/model_metadata.json\n",
    "# !aws s3 cp src/markov/actions/model_metadata_10_state-throtte_3-slow_2.json {s3_location}/model_metadata.json\n",
    "# !aws s3 cp src/markov/actions/model_metadata_10_state.json {s3_location}/model_metadata.json\n",
    "# !aws s3 cp src/markov/actions/model_metadata_10_state-steering_20.json {s3_location}/model_metadata.json\n",
    "!aws s3 cp src/markov/actions/model_metadata.json {s3_location}/model_metadata.json\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### change the network or hyper-parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: src/markov/presets/preset.py to s3://sahika-lidar-oldcar-tf12-wide-ic/sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/presets/preset.py\r\n"
     ]
    }
   ],
   "source": [
    "#!aws s3 cp src/markov/presets/preset_lidar_stereo_shallow.py {s3_location}/presets/preset.py\n",
    "#!aws s3 cp src/markov/presets/preset_lidar_stereo_semi_shallow.py {s3_location}/presets/preset.py\n",
    "#!aws s3 cp src/markov/presets/preset_lidar_stereo_attention.py {s3_location}/presets/preset.py\n",
    "#!aws s3 cp src/markov/presets/preset_lidar_stereo_conv1D.py {s3_location}/presets/preset.py\n",
    "#!aws s3 cp src/markov/presets/preset_lidar_stereo_deep.py {s3_location}/presets/preset.py\n",
    "\n",
    "# !aws s3 cp src/markov/presets/preset_vgg_attention.py {s3_location}/presets/preset.py\n",
    "# !aws s3 cp src/markov/presets/preset_vgg_attention_bnorm.py {s3_location}/presets/preset.py\n",
    "# !aws s3 cp src/markov/presets/preset_shallow_attention.py {s3_location}/presets/preset.py\n",
    "\n",
    "!aws s3 cp src/markov/presets/preset.py {s3_location}/presets/preset.py\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the RL model using the Python SDK Script mode\n",
    "\n",
    "Next, we define the following algorithm metrics that we want to capture from cloudwatch logs to monitor the training progress. These are algorithm specific parameters and might change for different algorithm. We use [Clipped PPO](https://coach.nervanasys.com/algorithms/policy_optimization/cppo/index.html) for this example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_definitions = [\n",
    "    # Training> Name=main_level/agent, Worker=0, Episode=19, Total reward=-102.88, Steps=19019, Training iteration=1\n",
    "    {'Name': 'reward-training',\n",
    "     'Regex': '^Training>.*Total reward=(.*?),'},\n",
    "    \n",
    "    # Policy training> Surrogate loss=-0.32664725184440613, KL divergence=7.255815035023261e-06, Entropy=2.83156156539917, training epoch=0, learning_rate=0.00025\n",
    "    {'Name': 'ppo-surrogate-loss',\n",
    "     'Regex': '^Policy training>.*Surrogate loss=(.*?),'},\n",
    "     {'Name': 'ppo-entropy',\n",
    "     'Regex': '^Policy training>.*Entropy=(.*?),'},\n",
    "   \n",
    "    # Testing> Name=main_level/agent, Worker=0, Episode=19, Total reward=1359.12, Steps=20015, Training iteration=2\n",
    "    {'Name': 'reward-testing',\n",
    "     'Regex': '^Testing>.*Total reward=(.*?),'},\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use the RLEstimator for training RL jobs.\n",
    "\n",
    "1. Specify the source directory which has the environment file, preset and training code.\n",
    "2. Specify the entry point as the training code\n",
    "3. Specify the choice of RL toolkit and framework. This automatically resolves to the ECR path for the RL Container.\n",
    "4. Define the training parameters such as the instance count, instance type, job name, s3_bucket and s3_prefix for storing model checkpoints and metadata. **Only 1 training instance is supported for now.**\n",
    "4. Set the RLCOACH_PRESET as \"deepracer\" for this example.\n",
    "5. Define the metrics definitions that you are interested in capturing in your logs. These can also be visualized in CloudWatch and SageMaker Notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training job: sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456\n"
     ]
    }
   ],
   "source": [
    "estimator = RLEstimator(entry_point=\"training_worker.py\",\n",
    "                        source_dir='src',\n",
    "                        image_name=custom_image_name,\n",
    "                        dependencies=[\"common/\"],\n",
    "                        role=sagemaker_role,\n",
    "                        train_instance_type=instance_type,\n",
    "                        train_instance_count=1,\n",
    "                        output_path=s3_output_path,\n",
    "                        base_job_name=job_name_prefix,\n",
    "                        metric_definitions=metric_definitions,\n",
    "                        train_max_run=job_duration_in_seconds,\n",
    "                        hyperparameters={\n",
    "                            \"s3_bucket\": s3_bucket,\n",
    "                            \"s3_prefix\": s3_prefix,\n",
    "                            \"aws_region\": aws_region,\n",
    "                            \"preset_s3_key\": \"%s/presets/preset.py\"% s3_prefix,\n",
    "                            \"model_metadata_s3_key\": \"%s/model_metadata.json\" % s3_prefix,\n",
    "                            \"environment_s3_key\": \"%s/environments/deepracer_racetrack_env.py\" % s3_prefix,\n",
    "#                             \"pretrained_s3_bucket\": \"yunzhe-multicar\",\n",
    "#                             \"pretrained_s3_prefix\": \"yunzhe-singlecar-oldcar-sagemaker-191007-172132\"\n",
    "                        },\n",
    "                        subnets=deepracer_subnets,\n",
    "                        security_group_ids=deepracer_security_groups,\n",
    "                    )\n",
    "\n",
    "estimator.fit(wait=False, job_name=s3_prefix)\n",
    "job_name = estimator.latest_training_job.job_name\n",
    "print(\"Training job: %s\" % job_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start the Robomaker job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "robomaker = boto3.client(\"robomaker\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Simulation Application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# robomaker_s3_key = 'sim_app_distributed/gsaur_sim_app_new.tar.gz'\n",
    "# robomaker_source = {'s3Bucket': \"gsaur-test\",\n",
    "#                     's3Key': robomaker_s3_key,\n",
    "#                     'architecture': \"X86_64\"}\n",
    "robomaker_s3_key = 'robomaker/simulation_ws.tar.gz'\n",
    "robomaker_source = {'s3Bucket': s3_bucket,\n",
    "                    's3Key': robomaker_s3_key,\n",
    "                    'architecture': \"X86_64\"}\n",
    "simulation_software_suite={'name': 'Gazebo',\n",
    "                           'version': '7'}\n",
    "robot_software_suite={'name': 'ROS',\n",
    "                      'version': 'Kinetic'}\n",
    "rendering_engine={'name': 'OGRE',\n",
    "                  'version': '1.x'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the DeepRacer bundle provided by RoboMaker service and upload it in our S3 bucket to create a RoboMaker Simulation Application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: build/output.tar.gz to s3://sahika-lidar-oldcar-tf12-wide-ic/robomaker/simulation_ws.tar.gz\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp ./build/output.tar.gz s3://{s3_bucket}/{robomaker_s3_key}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sahika-multicar-oldcar-191205-064523\n",
      "Created a new simulation app with ARN: arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694\n"
     ]
    }
   ],
   "source": [
    "app_name = \"sahika-multicar-oldcar-\" + strftime(\"%y%m%d-%H%M%S\", gmtime())\n",
    "\n",
    "print(app_name)\n",
    "try:\n",
    "    response = robomaker.create_simulation_application(name=app_name,\n",
    "                                                       sources=[robomaker_source],\n",
    "                                                       simulationSoftwareSuite=simulation_software_suite,\n",
    "                                                       robotSoftwareSuite=robot_software_suite,\n",
    "                                                       renderingEngine=rendering_engine)\n",
    "    simulation_app_arn = response[\"arn\"]\n",
    "    print(\"Created a new simulation app with ARN:\", simulation_app_arn)\n",
    "except Exception as e:\n",
    "    if \"AccessDeniedException\" in str(e):\n",
    "        display(Markdown(generate_help_for_robomaker_all_permissions(role)))\n",
    "        raise e\n",
    "    else:\n",
    "        raise e"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Launch the Simulation job on RoboMaker\n",
    "\n",
    "We create [AWS RoboMaker](https://console.aws.amazon.com/robomaker/home#welcome) Simulation Jobs that simulates the environment and shares this data with SageMaker for training. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ResponseMetadata': {'RequestId': '1d8768ed-7660-4f84-ae73-93037f8795c8', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:25 GMT', 'content-type': 'application/json', 'content-length': '2605', 'connection': 'keep-alive', 'x-amzn-requestid': '1d8768ed-7660-4f84-ae73-93037f8795c8', 'x-amz-apigw-id': 'EN8cnHZPvHcFpfQ=', 'x-amzn-trace-id': 'Root=1-5de8a783-ca5d27366bbaede8dbc4b67c'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-5gnqcb4wgnjy', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 25, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-23', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n",
      "{'ResponseMetadata': {'RequestId': '8cd3bcc8-c72d-46de-b64f-b1a99a3ba287', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:26 GMT', 'content-type': 'application/json', 'content-length': '2605', 'connection': 'keep-alive', 'x-amzn-requestid': '8cd3bcc8-c72d-46de-b64f-b1a99a3ba287', 'x-amz-apigw-id': 'EN8c0HpivHcFQDw=', 'x-amzn-trace-id': 'Root=1-5de8a785-99d473be56775ef65a6a4242'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-3djt2c4yxjcj', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 25, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-25', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n",
      "{'ResponseMetadata': {'RequestId': 'bdc8d022-9f90-4e1a-bf34-f014f88cb36f', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:27 GMT', 'content-type': 'application/json', 'content-length': '2605', 'connection': 'keep-alive', 'x-amzn-requestid': 'bdc8d022-9f90-4e1a-bf34-f014f88cb36f', 'x-amz-apigw-id': 'EN8c9E5fPHcFjBw=', 'x-amzn-trace-id': 'Root=1-5de8a786-45a1f42c1131603ea2ac35a2'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-cwck722gh4bk', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 27, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-26', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n",
      "{'ResponseMetadata': {'RequestId': 'bc36e4d4-d2ee-4adc-a48e-49bf64f850d3', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:28 GMT', 'content-type': 'application/json', 'content-length': '2605', 'connection': 'keep-alive', 'x-amzn-requestid': 'bc36e4d4-d2ee-4adc-a48e-49bf64f850d3', 'x-amz-apigw-id': 'EN8dJEe5vHcFZ8Q=', 'x-amzn-trace-id': 'Root=1-5de8a787-695044d0a0f74752ec6ab8d0'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-4lcp9fs81k6q', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 28, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-27', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n",
      "{'ResponseMetadata': {'RequestId': '1b6a960f-50c8-4b0a-aa27-deb957bedc77', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:29 GMT', 'content-type': 'application/json', 'content-length': '2605', 'connection': 'keep-alive', 'x-amzn-requestid': '1b6a960f-50c8-4b0a-aa27-deb957bedc77', 'x-amz-apigw-id': 'EN8dTH3rPHcF20w=', 'x-amzn-trace-id': 'Root=1-5de8a788-c7142b7a468b85eec80a7fb8'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-wyq8rsft8vhc', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 29, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-28', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n",
      "{'ResponseMetadata': {'RequestId': 'f1c1d554-1272-4fbc-9f86-e009dffe302e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:30 GMT', 'content-type': 'application/json', 'content-length': '2604', 'connection': 'keep-alive', 'x-amzn-requestid': 'f1c1d554-1272-4fbc-9f86-e009dffe302e', 'x-amz-apigw-id': 'EN8ddFsMPHcFQQQ=', 'x-amzn-trace-id': 'Root=1-5de8a789-12f903fc3ecce644e5888f68'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-ck8f52fg5y6k', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 30, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-29', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n",
      "{'ResponseMetadata': {'RequestId': '5e09446f-1968-420f-91b0-85f379c11c27', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:31 GMT', 'content-type': 'application/json', 'content-length': '2605', 'connection': 'keep-alive', 'x-amzn-requestid': '5e09446f-1968-420f-91b0-85f379c11c27', 'x-amz-apigw-id': 'EN8dpEbFPHcF6yA=', 'x-amzn-trace-id': 'Root=1-5de8a78a-e3a10e905a9b4cc61e8e94e8'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-fvd48kbwfqv0', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 31, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-30', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n",
      "{'ResponseMetadata': {'RequestId': '5b448d94-149c-49e0-b357-2542c7032e2e', 'HTTPStatusCode': 200, 'HTTPHeaders': {'date': 'Thu, 05 Dec 2019 06:45:33 GMT', 'content-type': 'application/json', 'content-length': '2605', 'connection': 'keep-alive', 'x-amzn-requestid': '5b448d94-149c-49e0-b357-2542c7032e2e', 'x-amz-apigw-id': 'EN8d1GH7PHcF5wA=', 'x-amzn-trace-id': 'Root=1-5de8a78b-9ecf97a803edcd6cbec329b8'}, 'RetryAttempts': 0}, 'arn': 'arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-2cbst0sj20bw', 'status': 'Pending', 'lastUpdatedAt': datetime.datetime(2019, 12, 5, 6, 45, 32, tzinfo=tzlocal()), 'failureBehavior': 'Fail', 'clientRequestToken': '2019-12-05-06-45-31', 'loggingConfig': {'recordAllRosTopics': False}, 'maxJobDurationInSeconds': 72000, 'simulationTimeMillis': 0, 'iamRole': 'arn:aws:iam::828753602420:role/service-role/AmazonSageMaker-ExecutionRole-20190604T142184', 'simulationApplications': [{'application': 'arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694', 'applicationVersion': '$LATEST', 'launchConfig': {'packageName': 'deepracer_simulation_environment', 'launchFile': 'distributed_training.launch', 'environmentVariables': {'ALTERNATE_DRIVING_DIRECTION': 'false', 'APP_REGION': 'us-west-2', 'KINESIS_VIDEO_STREAM_NAME': 'SilverstoneStream', 'METRICS_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'METRICS_S3_OBJECT_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/training_metrics.json', 'METRIC_NAME': 'TrainingRewardScore', 'METRIC_NAMESPACE': 'AWSDeepRacer', 'MODEL_METADATA_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/model_metadata.json', 'PYTHONPATHS': 'wohooo this worked', 'REWARD_FILE_S3_KEY': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456/rewards/reward_function.py', 'ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID': '828753602420', 'SAGEMAKER_SHARED_S3_BUCKET': 'sahika-lidar-oldcar-tf12-wide-ic', 'SAGEMAKER_SHARED_S3_PREFIX': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'TARGET_REWARD_SCORE': 'None', 'TRAINING_JOB_ARN': 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456', 'WORLD_NAME': 'reInvent2019_track'}}}], 'tags': {}, 'vpcConfig': {'subnets': ['subnet-57a5de2e', 'subnet-67cc983d', 'subnet-b63155fd'], 'securityGroups': ['sg-fca8088c'], 'vpcId': 'vpc-b418facc', 'assignPublicIp': True}}\n"
     ]
    }
   ],
   "source": [
    "available_tracks = [\"reinvent_base\", # 0\n",
    "                    \"AWS_track\", # 1\n",
    "                    \"Tokyo_Training_track\", #2\n",
    "                    \"Virtual_May19_Train_track\", #3 (london)\n",
    "                    \"reInvent2018_36inch\", #4\n",
    "                    \"reInvent2018_mirror\", #5\n",
    "                    \"reInvent2019_track\"] #6\n",
    "\n",
    "training_tracks_indices = [6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6]\n",
    "#training_tracks_indices = [6]\n",
    "training_tracks_indices = [6, 6, 6, 6, 6, 6, 6, 6]\n",
    "\n",
    "num_simulation_workers = len(training_tracks_indices)\n",
    "\n",
    "\n",
    "\n",
    "envriron_vars = {\n",
    "    \"WORLD_NAME\": \"reinvent_base\",\n",
    "    \"KINESIS_VIDEO_STREAM_NAME\": \"SilverstoneStream\",\n",
    "    \"PYTHONPATHS\":\"wohooo this worked\",\n",
    "    \"ALTERNATE_DRIVING_DIRECTION\": \"false\",\n",
    "    \"SAGEMAKER_SHARED_S3_BUCKET\": s3_bucket,\n",
    "    \"SAGEMAKER_SHARED_S3_PREFIX\": s3_prefix,\n",
    "    \"TRAINING_JOB_ARN\": job_name,\n",
    "    \"APP_REGION\": aws_region,\n",
    "    \"METRIC_NAME\": \"TrainingRewardScore\",\n",
    "    \"METRIC_NAMESPACE\": \"AWSDeepRacer\",\n",
    "    \"REWARD_FILE_S3_KEY\": \"%s/rewards/reward_function.py\" % s3_prefix,\n",
    "    \"MODEL_METADATA_FILE_S3_KEY\": \"%s/model_metadata.json\" % s3_prefix,\n",
    "    \"METRICS_S3_BUCKET\": s3_bucket,\n",
    "    \"METRICS_S3_OBJECT_KEY\": s3_prefix + \"/training_metrics.json\",\n",
    "    \"TARGET_REWARD_SCORE\": \"None\",\n",
    "    \"ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID\": account_id\n",
    "}\n",
    "\n",
    "simulation_application = {\"application\":simulation_app_arn,\n",
    "                          \"launchConfig\": {\"packageName\": \"deepracer_simulation_environment\",\n",
    "                                           \"launchFile\": \"distributed_training.launch\",\n",
    "                                           \"environmentVariables\": envriron_vars}\n",
    "                         }\n",
    "\n",
    "vpcConfig = {\"subnets\": deepracer_subnets,\n",
    "             \"securityGroups\": deepracer_security_groups,\n",
    "             \"assignPublicIp\": True}\n",
    "\n",
    "responses = []\n",
    "for job_no in range(num_simulation_workers):\n",
    "    simulation_application[\"launchConfig\"][\"environmentVariables\"][\"WORLD_NAME\"] = available_tracks[training_tracks_indices[job_no]]\n",
    "    client_request_token = strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n",
    "    response =  robomaker.create_simulation_job(iamRole=sagemaker_role,\n",
    "                                            clientRequestToken=client_request_token,\n",
    "                                            maxJobDurationInSeconds=job_duration_in_seconds,\n",
    "                                            failureBehavior=\"Fail\",\n",
    "                                            simulationApplications=[simulation_application],\n",
    "                                            vpcConfig=vpcConfig\n",
    "                                            )\n",
    "    print(response)\n",
    "    responses.append(response)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created the following jobs:\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-5gnqcb4wgnjy\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-3djt2c4yxjcj\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-cwck722gh4bk\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-4lcp9fs81k6q\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-wyq8rsft8vhc\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-ck8f52fg5y6k\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-fvd48kbwfqv0\n",
      "Job ARN arn:aws:robomaker:us-west-2:828753602420:simulation-job/sim-2cbst0sj20bw\n"
     ]
    }
   ],
   "source": [
    "print(\"Created the following jobs:\")\n",
    "job_arns = [response[\"arn\"] for response in responses]\n",
    "for response in responses:\n",
    "    print(\"Job ARN\", response[\"arn\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-2Dense-Narrow_5Cars_run4'\n",
      "stream_name_list =['sim-5gnqcb4wgnjy', 'sim-3djt2c4yxjcj', 'sim-cwck722gh4bk', 'sim-4lcp9fs81k6q', 'sim-wyq8rsft8vhc', 'sim-ck8f52fg5y6k', 'sim-fvd48kbwfqv0', 'sim-2cbst0sj20bw']\n",
      "sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456']\n",
      "s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'\n",
      "s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191205-064456'\n",
      "simulation_app_arn = \"arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191205-064523/1575528323694\"\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"S3_MODEL_SAVE_PREFIX = '%s'\" %experiment_name)\n",
    "\n",
    "arn_str = list()\n",
    "for response in responses:\n",
    "    arn_str.append(response[\"arn\"].split('/')[-1])\n",
    "print('stream_name_list =' + str(arn_str))\n",
    "print(\"sagemaker_simapp_name_list = ['\" + str(job_name) + \"']\")\n",
    "\n",
    "print(\"s3_bucket = '%s'\" %s3_bucket)\n",
    "print(\"s3_prefix = '%s'\" %s3_prefix)\n",
    "print('simulation_app_arn = \"%s\"' %simulation_app_arn)\n",
    "\n",
    "#print('s3 location:', s3_location)\n",
    "#print(available_tracks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing the simulations in RoboMaker\n",
    "You can visit the RoboMaker console to visualize the simulations or run the following cell to generate the hyperlinks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "> Click on the following links for visualization of simulation jobs on RoboMaker Console\n",
       "- [Simulation 1](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-5gnqcb4wgnjy)  \n",
       "- [Simulation 2](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-3djt2c4yxjcj)  \n",
       "- [Simulation 3](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-cwck722gh4bk)  \n",
       "- [Simulation 4](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-4lcp9fs81k6q)  \n",
       "- [Simulation 5](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-wyq8rsft8vhc)  \n",
       "- [Simulation 6](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-ck8f52fg5y6k)  \n",
       "- [Simulation 7](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-fvd48kbwfqv0)  \n",
       "- [Simulation 8](https://us-west-2.console.aws.amazon.com/robomaker/home?region=us-west-2#simulationJobs/sim-2cbst0sj20bw)  \n",
       "\n",
       "You can click on Gazebo after you open the above link to start the simulator."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(Markdown(generate_robomaker_links(job_arns, aws_region)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation - ReInvent Track"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-30-58b3e8b6d887>, line 46)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-30-58b3e8b6d887>\"\u001b[0;36m, line \u001b[0;32m46\u001b[0m\n\u001b[0;31m    https://sahika-rl-2.notebook.us-west-2.sagemaker.aws/notebooks/SAHIKA/sahika-yunzhe-tf12-lidar/deepracer-lidar-tf12-adv.ipynb#                                               simulationApplications=[simulation_application],\u001b[0m\n\u001b[0m         ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "sys.path.append(\"./src\")\n",
    "\n",
    "num_simulation_workers = 1\n",
    "\n",
    "\n",
    "envriron_vars = {\n",
    "    \"WORLD_NAME\": \"reInvent2019_track\",\n",
    "    \"KINESIS_VIDEO_STREAM_NAME\": \"SilverstoneStream\",\n",
    "    \"SAGEMAKER_SHARED_S3_BUCKET\": s3_bucket,\n",
    "    \"SAGEMAKER_SHARED_S3_PREFIX\": s3_prefix,\n",
    "    \"MODEL_S3_BUCKET\": s3_bucket,\n",
    "    \"PYTHONPATHS\":\"wohooo this worked\",\n",
    "    \"MODEL_S3_PREFIX\": s3_prefix,\n",
    "    \"ALTERNATE_DRIVING_DIRECTION\": \"false\",\n",
    "    \"APP_REGION\": aws_region,\n",
    "    \"MODEL_METADATA_FILE_S3_KEY\": \"%s/model_metadata.json\" % s3_prefix,\n",
    "    \"METRICS_S3_BUCKET\": s3_bucket,\n",
    "    \"METRICS_S3_OBJECT_KEY\": s3_prefix + \"/evaluation_metrics.json\",\n",
    "    \"NUMBER_OF_TRIALS\": \"5\", # Doesnt matter\n",
    "    \"ROBOMAKER_SIMULATION_JOB_ACCOUNT_ID\": account_id\n",
    "}\n",
    "\n",
    "simulation_application = {\n",
    "    \"application\":simulation_app_arn,\n",
    "    \"launchConfig\": {\n",
    "         \"packageName\": \"deepracer_simulation_environment\",\n",
    "         \"launchFile\": \"evaluation.launch\",\n",
    "         \"environmentVariables\": envriron_vars\n",
    "    }\n",
    "}\n",
    "                            \n",
    "vpcConfig = {\"subnets\": deepracer_subnets,\n",
    "             \"securityGroups\": deepracer_security_groups,\n",
    "             \"assignPublicIp\": True}\n",
    "\n",
    "responses_eval = []\n",
    "for job_no in range(num_simulation_workers):\n",
    "    response =  robomaker.create_simulation_job(clientRequestToken=strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime()),\n",
    "                                                outputLocation={ \n",
    "                                                  \"s3Bucket\": s3_bucket,\n",
    "                                                  \"s3Prefix\": s3_prefix\n",
    "                                                },\n",
    "                                                maxJobDurationInSeconds=job_duration_in_seconds*3,\n",
    "                                                iamRole=sagemaker_role,\n",
    "                                                failureBehavior=\"Continue\",\n",
    "                                                simulationApplications=[simulation_application],\n",
    "                                                vpcConfig=vpcConfig)\n",
    "    responses_eval.append(response)\n",
    "\n",
    "# print(\"Created the following jobs:\")\n",
    "#for response in responses_eval:\n",
    "#    print(\"Job ARN\", response[\"arn\"]) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('~~~ EVAL ~~~\\n')\n",
    "arn_str = list()\n",
    "for response in responses_eval:\n",
    "    arn_str.append(response[\"arn\"].split('/')[-1])\n",
    "print('eval_stream_name_list =' + str(arn_str))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean up RoboMaker and SageMaker training job\n",
    "\n",
    "Execute the cells below if you want to kill RoboMaker and SageMaker job."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Cancelling robomaker job\n",
    "# for job_arn in job_arns:\n",
    "#     robomaker.cancel_simulation_job(job=job_arn)\n",
    "\n",
    "# # # Stopping sagemaker training job\n",
    "# sage_session.sagemaker_client.stop_training_job(TrainingJobName=estimator._current_job_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean Up Simulation Application Resource"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# robomaker.delete_simulation_application(application=simulation_app_arn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean your S3 bucket (Uncomment the awscli commands if you want to do it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Uncomment if you only want to clean the s3 bucket\n",
    "# sagemaker_s3_folder = \"s3://{}/{}\".format(s3_bucket, s3_prefix)\n",
    "# !aws s3 rm --recursive {sagemaker_s3_folder}\n",
    "\n",
    "# robomaker_s3_folder = \"s3://{}/{}\".format(s3_bucket, job_name)\n",
    "# !aws s3 rm --recursive {robomaker_s3_folder}\n",
    "\n",
    "# robomaker_sim_app = \"s3://{}/{}\".format(s3_bucket, 'robomaker')\n",
    "# !aws s3 rm --recursive {robomaker_sim_app}\n",
    "\n",
    "# model_output = \"s3://{}/{}\".format(s3_bucket, s3_bucket)\n",
    "# !aws s3 rm --recursive {model_output}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean the docker images\n",
    "Remove this only when you want to completely remove the docker or clean up the space of the sagemaker instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !docker rmi -f $(docker images -q)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "conda_tensorflow_p36",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}