neurosymbolic_dialog_multiwoz.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"Ka0sO-Zh0Iii"},"source":["# Set up dependencies"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"GuDJe-_By_9Z"},"outputs":[],"source":["%%capture\n","!git clone https://github.com/Tomiinek/MultiWOZ_Evaluation.git\n","!pip install spacy\n","!pip install lexical-diversity python-Levenshtein fuzzywuzzy\n","!pip install sacrebleu sacremoses\n","!python -m spacy download en_core_web_sm"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"CIN1zZqr9k4G","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1651341948252,"user_tz":240,"elapsed":21818,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"43b2b5d9-5da6-4cde-bb85-38ceeab32222"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive/\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive/')\n","\n","# choose your own directory for data and output\n","root_dir = '/content/drive/MyDrive/CIS700 Project/'\n","#root_dir = '/content/drive/MyDrive/CIS700/neurosymbolic_dialog' "]},{"cell_type":"markdown","source":["# MultiWOZ 2.2 Constants & Database\n"],"metadata":{"id":"QdCZt9twBlz7"}},{"cell_type":"code","source":["ALL_DOMAINS = [\"attraction\", \"hotel\", \"restaurant\", \"taxi\", \"train\", \"hospital\", \"police\"]\n","\n","# SLOTS\n","\n","REQUESTABLE_SLOTS = {\n","    \"taxi\": [\"type\", \"phone\"],\n","    \"police\": [\"postcode\", \"address\", \"phone\"],\n","    \"hospital\": [\"address\", \"phone\", \"postcode\"],\n","    \"hotel\": [\"address\", \"postcode\", \"internet\", \"phone\", \"parking\",\n","              \"type\", \"pricerange\", \"stars\", \"area\", \"reference\", \"name\"],\n","    \"attraction\": [\"price\", \"type\", \"address\", \"postcode\", \"phone\", \"area\", \"reference\", \"name\"],\n","    \"train\": [\"duration\", \"leave\", \"price\", \"arrive\", \"id\", \"reference\"],\n","    \"restaurant\": [\"phone\", \"postcode\", \"address\", \"pricerange\", \"food\", \"area\", \"reference\", \"name\"]\n","}\n","\n","ALL_REQSLOT = [\"type\", \"address\", \"postcode\", \"phone\", \"internet\", \"parking\", \"pricerange\", \"food\",\n","               \"stars\", \"area\", \"reference\", \"duration\", \"leave\", \"price\", \"arrive\", \"id\"]\n","\n","INFORMABLE_SLOTS = {\n","    \"taxi\": [\"leave\", \"destination\", \"departure\", \"arrive\"],\n","    \"police\": [],\n","    \"hospital\": [\"department\"],\n","    \"hotel\": [\"type\", \"parking\", \"pricerange\", \"internet\", \"stay\", \"day\", \"people\", \"area\", \"stars\", \"name\"],\n","    \"attraction\": [\"area\", \"type\", \"name\"],\n","    \"train\": [\"destination\", \"day\", \"arrive\", \"departure\", \"people\", \"leave\"],\n","    \"restaurant\": [\"food\", \"pricerange\", \"area\", \"name\", \"time\", \"day\", \"people\"]\n","}\n","\n","ALL_INFSLOT = [\"type\", \"parking\", \"pricerange\", \"internet\", \"stay\", \"day\", \"people\", \"area\", \"stars\", \"name\",\n","               \"leave\", \"destination\", \"departure\", \"arrive\", \"department\", \"food\", \"time\"]\n","\n","CATEGORICAL_SLOTS = {\n","    \"hotel\": {\n","        \"pricerange\": [\"expensive\", \"moderate\", \"cheap\"],\n","        \"area\": [\"centre\", \"east\", \"north\", \"south\", \"west\"],\n","        \"type\": [\"guesthouse\", \"hotel\"],\n","        \"parking\": [\"yes\", \"no\"],\n","        \"internet\": [\"yes\", \"no\"],\n","        \"stay\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\"],\n","        \"people\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\"],\n","        \"day\": [\"monday\", \"tuesday\", \"wednesday\", \"thursday\", \"friday\", \"saturday\", \"sunday\"],\n","        \"stars\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\"]\n","    },\n","    \"restaurant\": {\n","        \"pricerange\": [\"expensive\", \"moderate\", \"cheap\"],\n","        \"area\": [\"centre\", \"east\", \"north\", \"south\", \"west\"],\n","        \"people\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\"],\n","        \"day\": [\"monday\", \"tuesday\", \"wednesday\", \"thursday\", \"friday\", \"saturday\", \"sunday\"],\n","    },\n","    \"attraction\": {\n","        \"area\": [\"centre\", \"east\", \"north\", \"south\", \"west\"],\n","        \"type\": [\"architecture\", \"boat\", \"cinema\", \"college\", \"concerthall\", \"entertainment\",\n","                 \"museum\", \"multiple sports\", \"nightclub\", \"park\", \"swimmingpool\", \"theatre\"],\n","    },\n","    \"train\": {\n","        \"departure\": [\"birmingham new street\", \"bishops stortford\", \"broxbourne\", \"cambridge\",\n","                      \"ely\", \"kings lynn\", \"leicester\", \"london kings cross\", \"london liverpool street\",\n","                      \"norwich\", \"peterborough\", \"stansted airport\", \"stevenage\"],\n","        \"destination\": [\"birmingham new street\", \"bishops stortford\", \"broxbourne\", \"cambridge\",\n","                        \"ely\", \"kings lynn\", \"leicester\", \"london kings cross\", \"london liverpool street\",\n","                        \"norwich\", \"peterborough\", \"stansted airport\", \"stevenage\"],\n","        \"day\": [\"monday\", \"tuesday\", \"wednesday\", \"thursday\", \"friday\", \"saturday\", \"sunday\"],\n","        \"people\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"15\"]\n","    },\n","    \"taxi\": {},\n","    \"hospital\": {},\n","    \"police\": {\n","        \"name\": [\"parkside police station\"]\n","    }\n","}\n","\n","# Ordered by priority\n","ENTITY_SLOTS = {\n","    \"hotel\": [\"area\", \"pricerange\", \"type\", \"stars\", \"parking\",\n","              \"internet\", \"name\", \"phone\", \"address\", \"postcode\"],\n","    \"restaurant\": [\"food\", \"area\", \"pricerange\", \"name\", \"address\", \"phone\", \"postcode\"],\n","    \"attraction\": [],\n","    \"train\": [],\n","    \"taxi\": [],\n","    \"police\": [],\n","    \"hospital\": []\n","}\n","\n","USER_SLOTS = {\n","    \"hotel\": [\"stay\", \"day\", \"people\", \"reference\"],\n","    \"restaurant\": [\"time\", \"day\", \"people\", \"reference\"],\n","    \"attraction\": [],\n","    \"train\": [],\n","    \"taxi\": [],\n","    \"police\": [],\n","    \"hospital\": []\n","}\n","\n","\n","# Dialog Actions\n","\n","USER_ACTIONS = ['inform', 'request', 'thank', 'bye', 'greet']\n","\n","SYSTEM_ACTIONS = {\n","    'restaurant': ['inform', 'request', 'nooffer', 'recommend', 'select', 'offerbook', 'offerbooked', 'nobook'],\n","    'hotel': ['inform', 'request', 'nooffer', 'recommend', 'select', 'offerbook', 'offerbooked', 'nobook'],\n","    'attraction': ['inform', 'request', 'nooffer', 'recommend', 'select'],\n","    'train': ['inform', 'request', 'nooffer', 'offerbook', 'offerbooked', 'select'],\n","    'taxi': ['inform', 'request'],\n","    'police': ['inform', 'request'],\n","    'hospital': ['inform', 'request'],\n","    'general': ['bye', 'greet', 'reqmore', 'thank', 'welcome']\n","}"],"metadata":{"id":"UpmZ_rJIwt3P"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Fuzzy matching is supported so that we can match different variations of slot values extracted from utterances"],"metadata":{"id":"UR_r51RTw99x"}},{"cell_type":"code","source":["%cd /content/MultiWOZ_Evaluation\n","from pathlib import Path\n","import random\n","import json\n","\n","from fuzzywuzzy import fuzz\n","from mwzeval.normalization import normalize_state_slot_value, time_str_to_minutes\n","\n","\n","class MultiWozDB:\n","    \"\"\" MultiWoZ JSON Handler class \"\"\"\n","\n","    DB_DOMAINS = [\"attraction\", \"hotel\", \"restaurant\", \"train\"]\n","\n","    FUZZY_KEYS = {\n","        'hotel' : {'name'},\n","        'attraction' : {'name'},\n","        'restaurant' : {'name', 'food'},\n","        'train' : {'departure', 'destination'}\n","    }\n","\n","    def __init__(self, db_dir):\n","        self.dbs = {}\n","        self.db_keys = {}\n","\n","        for domain in ALL_DOMAINS:\n","            with open(Path(db_dir) / f\"{domain}_db_processed.json\", \"r\", encoding=\"utf-8\") as f:\n","                self.dbs[domain] = json.loads(f.read().lower())\n","            self.db_keys[domain] = set(self.dbs[domain][0].keys())\n","\n","    def one_hot_vector(self, domain, num):\n","        \"\"\"Return number of available entities for particular domain.\"\"\"\n","        vector = [0, 0, 0, 0]\n","        if num == '':\n","            return vector\n","        if domain != 'train':\n","            if num == 0:\n","                vector = [1, 0, 0, 0]\n","            elif num == 1:\n","                vector = [0, 1, 0, 0]\n","            elif num <= 3:\n","                vector = [0, 0, 1, 0]\n","            else:\n","                vector = [0, 0, 0, 1]\n","        else:\n","            if num == 0:\n","                vector = [1, 0, 0, 0]\n","            elif num <= 5:\n","                vector = [0, 1, 0, 0]\n","            elif num <= 10:\n","                vector = [0, 0, 1, 0]\n","            else:\n","                vector = [0, 0, 0, 1]\n","        return vector\n","\n","    def addBookingPointer(self, turn_da):\n","        \"\"\"Add information about availability of the booking option.\"\"\"\n","        # Booking pointer\n","        # Do not consider booking two things in a single turn.\n","        vector = [0, 0]\n","        if turn_da.get('booking-nobook'):\n","            vector = [1, 0]\n","        if turn_da.get('booking-book') or turn_da.get('train-offerbooked'):\n","            vector = [0, 1]\n","        return vector\n","\n","    def addDBPointer(self, domain, match_num, return_num=False):\n","        \"\"\"Create database pointer for all related domains.\"\"\"\n","        if domain in self.DB_DOMAINS:\n","            vector = self.one_hot_vector(domain, match_num)\n","        else:\n","            vector = [0, 0, 0, 0]\n","        return vector\n","\n","    def addDBIndicator(self, domain, match_num, return_num=False):\n","        \"\"\"Create database indicator for all related domains.\"\"\"\n","        if domain in self.DB_DOMAINS:\n","            vector = self.one_hot_vector(domain, match_num)\n","        else:\n","            vector = [0, 0, 0, 0]\n","\n","        # '[db_nores]', '[db_0]', '[db_1]', '[db_2]', '[db_3]'\n","        if vector == [0, 0, 0, 0]:\n","            indicator = '[db_null]'\n","        else:\n","            indicator = '[db_%s]' % vector.index(1)\n","        return indicator\n","\n","    def get_match_num(self, constraints, return_entry=False):\n","        \"\"\"Create database pointer for all related domains.\"\"\"\n","        match = {'general': ''}\n","        entry = {}\n","        for domain in ALL_DOMAINS:\n","            match[domain] = ''\n","            if domain in self.DB_DOMAINS and constraints.get(domain):\n","                matched_ents = self.query(domain, constraints[domain])\n","                match[domain] = len(matched_ents)\n","                if return_entry:\n","                    entry[domain] = matched_ents\n","        if return_entry:\n","            return entry\n","        return match\n","\n","    def pointerBack(self, vector, domain):\n","        # multi domain implementation\n","        if domain.endswith(']'):\n","            domain = domain[1:-1]\n","        if domain != 'train':\n","            nummap = {\n","                0: '0',\n","                1: '1',\n","                2: '2-3',\n","                3: '>3'\n","            }\n","        else:\n","            nummap = {\n","                0: '0',\n","                1: '1-5',\n","                2: '6-10',\n","                3: '>10'\n","            }\n","        if vector[:4] == [0, 0, 0, 0]:\n","            report = ''\n","        else:\n","            num = vector.index(1)\n","            report = domain+': '+nummap[num] + '; '\n","\n","        if vector[-2] == 0 and vector[-1] == 1:\n","            report += 'booking: ok'\n","        if vector[-2] == 1 and vector[-1] == 0:\n","            report += 'booking: unable'\n","\n","        return report\n","\n","    def query(self, domain, constraints, fuzzy_ratio=85):\n","\n","        # Hotel database keys:      address, area, name, phone, postcode, pricerange, type, internet, parking, stars (other are ignored)\n","        # Attraction database keys: address, area, name, phone, postcode, pricerange, type, entrance fee (other are ignored)\n","        # Restaurant database keys: address, area, name, phone, postcode, pricerange, type, food \n","        # Train database contains keys: arriveby, departure, day, leaveat, destination, trainid, price, duration\n","        \n","        # query the db\n","        if domain == 'taxi':\n","            return [{'taxi_colors': random.choice(self.dbs[domain]['taxi_colors']),\n","                     'taxi_types': random.choice(self.dbs[domain]['taxi_types']),\n","                     'taxi_phone': [random.randint(1, 9) for _ in range(10)]}]\n","        if domain == 'police':\n","            return self.dbs['police']\n","        if domain == 'hospital':\n","            if constraints.get('department'):\n","                for entry in self.dbs['hospital']:\n","                    if entry.get('department') == constraints.get('department'):\n","                        return [entry]\n","            else:\n","                return []\n","        \n","        results = []\n","        query = {}\n","        for key in self.db_keys[domain]:  \n","            if key in constraints:\n","                if constraints[key] in [\"dontcare\", \"not mentioned\", \"don't care\", \"dont care\", \"do n't care\", \"do not care\", \"none\", \"unknown\", \"\"]:\n","                    continue\n","                query[key] = normalize_state_slot_value(key, constraints[key])\n","                if key in ['arrive', 'leave']:\n","                    query[key] = time_str_to_minutes(query[key])\n","            else:\n","                query[key] = None\n","                        \n","        for item in self.dbs[domain]:\n","            for k, v in query.items():\n","                if v is None or item[k] == '?':\n","                    continue\n","\n","                if k == 'arrive':\n","                    time = time_str_to_minutes(item[k]) \n","                    if time > v:\n","                        break\n","                elif k == 'leave':\n","                    time = time_str_to_minutes(item[k]) \n","                    if time < v:\n","                        break\n","                else:\n","                    if k in self.FUZZY_KEYS.get(domain, {}):\n","                        f = (lambda x: fuzz.partial_ratio(item[k], x) < fuzzy_ratio)\n","                    else:\n","                        f = (lambda x: item[k] != x)\n","                    if f(v):\n","                        break\n","            else:\n","                results.append(item)\n","\n","        return results"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"a2pQtp0yBvI5","executionInfo":{"status":"ok","timestamp":1651341977642,"user_tz":240,"elapsed":551,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"98ca8fe0-7557-4310-b4ae-952d7bb0dda3"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MultiWOZ_Evaluation\n"]}]},{"cell_type":"markdown","metadata":{"id":"wDNmIaseoZfY"},"source":["# Preprocess & Delexicalize MultiWOZ 2.2\n","Delexicalization replaces specific values in system reponses with placeholders like `[value_food]`, `[value_area]`, etc.\n","\n","WARNING: this will take an hour! You can copy/mount MultiWOZ_2.1 and processed_multiwoz in shared drive to your root directory set above and skip this section."]},{"cell_type":"code","source":["%%capture\n","%cd /content\n","!git clone https://github.com/budzianowski/multiwoz.git\n","!git clone https://github.com/bepoetree/MTTOD.git\n","!apt install git-lfs"],"metadata":{"id":"rkzppfnd_CeB"},"execution_count":null,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NjkLOkn5Bcnz","executionInfo":{"status":"ok","timestamp":1651045313857,"user_tz":240,"elapsed":48658,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"6ee775c2-613a-455b-d646-40785796a5a6"},"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MTTOD\n","Updated git hooks.\n","Git LFS initialized.\n","Your branch is up to date with 'origin/main'.\n","I0427 07:41:29.686999 140589032601472 convert_to_multiwoz_format.py:86] Dialogue SNG01862.json doesn't exist in MultiWOZ 2.2.\n","I0427 07:41:52.481241 140589032601472 convert_to_multiwoz_format.py:109] Finish writing 10437 dialogues\n"]}],"source":["%cd /content/MTTOD\n","!git lfs install\n","!git lfs pull\n","!git checkout -f HEAD\n","!unzip -q -o data.zip -d data\n","\n","# Apply 2.2 corrections to 2.1 so that we can use 2.1's format\n","!python /content/multiwoz/data/MultiWOZ_2.2/convert_to_multiwoz_format.py --multiwoz21_data_dir=data/MultiWOZ_2.1 --output_file=data/MultiWOZ_2.1/data.json\n","!cp /content/multiwoz/data/MultiWOZ_2.2/schema.json data/MultiWOZ_2.1"]},{"cell_type":"markdown","source":["Next, we start preprocessing"],"metadata":{"id":"EVWJ42DZCtK0"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"outputId":"6d283770-7c55-468b-d251-b8d2e63f55e6","id":"-s0-w8KIu4ki","executionInfo":{"status":"ok","timestamp":1651052059341,"user_tz":240,"elapsed":3419506,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MTTOD\n"]},{"output_type":"stream","name":"stderr","text":["100%|██████████| 10437/10437 [56:38<00:00,  3.07it/s]\n"]},{"output_type":"stream","name":"stdout","text":["Save preprocessed data to /content/drive/MyDrive/CIS700/neurosymbolic_dialog/processed_multiwoz (#train: 8433, #dev: 999, #test: 1000)\n"]}],"source":["%cd /content/MTTOD\n","import os\n","import re\n","import copy\n","import argparse\n","from collections import OrderedDict\n","\n","import spacy\n","from tqdm import tqdm\n","\n","from utils import definitions\n","from utils.io_utils import load_json, save_json, load_text\n","from utils.clean_dataset import clean_time\n","\n","\n","def clean_text(text, mapping_pair=None):\n","    text = text.strip()\n","    text = text.lower()\n","\n","    text = text.replace(\"`\", \"\")\n","    text = text.replace(u\"’\", \"'\")\n","    text = text.replace(u\"‘\", \"'\")\n","    text = text.replace(';', ',')\n","    text = text.replace('\"', ' ')\n","    text = text.replace('/', ' and ')\n","    text = text.replace(\"don't\", \"do n't\")\n","    text = clean_time(text)\n","    baddata = {r'c\\.b (\\d), (\\d) ([a-z])\\.([a-z])': r'cb\\1\\2\\3\\4',\n","               'c.b. 1 7 d.y': 'cb17dy',\n","               'c.b.1 7 d.y': 'cb17dy',\n","               'c.b 25, 9 a.q': 'cb259aq',\n","               'isc.b 25, 9 a.q': 'is cb259aq',\n","               'c.b2, 1 u.f': 'cb21uf',\n","               'c.b 1,2 q.a': 'cb12qa',\n","               '0-122-336-5664': '01223365664',\n","               'postcodecb21rs': 'postcode cb21rs',\n","               r'i\\.d': 'id',\n","                        ' i d ': ' id ',\n","                        'Telephone:01223358966': 'Telephone: 01223358966',\n","                        'depature': 'departure',\n","                        'depearting': 'departing',\n","                        '-type': ' type',\n","                        r\"b[\\s]?&[\\s]?b\": \"bed and breakfast\",\n","                        \"b and b\": \"bed and breakfast\",\n","                        \"wo n\\'t\": \"will not\",\n","                        \" \\'d \": \" would \",\n","                        \" \\'m \": \" am \",\n","                        \" \\'re' \": \" are \",\n","                        \" \\'ll' \": \" will \",\n","                        \" \\'ve \": \" have \",\n","                        r'^\\'': '',\n","                        r'\\'$': '',\n","               }\n","    for tmpl, good in baddata.items():\n","        text = re.sub(tmpl, good, text)\n","\n","    text = re.sub(r'([a-zT]+)\\.([a-z])', r'\\1 . \\2', text)   # 'abc.xyz' -> 'abc . xyz'\n","    text = re.sub(r'(\\w+)\\.\\.? ', r'\\1 . ', text)   # if 'abc. ' -> 'abc . '\n","\n","    if mapping_pair is not None:\n","        for (fromx, tox) in mapping_pair:\n","            text = ' ' + text + ' '\n","            text = text.replace(' ' + fromx + ' ', ' ' + tox + ' ')[1:-1]\n","\n","    return text\n","\n","\n","def clean_slot_values(domain, slot, value, mapping_pair=None):\n","    value = clean_text(value, mapping_pair)\n","    if not value:\n","        value = ''\n","    elif value == 'not mentioned':\n","        value = ''\n","        # value = 'not mentioned' # if in DST setting\n","    elif domain == 'attraction':\n","        if slot == 'name':\n","            if value == 't':\n","                value = ''\n","            if value == 'trinity':\n","                value = 'trinity college'\n","        elif slot == 'area':\n","            if value in ['town centre', 'cent', 'center', 'ce']:\n","                value = 'centre'\n","            elif value in ['ely', 'in town', 'museum', 'norwich', 'same area as hotel']:\n","                value = \"\"\n","            elif value in ['we']:\n","                value = \"west\"\n","        elif slot == 'type':\n","            if value in ['m', 'mus', 'musuem']:\n","                value = 'museum'\n","            elif value in ['art', 'architectural']:\n","                value = \"architecture\"\n","            elif value in ['churches']:\n","                value = \"church\"\n","            elif value in ['coll']:\n","                value = \"college\"\n","            elif value in ['concert', 'concert hall']:\n","                value = 'concerthall'\n","            elif value in ['night club']:\n","                value = 'nightclub'\n","            elif value in ['mutiple sports', 'mutliple sports', 'sports', 'galleria']:\n","                value = 'multiple sports'\n","            elif value in ['ol', 'science', 'gastropub', 'la raza']:\n","                value = ''\n","            elif value in ['swimming pools', 'swimming pool', 'pool']:\n","                value = 'swimmingpool'\n","            elif value in ['fun']:\n","                value = 'entertainment'\n","        elif slot in ['entrancefee', 'entrance fee', 'fee']:\n","            slot = 'price'\n","        elif slot in ['open hours']:\n","            slot = 'openhours'\n","\n","    elif domain == 'hotel':\n","        if slot == 'area':\n","            if value in ['cen', 'centre of town', 'near city center', 'center']:\n","                value = 'centre'\n","            elif value in ['east area', 'east side']:\n","                value = 'east'\n","            elif value in ['in the north', 'north part of town']:\n","                value = 'north'\n","            elif value in ['we']:\n","                value = \"west\"\n","        elif slot == \"day\":\n","            if value == \"monda\":\n","                value = \"monday\"\n","            elif value == \"t\":\n","                value = \"tuesday\"\n","        elif slot == 'name':\n","            if value == 'uni':\n","                value = 'university arms hotel'\n","            elif value == 'university arms':\n","                value = 'university arms hotel'\n","            elif value == 'acron':\n","                value = 'acorn guesthouse'\n","            elif value == 'ashley':\n","                value = 'ashley hotel'\n","            elif value == 'la':\n","                value = 'la margherit'\n","            elif value == 'no':\n","                value = ''\n","        elif slot == 'internet':\n","            if value == 'does not':\n","                value = 'no'\n","            elif value in ['y', 'free', 'free internet']:\n","                value = 'yes'\n","            elif value in ['4']:\n","                value = ''\n","        elif slot == 'parking':\n","            if value == 'n':\n","                value = 'no'\n","            elif value in ['free parking', 'free']:\n","                value = 'yes'\n","            elif value in ['y']:\n","                value = 'yes'\n","        elif slot in ['pricerange', 'price range']:\n","            slot = 'pricerange'\n","            if 'moderate' in value:\n","                value = 'moderate'\n","            elif 'expensive' in value:\n","                value = 'expensive'\n","            elif value in ['any']:\n","                value = \"dontcare\"\n","            elif value == 'inexpensive' or 'cheap' in value:\n","                value = \"cheap\"\n","            elif value in ['2', '4']:\n","                value = ''\n","        elif slot == 'stars':\n","            if value == 'two':\n","                value = '2'\n","            elif value == 'three':\n","                value = '3'\n","            elif value in ['4-star', '4 stars', '4 star', 'four star', 'four stars']:\n","                value = '4'\n","        elif slot == 'type':\n","            if value == '0 star rarting':\n","                value = ''\n","            # elif value in ['guest house', 'questhouse', 'quest house']:\n","            #     value = 'guesthouse'\n","            elif value in ['guest house', 'questhouse', 'quest house', 'guest houses']:\n","                value = 'guesthouse'\n","            elif value == 'hotels':\n","                value = 'hotel'\n","            elif value not in ['hotel', 'guesthouse', \"dontcare\", '?']:\n","                value = ''\n","    elif domain == 'restaurant':\n","        if slot == \"area\":\n","            if value in [\"center\", 'scentre', \"center of town\", \"city center\",\n","                         \"cb30aq\", \"town center\", 'centre of cambridge', 'city centre']:\n","                value = \"centre\"\n","            elif value == \"west part of town\":\n","                value = \"west\"\n","            elif value == \"n\":\n","                value = \"north\"\n","            elif value in ['the south']:\n","                value = 'south'\n","            elif value not in ['centre', 'south', \"dontcare\", 'west', 'east', 'north', '?']:\n","                value = ''\n","        elif slot == \"day\":\n","            if value == \"monda\":\n","                value = \"monday\"\n","            elif value == \"t\":\n","                value = \"tuesday\"\n","        elif slot in ['pricerange', 'price range']:\n","            slot = 'pricerange'\n","            if value in ['mode', 'mo'] or 'moderate' in value:\n","                value = 'moderate'\n","            elif value in ['not']:\n","                value = ''\n","            elif value in ['inexpensive', 'ch'] or 'cheap' in value:\n","                value = \"cheap\"\n","            elif 'expensive' in value:\n","                value = 'expensive'\n","        elif slot == \"food\":\n","            if value == \"barbecue\":\n","                value = \"barbeque\"\n","        elif slot == \"time\":\n","            if value == \"9:00\":\n","                value = \"09:00\"\n","            elif value == \"9:45\":\n","                value = \"09:45\"\n","            elif value == \"1330\":\n","                value = \"13:30\"\n","            elif value == \"1430\":\n","                value = \"14:30\"\n","            elif value == \"9:15\":\n","                value = \"09:15\"\n","            elif value == \"9:30\":\n","                value = \"09:30\"\n","            elif value == \"1830\":\n","                value = \"18:30\"\n","            elif value == \"9\":\n","                value = \"09:00\"\n","            elif value == \"2:00\":\n","                value = \"14:00\"\n","            elif value == \"1:00\":\n","                value = \"13:00\"\n","            elif value == \"3:00\":\n","                value = \"15:00\"\n","    elif domain == 'taxi':\n","        if slot in ['arriveBy', 'arrive by']:\n","            slot = 'arriveby'\n","            if value == '1530':\n","                value = '15:30'\n","            elif value == '15 minutes':\n","                value = ''\n","        elif slot in ['leaveAt', 'leave at']:\n","            slot = 'leaveat'\n","            if value == '1:00':\n","                value = '01:00'\n","            elif value == '21:4':\n","                value = '21:04'\n","            elif value == '4:15':\n","                value = '04:15'\n","            elif value == '5:45':\n","                value = '05:45'\n","            elif value == '0700':\n","                value = '07:00'\n","            elif value == '4:45':\n","                value = '04:45'\n","            elif value == '8:30':\n","                value = '08:30'\n","            elif value == '9:30':\n","                value = '09:30'\n","            value = value.replace(\".\", \":\")\n","        elif slot in ['car', 'car type']:\n","            slot = 'type'\n","\n","    elif domain == 'train':\n","        if slot in ['arriveBy', 'arrive by']:\n","            slot = 'arriveby'\n","            if value == '1':\n","                value = '01:00'\n","            elif value in ['does not care', 'doesnt care', \"doesn't care\"]:\n","                value = \"dontcare\"\n","            elif value == '8:30':\n","                value = '08:30'\n","            elif value == 'not 15:45':\n","                value = ''\n","            value = value.replace(\".\", \":\")\n","        elif slot == 'day':\n","            if value == 'doesnt care' or value == \"doesn't care\":\n","                value = \"dontcare\"\n","        elif slot in ['leaveAt', 'leave at']:\n","            slot = 'leaveat'\n","            if value == '2:30':\n","                value = '02:30'\n","            elif value == '7:54':\n","                value = '07:54'\n","            elif value == 'after 5:45 pm':\n","                value = '17:45'\n","            elif value in ['early evening', 'friday', 'sunday', 'tuesday', 'afternoon']:\n","                value = ''\n","            elif value == '12':\n","                value = '12:00'\n","            elif value == '1030':\n","                value = '10:30'\n","            elif value == '1700':\n","                value = '17:00'\n","            elif value in ['does not care', 'doesnt care', 'do nt care', \"doesn't care\"]:\n","                value = \"dontcare\"\n","            value = value.replace(\".\", \":\")\n","        elif slot in ['trainid', 'train id']:\n","            slot = 'id'\n","        elif slot == 'time':\n","            slot = 'duration'\n","    if value in ['dont care', \"don't care\", \"do nt care\", \"doesn't care\"]:\n","        value = \"dontcare\"\n","    if definitions.NORMALIZE_SLOT_NAMES.get(slot):\n","        slot = definitions.NORMALIZE_SLOT_NAMES[slot]\n","    if slot.startswith('book'):\n","        slot = slot[4:]\n","    return slot, value\n","\n","\n","class Preprocessor(object):\n","    def __init__(self):\n","        self.nlp = spacy.load(\"en_core_web_sm\")\n","\n","        suffixes = list(self.nlp.Defaults.suffixes)\n","        suffixes.remove(\"\\]\")\n","        suffix_regex = spacy.util.compile_suffix_regex(suffixes)\n","        self.nlp.tokenizer.suffix_search = suffix_regex.search\n","\n","        prefixes = list(self.nlp.Defaults.prefixes)\n","        prefixes.remove(\"\\[\")\n","        prefix_regex = spacy.util.compile_prefix_regex(prefixes)\n","        self.nlp.tokenizer.prefix_search = prefix_regex.search\n","\n","        self.data_dir = os.path.join('data', 'MultiWOZ_2.1')\n","\n","        self.save_dir = os.path.join(root_dir, 'processed_multiwoz')\n","        os.makedirs(self.save_dir, exist_ok=True)\n","\n","        data_name = \"data.json\"\n","        self.dev_list = load_text(os.path.join(self.data_dir, \"valListFile.txt\"))\n","        self.test_list = load_text(os.path.join(self.data_dir, \"testListFile.txt\"))\n","        self.do_tokenize_text = True\n","\n","        self.mapping_pair = self.load_mapping_pair()\n","\n","        # self.preprocess_db()\n","\n","        self.db = MultiWozDB(os.path.join(self.data_dir, \"db\"))\n","        self.data = load_json(os.path.join(self.data_dir, data_name))\n","\n","        self.delex_sg_valdict_path = os.path.join(self.save_dir, \"delex_single_valdict.json\")\n","        self.delex_mt_valdict_path = os.path.join(self.save_dir, \"delex_multi_valdict.json\")\n","        self.ambiguous_val_path = os.path.join(self.save_dir, \"ambiguous_values.json\")\n","\n","        if (not os.path.exists(self.delex_sg_valdict_path) or\n","                not os.path.exists(self.delex_mt_valdict_path) or\n","                not os.path.exists(self.ambiguous_val_path)):\n","            self.delex_sg_valdict, self.delex_mt_valdict, self.ambiguous_vals = self.get_delex_valdict()\n","        else:\n","            self.delex_sg_valdict = load_json(self.delex_sg_valdict_path)\n","            self.delex_mt_valdict = load_json(self.delex_mt_valdict_path)\n","            self.ambiguous_vals = load_json(self.ambiguous_val_path)\n","\n","    def load_mapping_pair(self):\n","        mapping_pair = []\n","\n","        with open('utils/mapping.pair', 'r') as fin:\n","            for line in fin.readlines():\n","                fromx, tox = line.replace('\\n', '').split('\\t')\n","                mapping_pair.append((fromx, tox))\n","        return mapping_pair\n","\n","    def preprocess_db(self):\n","        dbs = {}\n","        for domain in definitions.ALL_DOMAINS:\n","            db_path = os.path.join(self.data_dir, \"db\", \"{}_db.json\".format(domain))\n","\n","            dbs[domain] = load_json(db_path)\n","\n","            for idx, entry in enumerate(dbs[domain]):\n","                new_entry = copy.deepcopy(entry)\n","                for key, value in entry.items():\n","                    if type(value) is not str:\n","                        continue\n","                    del new_entry[key]\n","                    key, value = clean_slot_values(domain, key, value)\n","                    tokenize_and_back = ' '.join(\n","                        [token.text for token in self.nlp(value)]).strip()\n","                    new_entry[key] = tokenize_and_back\n","                dbs[domain][idx] = new_entry\n","\n","            save_json(dbs[domain], os.path.join(self.data_dir, \"db\", \"{}_db_processed.json\".format(domain)))\n","\n","            print(\"[{}] DB processed !\".format(domain))\n","\n","    def get_delex_valdict(self):\n","        skip_entry_type = {\n","            'taxi': ['taxi_phone'],\n","            'police': ['id'],\n","            'hospital': ['id'],\n","            'hotel': ['id', 'location', 'internet', 'parking', 'takesbookings',\n","                      'stars', 'price', 'n', 'postcode', 'phone'],\n","            'attraction': ['id', 'location', 'pricerange', 'price', 'openhours', 'postcode', 'phone'],\n","            'train': ['price', 'id'],\n","            'restaurant': ['id', 'location', 'introduction', 'signature', 'type', 'postcode', 'phone'],\n","        }\n","        entity_value_to_slot = {}\n","        ambiguous_entities = []\n","        for domain, db_data in self.db.dbs.items():\n","            print('Processing entity values in [%s]' % domain)\n","            if domain != 'taxi':\n","                for db_entry in db_data:\n","                    for slot, value in db_entry.items():\n","                        if slot not in skip_entry_type[domain]:\n","                            if type(value) is not str:\n","                                raise TypeError(\n","                                    \"value '%s' in domain '%s' should be rechecked\" % (slot, domain))\n","                            else:\n","                                slot, value = clean_slot_values(domain, slot, value)\n","                                value = ' '.join(\n","                                    [token.text for token in self.nlp(value)]).strip()\n","                                if value in entity_value_to_slot and entity_value_to_slot[value] != slot:\n","                                    # print(value, \": \",entity_value_to_slot[value], slot)\n","                                    ambiguous_entities.append(value)\n","                                entity_value_to_slot[value] = slot\n","            else:   # taxi db specific\n","                db_entry = db_data[0]\n","                for slot, ent_list in db_entry.items():\n","                    if slot not in skip_entry_type[domain]:\n","                        for ent in ent_list:\n","                            entity_value_to_slot[ent] = 'car'\n","        ambiguous_entities = set(ambiguous_entities)\n","        ambiguous_entities.remove('cambridge')\n","        ambiguous_entities = list(ambiguous_entities)\n","        for amb_ent in ambiguous_entities:   # departure or destination? arrive time or leave time?\n","            entity_value_to_slot.pop(amb_ent)\n","        entity_value_to_slot['parkside'] = 'address'\n","        entity_value_to_slot['parkside, cambridge'] = 'address'\n","        entity_value_to_slot['cambridge belfry'] = 'name'\n","        entity_value_to_slot['hills road'] = 'address'\n","        entity_value_to_slot['hills rd'] = 'address'\n","        entity_value_to_slot['Parkside Police Station'] = 'name'\n","\n","        single_token_values = {}\n","        multi_token_values = {}\n","        for val, slt in entity_value_to_slot.items():\n","            if val in ['cambridge']:\n","                continue\n","            if len(val.split()) > 1:\n","                multi_token_values[val] = slt\n","            else:\n","                single_token_values[val] = slt\n","\n","        single_token_values = OrderedDict(\n","            sorted(single_token_values.items(), key=lambda kv: len(kv[0]), reverse=True))\n","\n","        save_json(single_token_values, self.delex_sg_valdict_path)\n","        print('single delex value dict saved!')\n","\n","        multi_token_values = OrderedDict(\n","            sorted(multi_token_values.items(), key=lambda kv: len(kv[0]), reverse=True))\n","\n","        save_json(multi_token_values, self.delex_mt_valdict_path)\n","        print('multi delex value dict saved!')\n","\n","        save_json(ambiguous_entities, self.ambiguous_val_path)\n","        print('ambiguous value dict saved!')\n","\n","        return single_token_values, multi_token_values, ambiguous_entities\n","\n","    def delex_by_annotation(self, dial_turn):\n","        span = dial_turn['span_info']\n","        u = list(dial_turn['orig_text'])\n","        for s in span:\n","            slot = s[1]\n","            if slot == 'open':\n","                continue\n","            if definitions.DA_ABBR_TO_SLOT_NAME.get(slot):\n","                slot = definitions.DA_ABBR_TO_SLOT_NAME[slot]\n","\n","            ### my code\n","            if slot == \"price\" and (\"cheap\" in s[2] or \"moderate\" in s[2] or \"expensive\" in s[2]):\n","                slot = \"pricerange\"\n","            if slot.startswith('book'):\n","                slot = slot[4:]\n","            if slot == 'leaveat':\n","                slot = 'leave'\n","            if slot == 'arriveby':\n","                slot = 'arrive'\n","            if slot == 'trainid':\n","                slot = 'id'\n","            if slot == 'entrancefee':\n","                slot = 'price'\n","            ### my code (end)\n","\n","            for idx in range(s[3], s[4]):\n","                if idx >= len(u):\n","                    print(dial_turn)\n","\n","                u[idx] = ''\n","\n","            if s[3] != s[4]:\n","                u[s[3]] = '[value_'+slot+']'\n","            # try:\n","            #     u[s[3]] = '[value_'+slot+']'\n","            # except (NameError, IndexError):\n","            #     u[5] = '[value_'+slot+']'\n","\n","        if self.do_tokenize_text:\n","            u = ''.join(u)\n","            u = ' '.join([t.text for t in self.nlp(u)])\n","            u_delex = ' '.join(u.replace(\".\", \" . \").split())\n","\n","            u_delex = u_delex.replace(\n","                '[value_address] , [value_address] , [value_address]', '[value_address]')\n","            u_delex = u_delex.replace(\n","                '[value_address] , [value_address]', '[value_address]')\n","            u_delex = u_delex.replace('[value_name] [value_name]', '[value_name]')\n","            u_delex = u_delex.replace(\n","                '[value_name]([value_phone] )', '[value_name] ( [value_phone] )')\n","        else:\n","            u = ''.join(u)\n","\n","            u_delex = u.replace(\n","                '[value_address], [value_address], [value_address]', '[value_address]')\n","            u_delex = u_delex.replace(\n","                '[value_address], [value_address]', '[value_address]')\n","            u_delex = u_delex.replace('[value_name] [value_name]', '[value_name]')\n","\n","        # print(u_delex)\n","        return u_delex\n","\n","    def delex_by_valdict(self, text):\n","        text = clean_text(text)\n","\n","        text = re.sub(r'\\d{5}\\s?\\d{5,7}', '[value_phone]', text)\n","        text = re.sub(r'\\d[\\s-]stars?', '[value_stars]', text)\n","        text = re.sub(r'\\$\\d+|\\$?\\d+.?(\\d+)?\\s(pounds?|gbps?)',\n","                      '[value_price]', text)\n","        text = re.sub(r'tr[\\d]{4}', '[value_id]', text)\n","        text = re.sub(\n","            r'([a-z]{1}[\\. ]?[a-z]{1}[\\. ]?\\d{1,2}[, ]+\\d{1}[\\. ]?[a-z]{1}[\\. ]?[a-z]{1}|[a-z]{2}\\d{2}[a-z]{2})',\n","            '[value_postcode]', text)\n","\n","        for value, slot in self.delex_mt_valdict.items():\n","            text = text.replace(value, '[value_%s]' % slot)\n","\n","        for value, slot in self.delex_sg_valdict.items():\n","            tokens = text.split()\n","            for idx, tk in enumerate(tokens):\n","                if tk == value:\n","                    tokens[idx] = '[value_%s]' % slot\n","            text = ' '.join(tokens)\n","\n","        for ambg_ent in self.ambiguous_vals:\n","            # ely is a place, but appears in words like moderately\n","            start_idx = text.find(' '+ambg_ent)\n","            if start_idx == -1:\n","                continue\n","            front_words = text[:start_idx].split()\n","            ent_type = 'time' if ':' in ambg_ent else 'place'\n","\n","            for fw in front_words[::-1]:\n","                if fw in ['arrive', 'arrives', 'arrived', 'arriving', 'arrival',\n","                          'destination', 'there', 'reach', 'to', 'by', 'before']:\n","                    slot = '[value_arrive]' if ent_type == 'time' else '[value_destination]'\n","                    text = re.sub(' '+ambg_ent, ' '+slot, text)\n","                elif fw in ['leave', 'leaves', 'leaving', 'depart', 'departs', 'departing', 'departure',\n","                            'from', 'after', 'pulls']:\n","                    slot = '[value_leave]' if ent_type == 'time' else '[value_departure]'\n","                    text = re.sub(' '+ambg_ent, ' '+slot, text)\n","\n","        text = text.replace('[value_car] [value_car]', '[value_car]')\n","\n","        return text\n","\n","    def preprocess(self):\n","        # preprocess_main\n","        train_data, dev_data, test_data = {}, {}, {}\n","        count = 0\n","        self.unique_da = {}\n","        ordered_sysact_dict = {}\n","        ordered_useract_dict = {}\n","        # yyy\n","        nn = 0\n","        for fn, raw_dial in tqdm(list(self.data.items())):\n","            nn += 1\n","            '''\n","            if nn == 100:\n","                break\n","            '''\n","            if \".json\" not in fn:\n","                fn += \".json\"\n","\n","            if fn in ['pmul4707.json', 'pmul2245.json', 'pmul4776.json', 'pmul3872.json', 'pmul4859.json']:\n","                continue\n","            count += 1\n","\n","            # NOTE: apply clean_slot_value to goal??\n","            compressed_goal = {}  # for every dialog, keep track the goal, domains, requests\n","            dial_domains, dial_reqs = [], []\n","            for dom, g in raw_dial['goal'].items():\n","                if dom != 'topic' and dom != 'message' and g:\n","                    if g.get('reqt'):  # request info. eg. postcode/address/phone\n","                        # normalize request slots\n","                        for i, req_slot in enumerate(g['reqt']):\n","                            if definitions.NORMALIZE_SLOT_NAMES.get(req_slot):\n","                                g['reqt'][i] = definitions.NORMALIZE_SLOT_NAMES[req_slot]\n","                                dial_reqs.append(g['reqt'][i])\n","                    compressed_goal[dom] = g\n","                    if dom in definitions.ALL_DOMAINS:\n","                        dial_domains.append(dom)\n","\n","            dial_reqs = list(set(dial_reqs))\n","\n","            dial = {'goal': compressed_goal, 'log': []}\n","            single_turn = {}\n","            constraint_dict = OrderedDict()\n","            prev_constraint_dict = {}\n","            prev_turn_domain = ['general']\n","            ordered_sysact_dict[fn] = {}\n","            ordered_useract_dict[fn] = {}\n","            ordered_useract_last_dict = {}\n","\n","            for turn_num, dial_turn in enumerate(raw_dial['log']):\n","                # for user turn, have text\n","                # sys turn: text, belief states(metadata), dialog_act, span_info\n","                dial_state = dial_turn['metadata']\n","\n","                dial_turn['orig_text'] = dial_turn['text']\n","                if self.do_tokenize_text:\n","                    dial_turn['text'] = ' '.join([t.text for t in self.nlp(dial_turn['text'])])\n","                    dial_turn['text'] = ' '.join(dial_turn['text'].replace(\".\", \" . \").split())\n","                    \n","                turn_dom_da = set()\n","                for act in dial_turn['dialog_act']:\n","                    d, a = act.split('-')  # split domain-act\n","                    turn_dom_da.add(d)\n","                turn_dom_da = list(turn_dom_da)\n","                if len(turn_dom_da) != 1 and 'general' in turn_dom_da:\n","                    turn_dom_da.remove('general')\n","                if len(turn_dom_da) != 1 and 'booking' in turn_dom_da:\n","                    turn_dom_da.remove('booking')\n","\n","                if not dial_state:   # user\n","                    # delexicalize user utterance, either by annotation or by val_dict\n","                    u = ' '.join(clean_text(dial_turn['text']).split())\n","                    if 'span_info' in dial_turn and dial_turn['span_info']:\n","                        u_delex = clean_text(self.delex_by_annotation(dial_turn))\n","                    else:\n","                        u_delex = dial_turn['text']\n","                        # No need to use val_dict b/c 2.2 should have complete span annotations\n","                        # u_delex = self.delex_by_valdict(dial_turn['text'])\n","\n","                    single_turn['user'] = u\n","                    single_turn['user_delex'] = u_delex\n","\n","                    # Get user action\n","                    # TODO: maybe only add actions other than inform? Noisy labels\n","                    user_act_dict = {}\n","                    for dom in turn_dom_da:\n","                        user_act_dict[dom] = {}\n","                    add_to_last_collect = []\n","                    for act, params in dial_turn['dialog_act'].items():\n","                        d, a = act.split('-')\n","                        if a == 'inform':\n","                            continue\n","                        if d == 'general' and d not in user_act_dict:\n","                            user_act_dict[d] = {}\n","                        add_pv = {}\n","                        for param in params:\n","                            p, v = param\n","                            if p == 'none':\n","                                continue\n","                            elif definitions.DA_ABBR_TO_SLOT_NAME.get(p):\n","                                p = definitions.DA_ABBR_TO_SLOT_NAME[p]\n","\n","                            p, v = clean_slot_values(d, p, v)\n","                            if len(v.split()) > 1:\n","                                v = ' '.join(\n","                                    [token.text for token in self.nlp(v)]).strip()\n","                            if v != '' and p not in add_pv:\n","                                add_pv[p] = v\n","                        if a in ['inform', 'request'] and len(add_pv) == 0:\n","                            continue\n","                        add_to_last = True if a in ['request', 'bye'] else False\n","                        if add_to_last:\n","                            add_to_last_collect.append((d, a, add_pv))\n","                        else:\n","                            user_act_dict[d][a] = add_pv\n","\n","                    ordered_useract_dict[fn][len(dial['log'])] = user_act_dict\n","                    ordered_useract_last_dict[len(dial['log'])] = add_to_last_collect\n","                else:   # system\n","                    # delexicalize system response, either by annotation or by val_dict\n","                    if 'span_info' in dial_turn and dial_turn['span_info']:\n","                        s_delex = clean_text(self.delex_by_annotation(dial_turn))\n","                    else:\n","                        s_delex = dial_turn['text']\n","                        # No need to use val_dict b/c 2.2 should have complete span annotations\n","                        # if not dial_turn['text']:\n","                        #     print(fn)\n","                        # s_delex = self.delex_by_valdict(dial_turn['text'])\n","                    single_turn['resp'] = s_delex\n","                    single_turn['nodelx_resp'] = ' '.join(clean_text(dial_turn['text']).split())\n","\n","                    # get belief state, semi=informable/book=requestable, put into constraint_dict\n","\n","                    # this has no delete operations because it has cumulative property\n","                    for domain in dial_domains:\n","                        if not constraint_dict.get(domain):\n","                            constraint_dict[domain] = OrderedDict()\n","                        info_sv = dial_state[domain]['semi']\n","                        for s, v in info_sv.items():\n","                            if len(v) == 0:\n","                                continue\n","                            v = v[0]\n","                            s, v = clean_slot_values(domain, s, v)\n","                            if len(v.split()) > 1:\n","                                v = ' '.join(\n","                                    [token.text for token in self.nlp(v)]).strip()\n","                            if v != '':\n","                                constraint_dict[domain][s] = v\n","                        book_sv = dial_state[domain]['book']\n","                        for s, v in book_sv.items():\n","                            if s == 'booked' or len(v) == 0:\n","                                continue\n","                            v = v[0]\n","                            s, v = clean_slot_values(domain, s, v)\n","                            if len(v.split()) > 1:\n","                                v = ' '.join(\n","                                    [token.text for token in self.nlp(v)]).strip()\n","                            if v != '':\n","                                constraint_dict[domain][s] = v\n","                    '''\n","                    # constraint_dict update\n","                    for domain, sv_dict in curr_constraint_dict.items():\n","                        if domain not in constraint_dict:\n","                            constraint_dict[domain] = OrderedDict()\n","                        for s, v in sv_dict.items():\n","                            constraint_dict[domain][s] = v\n","                    '''\n","                    user_act_dict = ordered_useract_dict[fn].get(len(dial['log']))\n","                    user_act_last_collect = ordered_useract_last_dict.get(len(dial['log']))\n","                    prev_sys_act_dict = ordered_sysact_dict[fn].get(len(dial['log']) - 1)\n","\n","                    constraints = []  # list in format of [domain] slot value\n","                    cons_delex = []\n","                    turn_dom_bs = []\n","                    for domain, info_slots in constraint_dict.items():\n","                        if info_slots:\n","                            constraints.append('['+domain+']')\n","                            cons_delex.append('['+domain+']')\n","                            for slot, value in info_slots.items():\n","                                constraints.append('[value_' + slot + ']')\n","                                constraints.extend(value.split())\n","                                cons_delex.append('[value_' + slot + ']')\n","\n","                                if user_act_dict is not None:\n","                                    # prev_sys_info_slots = set()\n","                                    # if prev_sys_act_dict is not None and domain in prev_sys_act_dict:\n","                                    #     if 'inform' in prev_sys_act_dict[domain]:\n","                                    #         prev_sys_info_slots.update(prev_sys_act_dict[domain]['inform'].keys())\n","                                    #     if 'recommend' in prev_sys_act_dict[domain]:\n","                                    #         prev_sys_info_slots.update(prev_sys_act_dict[domain]['recommend'].keys())\n","                                    #     if 'select' in prev_sys_act_dict[domain]:\n","                                    #         prev_sys_info_slots.update(prev_sys_act_dict[domain]['select'].keys())\n","\n","                                    if (domain not in prev_constraint_dict\n","                                        or slot not in prev_constraint_dict[domain]\n","                                        or prev_constraint_dict[domain][slot] != constraint_dict[domain][slot]):\n","                                        # TODO: don't exclude system?\n","                                        # if slot not in prev_sys_info_slots:\n","                                        if domain not in user_act_dict:\n","                                            user_act_dict[domain] = {}\n","                                        if 'inform' not in user_act_dict[domain]:\n","                                            user_act_dict[domain]['inform'] = {}\n","                                        # if slot not in user_act_dict[domain]['inform']:\n","                                        user_act_dict[domain]['inform'][slot] = value\n","                            if domain not in prev_constraint_dict:\n","                                turn_dom_bs.append(domain)\n","                            elif prev_constraint_dict[domain] != constraint_dict[domain]:\n","                                turn_dom_bs.append(domain)\n","\n","                    # get turn domain\n","                    turn_domain = turn_dom_bs\n","                    for dom in turn_dom_da:\n","                        if dom != 'booking' and dom not in turn_domain:\n","                            turn_domain.append(dom)\n","                    if not turn_domain:\n","                        turn_domain = prev_turn_domain\n","                    if len(turn_domain) == 2 and 'general' in turn_domain:\n","                        turn_domain.remove('general')\n","                    if len(turn_domain) == 2:\n","                        if len(prev_turn_domain) == 1 and prev_turn_domain[0] == turn_domain[1]:\n","                            turn_domain = turn_domain[::-1]\n","\n","                    # Add more user inform actions based on constraint dict difference\n","                    if user_act_dict is not None:\n","                        for d, a, add_pv in user_act_last_collect:\n","                            user_act_dict[d][a] = add_pv\n","\n","                        user_act = []\n","                        for d, acts in user_act_dict.items():\n","                            user_act += ['[' + d + ']']\n","                            for a, slots in acts.items():\n","                                self.unique_da[d+'-'+a] = 1\n","                                user_act += ['[' + a + ']']\n","                                for s, v in slots.items():\n","                                    user_act += ['[value_' + s + ']']\n","                                    if v != '?':\n","                                        user_act += [v]\n","                        single_turn['user_act'] = ' '.join(user_act)\n","\n","                    # get system action\n","                    # TODO: also should remove empty domain?\n","                    # TODO: [inform] type [request] type -> [select] type\n","                    # TODO: [inform] type [recommend] type -> delete inform\n","                    # TODO: [inform] type [select] type -> [select] type\n","                    sys_act_dict = {}\n","                    for dom in turn_domain:\n","                        sys_act_dict[dom] = {}\n","                    add_to_last_collect = []\n","                    booking_act_map = {\n","                        'inform': 'offerbook', 'book': 'offerbooked'}\n","                    for act, params in dial_turn['dialog_act'].items():\n","                        if act == 'general-greet':\n","                            continue\n","                        d, a = act.split('-')\n","                        if d == 'general' and d not in sys_act_dict:\n","                            sys_act_dict[d] = {}\n","                        if d == 'booking':\n","                            d = turn_domain[0]\n","                            a = booking_act_map.get(a, a)\n","                        add_pv = {}\n","                        for param in params:\n","                            p, v = param\n","                            if p == 'none':\n","                                continue\n","                            elif definitions.DA_ABBR_TO_SLOT_NAME.get(p):\n","                                p = definitions.DA_ABBR_TO_SLOT_NAME[p]\n","\n","                            p, v = clean_slot_values(d, p, v)\n","                            if len(v.split()) > 1:\n","                                v = ' '.join(\n","                                    [token.text for token in self.nlp(v)]).strip()\n","                            if v != '' and p not in add_pv:\n","                                add_pv[p] = v\n","                        if a in ['inform', 'request', 'recommend'] and len(add_pv) == 0:\n","                            continue\n","                        add_to_last = True if a in [\n","                            'request', 'reqmore', 'bye', 'offerbook'] else False\n","                        if add_to_last:\n","                            add_to_last_collect.append((d, a, add_pv))\n","                        else:\n","                            sys_act_dict[d][a] = add_pv\n","                    for d, a, add_pv in add_to_last_collect:\n","                        sys_act_dict[d][a] = add_pv\n","\n","                    for d in copy.copy(sys_act_dict):\n","                        acts = sys_act_dict[d]\n","                        if not acts:\n","                            del sys_act_dict[d]\n","                        if 'inform' in acts and 'offerbooked' in acts:\n","                            sys_act_dict[d]['offerbooked'] = sys_act_dict[d]['inform']\n","                            del sys_act_dict[d]['inform']\n","\n","                    ordered_sysact_dict[fn][len(dial['log'])] = sys_act_dict\n","                    \n","                    sys_act = []\n","                    nodelx_sys_act = []\n","                    if 'general-greet' in dial_turn['dialog_act']:\n","                        sys_act.extend(['[general]', '[greet]'])\n","                        nodelx_sys_act.extend(['[general]', '[greet]'])\n","                    for d, acts in sys_act_dict.items():\n","                        sys_act += ['[' + d + ']']\n","                        nodelx_sys_act += ['[' + d + ']']\n","                        for a, slots in acts.items():\n","                            self.unique_da[d+'-'+a] = 1\n","                            sys_act += ['[' + a + ']']\n","                            nodelx_sys_act += ['[' + a + ']']\n","                            if 'choice' in slots:\n","                                sys_act += ['choice']\n","                                # there can be multiple [value_choice] so exclude its value\n","                                nodelx_sys_act += ['[value_choice]']\n","                            for s, v in slots.items():\n","                                if s != 'choice':\n","                                    sys_act += [s]\n","                                    nodelx_sys_act += ['[value_' + s + ']']\n","                                    if v != '?': \n","                                        nodelx_sys_act += [v]\n","\n","                    # get db pointers\n","                    matnums = self.db.get_match_num(constraint_dict)\n","                    match_dom = turn_domain[0] if len(\n","                        turn_domain) == 1 else turn_domain[1]\n","                    match = matnums[match_dom]\n","                    dbvec = self.db.addDBPointer(match_dom, match)\n","                    bkvec = self.db.addBookingPointer(dial_turn['dialog_act'])\n","\n","                    # 4 database pointer for domains, 2 for booking\n","                    single_turn['pointer'] = ','.join(\n","                        [str(d) for d in dbvec + bkvec])\n","                    single_turn['match'] = str(match)\n","                    single_turn['constraint'] = ' '.join(constraints)\n","                    single_turn['cons_delex'] = ' '.join(cons_delex)\n","                    single_turn['sys_act'] = ' '.join(sys_act)\n","                    single_turn['nodelx_sys_act'] = ' '.join(nodelx_sys_act)\n","                    single_turn['turn_num'] = len(dial['log'])\n","                    single_turn['turn_domain'] = ' '.join(\n","                        ['['+d+']' for d in turn_domain])\n","                    \n","                    # print('user', single_turn['user'])\n","                    # print('user_delex', single_turn['user_delex'])\n","                    # print('user_act', single_turn['user_act'])\n","                    # print('resp', single_turn['resp'])\n","                    # print('nodelx_sys_act', single_turn['nodelx_sys_act'])\n","                    # print('sys_act', single_turn['sys_act'])\n","\n","                    prev_turn_domain = copy.deepcopy(turn_domain)\n","                    prev_constraint_dict = copy.deepcopy(constraint_dict)\n","\n","                    if 'user' in single_turn:\n","                        dial['log'].append(single_turn)\n","\n","                        for t in single_turn['user_delex'].split():\n","                            if '[' in t and ']' in t and not t.startswith('[') and not t.endswith(']'):\n","                                single_turn['user_delex'].replace(t, t[t.index('['): t.index(']')+1])\n","\n","                    single_turn = {}\n","\n","            if fn in self.dev_list:\n","                dev_data[fn] = dial\n","            elif fn in self.test_list:\n","                test_data[fn] = dial\n","            else:\n","                train_data[fn] = dial\n","\n","        print(\"Save preprocessed data to {} (#train: {}, #dev: {}, #test: {})\"\n","                .format(self.save_dir, len(train_data), len(dev_data), len(test_data)))\n","\n","        save_json(train_data, os.path.join(self.save_dir, \"train_data.json\"))\n","        save_json(dev_data, os.path.join(self.save_dir, \"dev_data.json\"))\n","        save_json(test_data, os.path.join(self.save_dir, \"test_data.json\"))\n","\n","\n","preprocessor = Preprocessor()\n","preprocessor.preprocess()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2479,"status":"ok","timestamp":1650318729776,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"id":"KB5K5jvEl777","outputId":"a34ac12d-1857-4130-8498-da9a44d0c81a"},"outputs":[{"name":"stdout","output_type":"stream","text":["/content/MTTOD\n"]}],"source":["!mv /content/MTTOD/data/MultiWOZ_2.1 {root_dir}"]},{"cell_type":"markdown","metadata":{"id":"iKGdNcGfx20G"},"source":["# Baseline: UBAR with GPT-3\n","\n","For each dialog turn, UBAR takes in the current user utterance, and generates system response along with belief states (slots) and system acts. The past dialog context is also a part of the input. Conditioning on generated slots and system acts helps GPT track states and output more consistent responses. The number of database entries that match the slot constraints is also fed into UBAR to control generation.\n","\n","![Screen Shot 2022-04-18 at 12.57.39 AM.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAB2cAAAE/CAYAAABy0MhUAAAK4GlDQ1BJQ0MgUHJvZmlsZQAASImVlwdYU9kSgM+96SEhQALSCb0J0gkgJfTQpYOohCSQUEJICAoWLCyu4KogIkVlAVdFFFxdAbEhFqwoNuwLsigo62LBhspe4BF2933vve/N9507f+bOmZlzv3PyzQGAHMQWidJhBQAyhNnicD9PemxcPB33DKCBIqABKrBkcyQiZlhYEEBkRv9d3t8F0KS+ZTEZ69/f/1dR4vIkHACgBISTuBJOBsIdyBjliMTZAKAOIXb9pdmiSb6NME2MFIjw0CSnTPOXSU6aYrTClE9kuBfCBgDgSWy2OAUAkhVip+dwUpA4pDCErYRcgRDh1Qi7cfhsLsJIXjA3IyNzkkcQNkH8RQCQaQgzkv4SM+Vv8ZNk8dnsFBlPr2tK8N4CiSidnft/fpr/LRnp0pkcRsgg8cX+4YhWRb7fvbTMQBkLk0JCZ1jAnfKfYr7UP2qGORKv+Bnmsr0DZXPTQ4JmOFngy5LFyWZFzjBP4hMxw+LMcFmuZLEXc4bZ4tm80rQomZ3PY8ni5/EjY2Y4RxAdMsOStIjAWR8vmV0sDZfVzxP6ec7m9ZWtPUPyl/UKWLK52fxIf9na2bP184TM2ZiSWFltXJ63z6xPlMxflO0pyyVKD5P589L9ZHZJToRsbjayOWfnhsm+YSo7IGyGQRDwA3QQhehIEA6YyC8f5BmezVuWPbkYr0xRrliQws+mM5ETx6OzhBzLuXQbKxsbACbP7/SWeBs+dS4hlVOztszdyFZ+j5yZkllbUhkArYVI6gezNoNdAFAKAGjp5EjFOdM29OQDA4iAgvwzqAFtoA9MgAWwAQ7ABXggFQaAUKTeOLAYcAAfZAAxWApWgDWgEBSDLWAbqAI1oB7sAwfBYdAKToAz4AK4Am6AO+Ah6AOD4CUYBe/BOARBOIgMUSE1SAcyhMwhG4gBuUE+UBAUDsVBiVAKJISk0ApoHVQMlUJVUC3UAP0MHYPOQJegHug+1A8NQ2+gzzAKJsE0WAs2gufBDJgJB8KR8CI4Bc6C8+ACeBNcAdfBB+AW+Ax8Bb4D98Ev4TEUQMmhVFC6KAsUA+WFCkXFo5JRYtQqVBGqHFWHakK1o7pQt1B9qBHUJzQWTUXT0RZoF7Q/OgrNQWehV6E3oqvQ+9At6HPoW+h+9Cj6G4aM0cSYY5wxLEwsJgWzFFOIKcfswRzFnMfcwQxi3mOxWBWsMdYR64+Nw6Zil2M3Yndim7Ed2B7sAHYMh8Op4cxxrrhQHBuXjSvEVeIO4E7jbuIGcR/xcngdvA3eFx+PF+LX4svx+/Gn8Dfxz/HjBAWCIcGZEErgEnIJmwm7Ce2E64RBwjhRkWhMdCVGElOJa4gVxCbieeIj4ls5OTk9OSe5BXICudVyFXKH5C7K9ct9IimRzEhepASSlLSJtJfUQbpPeksmk43IHuR4cjZ5E7mBfJb8hPxRnipvKc+S58rny1fLt8jflH9FIVAMKUzKYkoepZxyhHKdMqJAUDBS8FJgK6xSqFY4ptCrMKZIVbRWDFXMUNyouF/xkuKQEk7JSMlHiatUoFSvdFZpgIqi6lO9qBzqOupu6nnqIA1LM6axaKm0YtpBWjdtVFlJ2U45WnmZcrXySeU+FZSKkQpLJV1ls8phlbsqn+dozWHO4c3ZMKdpzs05H1Q1VD1UeapFqs2qd1Q/q9HVfNTS1ErUWtUeq6PVzdQXqC9V36V+Xn1Eg6bhosHRKNI4rPFAE9Y00wzXXK5Zr3lVc0xLW8tPS6RVqXVWa0RbRdtDO1W7TPuU9rAOVcdNR6BTpnNa5wVdmc6kp9Mr6Ofoo7qauv66Ut1a3W7dcT1jvSi9tXrNeo/1ifoM/WT9Mv1O/VEDHYNggxUGjQYPDAmGDEO+4XbDLsMPRsZGMUbrjVqNhoxVjVnGecaNxo9MyCbuJlkmdSa3TbGmDNM0052mN8xgM3szvlm12XVz2NzBXGC+07xnLmau01zh3Lq5vRYkC6ZFjkWjRb+limWQ5VrLVstX8wzmxc8rmdc175uVvVW61W6rh9ZK1gHWa63brd/YmNlwbKptbtuSbX1t823bbF/bmdvx7HbZ3bOn2gfbr7fvtP/q4OggdmhyGHY0cEx03OHYy6AxwhgbGRedME6eTvlOJ5w+OTs4Zzsfdv7DxcIlzWW/y9B84/m8+bvnD7jqubJda1373OhuiW4/uvW567qz3evcn3roe3A99ng8Z5oyU5kHmK88rTzFnkc9P3g5e6306vBGeft5F3l3+yj5RPlU+Tzx1fNN8W30HfWz91vu1+GP8Q/0L/HvZWmxOKwG1miAY8DKgHOBpMCIwKrAp0FmQeKg9mA4OCB4a/CjEMMQYUhrKAhlhW4NfRxmHJYVdnwBdkHYguoFz8Ktw1eEd0VQI5ZE7I94H+kZuTnyYZRJlDSqM5oSnRDdEP0hxjumNKYvdl7sytgrcepxgri2eFx8dPye+LGFPgu3LRxMsE8oTLi7yHjRskWXFqsvTl98cgllCXvJkURMYkzi/sQv7FB2HXssiZW0I2mU48XZznnJ9eCWcYd5rrxS3vNk1+TS5KEU15StKcN8d345f0TgJagSvE71T61J/ZAWmrY3bSI9Jr05A5+RmHFMqCRME57L1M5cltkjMhcVivqynLO2ZY2KA8V7JJBkkaQtm4Y0SlelJtLvpP05bjnVOR+XRi89skxxmXDZ1Vyz3A25z/N8835ajl7OWd65QnfFmhX9K5kra1dBq5JWdebr5xfkD672W71vDXFN2ppra63Wlq59ty5mXXuBVsHqgoHv/L5rLJQvFBf2rndZX/M9+nvB990bbDdUbvhWxC26XGxVXF78ZSNn4+UfrH+o+GFiU/Km7s0Om3dtwW4Rbrlb4l6yr1SxNK90YGvw1pYyellR2bttS7ZdKrcrr9lO3C7d3lcRVNFWaVC5pfJLFb/qTrVndfMOzR0bdnzYyd15c5fHrqYarZrims8/Cn68V+tX21JnVFdej63PqX+2O3p310+Mnxr2qO8p3vN1r3Bv377wfecaHBsa9mvu39wIN0obhw8kHLhx0PtgW5NFU22zSnPxIXBIeujFz4k/3z0ceLjzCONI0y+Gv+w4Sj1a1AK15LaMtvJb+9ri2nqOBRzrbHdpP3rc8vjeE7onqk8qn9x8iniq4NTE6bzTYx2ijpEzKWcGOpd0Pjwbe/b2uQXnus8Hnr94wffC2S5m1+mLrhdPXHK+dOwy43LrFYcrLVftrx69Zn/taLdDd8t1x+ttN5xutPfM7zl10/3mmVvety7cZt2+cifkTs/dqLv3ehN6++5x7w3dT7//+kHOg/GHqx9hHhU9Vnhc/kTzSd2vpr829zn0nez37r/6NOLpwwHOwMvfJL99GSx4Rn5W/lznecOQzdCJYd/hGy8Wvhh8KXo5PlL4u+LvO16ZvPrlD48/ro7Gjg6+Fr+eeLPxrdrbve/s3nWOhY09eZ/xfvxD0Ue1j/s+MT51fY75/Hx86Rfcl4qvpl/bvwV+ezSRMTEhYovZU60AChlwcjIAb/Yi/XEcANQbABAXTvfXUwJN3wmmCPwnnu7Bp8QBgPpeACKXAxB0DYDKKqSlReJTkHtBGAWxuwDY1lY2/iWSZFub6Vgkd6Q1eTwx8dYEAFwJAF9LJibG6ycmvtYjxT4EoCN3uq+fFG3kjrEwH0CVxx95uoB/ynTP/5c1/lODyQrswD/1n36YHJBgjFIsAAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAdnoAMABAAAAAEAAAE/AAAAAEFTQ0lJAAAAU2NyZWVuc2hvdKmmd7QAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjMxOTwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xODk1PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+CobUGngAAEAASURBVHgB7N0HeFzVmfDxV71Llq3mJlty771iXCimGAyEDoEAqZQku8mGfJuyCbubsKkkGzaQ0AMBOxB6NdjGxsbduPcqy7ZsWb33775XvlcjeUYatdGU/3me8dx+z/nNWNK57ylBDUYSEgIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAtwoEd+vVuTgCCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAgClAcJYvAgIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIOABAYKzHkDmFggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggADBWb4DCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAgAcECM56AJlbIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgRn+Q4ggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACHhAgOOsBZG6BAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIEJzlO4AAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgh4QIDgrAeQuQUCCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCBAcJbvAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIOABAYKzHkDmFggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggADBWb4DCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAgAcECM56AJlbIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgRn+Q4ggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACHhAgOOsBZG6BAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIEJzlO4AAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgh4QIDgrAeQuQUCCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCBAcJbvAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIOABAYKzHkDmFggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggADBWb4DCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAgAcECM56AJlbIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgRn+Q4ggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACHhAgOOsBZG6BAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIEJzlO4AAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgh4QIDgrAeQuQUCCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCBAcJbvAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIOABAYKzHkDmFggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggADBWb4DCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAgAcECM56AJlbIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgRn+Q4ggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACHhAgOOsBZG6BAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIEJzlO4AAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgh4QIDgrAeQuQUCCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCBAcJbvAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIOABgVAP3INbIODTAtXV1XLs2DE5e/as1NTUSENDg0+Xh8x3r0BwcLDEx8dLenq6JCUlde/NuHpACpSWlpo/k/Lz86W2tjYgDfyl0PrzIi4uTgYOHCgpKSn+Uiy3yqHf36ysLCkqKpK6ujq3zuEgBIKCgiQ6Olr69esnAwYMEF0nIeBvApWVlebv+XPnzlH38LcPt4vLoz8Dw8LCzDrH4MGDJTIysovvwOUQEDl16pRkZ2dLWVkZz0J8/Avh+PMiKirKx0vTvuzr71StexQXF0t9fX37TubogBUICQmRxMRE0d+x+pyPhAACCHS1AMHZrhblen4joIHYlStXyqZNm6SqqspvykVBPCcwaNAgWbhwoRl48dxduZO/CmgQ66OPPpI9e/ZQofTDD7l///5y+eWXS2Zmph+WrqlIp0+fNr/HR44cadrIEgIdEOjdu7csWLBAJkyY0IGzOQUB7xPQoOzy5cvliy++EG0cSkKgPQLh4eEyadIkueyyyyQiIqI9p3IsAk4FtM6hP5Nyc3Od7mej7wqEhoaafz9p3UMbvflz0oDssmXLzMCsP5eTsnWvgDaGGjFihFx55ZWidRASAggg0FUCQUYvQLoBdpUm1/EbgcLCQnnhhRckLy/Pb8pEQXpGQP+I0wDtRRdd1DMZ4K5+IXD48GFZsmQJDUX84tN0XQj9eTF//nwz4OT6KN/ds3nzZnn33XdpXOC7H6FX5nzcuHFy4403ivZEJyHgqwLao0frHtoQi4RAZwQSEhLkK1/5CiP4dAYxwM/VR4TvvPOO6N9tJP8WiI2Nlbvuukv69u3rlwVds2aNfPzxx/T49stPt2cKpb3Pb775Zhk5cmTPZIC7IoCA3wkQnPW7j5QCdVZAW60/8cQTUlBQ0NlLcT4CtsDixYtl6tSp9joLCLgrcPLkSXnmmWfsIYwLa8LkWFmMFFaHSx3Nq9xl9MrjjFisxIbWSf+ocuNVaeSx8QPVBh1z5szxyjx3NFO7du2SV1991X44klR2TAaW7JHYqjwJbmBY4466Btp5DcZ/msrQODkTO0ROxI+V+qAQk0B7i91www2BxkF5/URApyv485//LPpOQqArBDTg8sADD4i+kxBor8D7778v69evN09rkCDJKouS05VRUl4Xav8d195rcrx3CIQGN0jv8GrJiCkz6iCN0+Po8Mbf+ta3zKFbvSOXXZMLHQFPGxlYKbX0kPQv2SuxNYUSVE/dw3LhvXWB+uAwKY5IlqyEsVIQ2c88WIc61kZQg42hjkkIIIBAZwUIznZWkPP9TuCNN94whxPzu4JRoB4V0BZ23/72t6VXr149mg9u7lsCOh/O448/LtqjRtPB0jjZXxx3PoTnW2Uht60LpERUydTe+RIS1CBa4dOHqsnJya2f5CN7dY6yP/7xj6KNn4KMb+/YMx8bD0f2+Ujuyaa3CpSG95bN/W4wgrUxZhbvvPNOc7gxb80v+ULAlcArr7wie/fudbWb7Qh0SGD06NFy2223dehcTgpcgaNHj8pzzz1nAlTXB8v6vN5SVBMeuCB+WvJgo74xLqFI0qPLzRJmZGTIvffe6zel1Y4WWofWqcq0EeiEnA8ktYwpVfzmA+6hghzpNUUOJDWOiKfz0OrzPR0inIQAAgh0RoDxvzqjx7l+J1BcXCzbt2/3u3JRoJ4X0IrBunXrej4j5MCnBPRhrRWYzS6Pkn0EZn3q82tPZs9WRcj2wsbGG3V1dbJ27dr2nO7Vx27cuNEMzGomh+WtIzDr1Z+W72QutjpfJp9+R4Ia6s1Mr1692ncyT04ROC+gczkSmOXr0B0Cjn9Ddsf1uaZ/CjT9Lg2STfmJBGb982OW+oYgo96RIOeqG+en1qD8iRMn/Ka0+txFn79oGpW7isCs33yyPVuQzMItkl6808yENgDYubNxuWdzxd0RQMDXBQjO+vonSP67VGDPnj3MhdelolzMUWD37t2Oqywj0KaADgWrSQe73VcSby7zj/8KnKyIkmJj2GpN+vtI5/zyh2R9j8PrymVw4Rf+UCTK4CUC8VVnjQduh83c6ENFbWRHQsCXBPjb0Jc+Ld/Kq/4NwffLtz6zns5tRUWFHDnS2LtQGw3mnw/c9XS+uH93CQTJ3qI4++L+9PPCKktUTZEMKOYZjP0hs9BpgaF5G+wpeazvWacvygUQQCCgBQjOBvTHT+FbCuTk5LTcxDoCXSagD43LyxuHDuqyi3IhvxawfiaV1IRKRV3j3Ip+XWAKJ/owTJMOAVxYWOjzItoL2Or93af8hF2Z9fmCUQCvEUg25i+2kvUz01rnHQFvF+A76+2fkG/nj++Xb39+ns79mTNn7IaBZyoa/x71dB64n2cFCo0hq3X4ak3+8vNCp1MpKSkxy5RcftycUsVc4R8EukBAGxtr41BNp0+f7oIrcgkEEAh0AYKzgf4NoPzNBPRhOAmB7hTgO9aduv53bev7UllPYNb/Pl3nJapy+Kytz9/5kb6xtaqqyn7QF1FX4RuZJpc+JRBhPCSxkj/8n7HKwntgCPCdDYzPuadKyferp+R9876O35eqBuoevvkptj/XVXWNj4UdP//2X8V7znAsR3ht09+I3pNDcuLrAlbdw/G75utlIv8IINBzAgRne86eOyOAAAIIIIAAAi4F/GVYY5cFZAcCCCAQ4AL8nA/wL0A3F5/vVzcD+9nl+b742QfazuLw+bcTjMMRQAABBBDoAoHQLrgGl0AAAS8XiI6OlsjISLdzqX+Y61CU2hKsurra7fNcHRgUFCSJiYnSr18/CQsLk+DgYPO6em0ddubs2bNdch9X92c7Agh4XiDE+H9+8fh0+8bnispl19HGIYDsjSwg4AUCEUkDJCgk3L2cNNRLfU2l1JUXS10VrfHdQ+MoBBAINIH4+HgJDXX/UUN9fb1Z99DpP7QO0tmkdY++fftKUlKSmQ+t29TU1Jj1DZ1mJDc3t0vu09l8cj4CCHSdQK/YSJk4NM2+4NHTBXL8TJG9zgICXiEQFCyRKYPdzkpDfa001FRLTWm+NNR2/tmc2zfmQAQQQAABjwi4X2PySHa4CQIIdIfAzJkzZf78+e2+tD7I0LkCs7OzzdeOHTtEh6h0N+kDkdmzZ8vQoUOlV69eLk/T++Tl5cmePXvkk08+cXkcOxBAwHcEZozqL489dKWd4fySCrni316U2rp6exsLCHiDwPDvPC9xQ6e2LytGkLbs+C4pPrBeSvatk7zN7/HApH2CHI0AAn4ssHjxYhk+fHi7S6iBWZ33UOseWVlZsnv3btHArbspIyNDpk6dKkOGDBFtnOoq6X20cei6detk27Ztrg5jOwII+JDAzfPHyAPXT7Nz/PnuE/LgY+/Z6ywg4A0CwaHhMvF/1kpwRFS7slNfUyWlh7dK8f51UrR7tfFaJcbcMe26BgcjgAACCHifAMFZ7/tMyBECXS6wYsUKWbNmjdl7dfz48XLxxRfb9zhw4IDk5+fb69rSXHu3Jicnmz1d9V1fkyZNkksvvVTefvttM4hqn+BiYfLkyXLNNdeYrdW1B+6WLVvk8OHDUlhYaM4/GBUVJfoARR/cpKammi3bMzMzXVyNzQgg4GsCV88a1izLveOiZPaYgbJ6x/Fm21lBoKcFdv7scgmL7S2RaUMk/eYfS8LYeXaWcj55xgi61jSuG73Bg8MiJarvMCOYO0ViBo83X30XfkMqTu6XA//3dSk7tsM+lwUEEEAgUAVeeukl0b/1+/TpY9Y7Ro0aZVN88cUXzRp76og6ERERZk9XrXP079/ffM2YMcNsXPraa6/J6dOn7fOdLeg1Fi5caDYK1f1a39D7HDt2TEpLS0XrN3FxcWbQdsSIEZKQkGDeT0f1ITjrTJRtCPiWgPFfXK6e2bzuMXP0AElJjJGzBWW+VRhy69cCOgLP+vv6SXhimkSlDZWh33pCdBQfTbWlBZK7Zqld/qCQUCOIGy0xg8ZJTPpYiR85y3wNuO57UrTrUznw529KTeEZ+3gWEEAAAQR8T4DgrO99ZuQYgQ4J6BDCZ86ckZMnTzY7f+vWrS6DrRqkHTt2rFx99dXmQxNtgX7bbbfJc889J0ePHm12HccVfehx/fXXm5u05fvLL79sPhhxPEaXDx06JB9//LF5j1tuuaXlbtYRQMBHBaIiQuWSSZmSW1gmiUZQNjSkcYr7xReNIDjro5+pX2dbh7ssyTNflbnHJcGhsMf+/hOpr6pw2NK4GBweKYmTrpQhX/29hBqB3aj+I2Tczz6S7T+ZbwZqLziBDQgggECACVRUVJg9YHUUHsekjUaLipwPNaoB3VmzZsm8efPMgKoGa7/61a/KY489Zk6F4ngdx+UFCxbYgdnt27ebjUl1GOOWae/evfL++++bDU4dG6u2PI51BBDwLYGR6ckyOK2XnDxXIv2T4szMBxsR20Uzh8tzH3zhW4Uht/4vYNQ9qvNPm6/askI7OFttBFqP/u3/OS1/aEwvSZn/ZRl028+N6VhCjcak82X8f34iX/xgutO6itOLsBEBBBBAwOsEGp+Wel22yBACCHiDgD7U0Fbn//jHP5plZ34rQyRry3TH/W+++abTwKzjBXXIMp17loQAAv4hMH9ihmiA9u21++Uzh56ycycMloSYiB4t5KVTMuXuKyb0aB64ue8L1FdXSt6GN2XPr262C6PDk/W7+kF7nQUEEEAAgfYJaEBXg7crV660TwwPD5eLLrrIXm+5EBkZKTqFiyads/add94x55dteZy1rsMkb9q0yVrlHQEE/EBg0fles394dZ3k5JfaJbp29nCjoYe92iMLdy0cL1dMG9Ij9+am/iOgQdxT7z0ux178kV2oiKSBknzRrfY6CwgggAACvidAcNb3PjNyjIDHBbSHq87LZCUdjliHKXOWBg4caA5Fpvu0p67jec6O120656z2sCUhgIB/CFgPSN7fcFDeXXfALlRYaLBcMX2ove7phfCwEPn3O+bIZUaAloRAVwiUHt7SbCjj5Dm3Slh8UldcmmsggAACASuwceNGqa2ttcs/bdo0e7nlgk69osMia9q3b5/oaEFtJe25S8PQtpTYj4BvCIQYw5pfOWOYlFZUmyP0vOdQ98jomyijB6X0WEEGpSbIv948WyYN79tjeeDG/iVwdvXLokMjW6nf1Q9Ij7dAsDLDOwIIIIBAuwUIzrabjBMQCDwBDZ6eOnWqWcFjY2ObrVsriYmJ1qIZdLVX2ljYsWOHOcxxG4exGwEEvFxA55adZcwte+BEnhw5VSBrdmRJUVmVnWsd2rin0uLZI6RPQnRP3Z77+qlA/pb37ZIFh0VItDEnFAkBBBBAoOMC2gO2oKDAvoAGX0NDnc/I1Lt3b/s4rbO4k/S49evXX1C/cedcjkEAAe8SmDGqv/SJj5JPNh+R6pq6Zg1DNac9Wfe458pJxM286+vi87mpqyyVot2r7XJE9RsuIVGNQ3nbG1lAAAEEEPAZAYKzPvNRkVEEelag5cMOHRLMWQo2Wq5aSXvXxsTEWKutvu/cudMcxqzVg9iJAAJeL7Bw+hAJDg4S7TWrqbq2TpZtOmTne8zgFMns19SIw97RgQW9T0xUuERHhJn3bO0S2qr+K1dObO0Q9iHQIQGdr9YxhUQ5b7zkeIzjclBwiPlQReexbfcTPGOsPj0vJDrenH/K8brtXTbzodcJDW/vqRyPAAIIdLuAq7qHTqliJR3Bx7EuYm139r5q1SrZtm2bs11sQwABHxK4+vyQxh+cr3scyymU3ceaRv26csZQCQ8N6ZIShYYES1x0hESEh7b5J1tqYoxcYwyrTEKgqwVqilvUPSLbWfcw5qw16w4d+Zvf+J0bEhlj1l207tCZFBQS1piPTl6nM3ngXAQQQKCnBZw3P+3pXHF/BBDwOgHHHrFVVVWSk5PjNI86V5SVwsLC5LrrrpMlS5aIqwcq1rG8I4CAfwgsmjnc6DUv8uH5ByRaqnc+PyA3zx9jF/BaowfrH19bb6+3Z0GHJ9OHMAunDpF0Y6gwx1RWWWP21j18Ml9e+niHHD6VL8FGBXJcZqpcN2eEDEiONw/vb7z/+K659qlnjLmpnn5vq73OAgLtEWj5YELno20raQC0z7Rrpe8VX5e44Y1zJeo5NcXn5NzaV+XMqpekPGu3y8tE9OkvaZd/TXob14jq2zRUuLamL8/ea5574vVfS3V+46gX4b37ysAbHm52vSpj35nlz0nKvC9Ln+nXSmzmJCM43NjAqvLMETn1wROSu+rvUldV3uw8VhBAAIHuFtBesvHxjb+z9V5ZWVku6xKOdY+UlBSZP38+DT67+wPi+gh4iUBURKhcMjlTcgvLZPOBppG+dFoVbRCqKd4Ips6dMEg+2XKk3bnWth+ThvWVq6YPk0umZIiOEOSYCkoqjPpGgRwy6h7/98ZGc2hlnUZl+sj+cufl40WDuZqmjejfrO6x7WCOvLe+aeoXx2uyjEBbAkEOHSL0WHfqHhqMTZl7h1F/+KpRdxhm30L/5j+76mXJ/ewVqco7aW9vuRCTMUHSLrlHek9dJGEJjf+39BhtpKp1Fn0dfcmYD/f8CBZav0m5uPl8uMUHNkjJwU2SeslXpPfkK0V7/Vqp5NAmc07d/E3vSkN9nbWZdwQCWuD111+3pwBMT0+X66+/vlWPDz74QA4ebOwkoZ2l7rzzzlaP//zzz2Xz5s3mMfq39wMPGMOkkzwuQHDW4+TcEAHfE+jVq5cMHjzYzviuXbukpqbGXndcOHbsmDlHlDX02MiRI+W+++6Tt99+2635Zx2vxTICCPiWgAZLx2akyOb9RtCnoMzO/K6jZyTrTJEdTNU5af/0+gbjQat7ww/qhbTn64M3TDN7v2rAtaa23gj67peT50qMfUGiQdtJQ9OMQGyK+Vq354QZnH3k3gUXtFrXBys3zRtt52//iXMEZ20NFtorENFnQNMpDfVSerj1QH90/5Ey+t9fl/De/czzCrYtM88Ji+sjyfPulL5X3W++Dj35gOi8Ui1T3LDpMurhf0hoTC+pLsyRnGVPSXVRroTFJkrCmLkSN3Sa+Tr76d/t4GxdeYmUn9grCeMvMR+G6DVry4uM4PA3zTlyNQBbU5JvtIKPleCwSIlMzZTMe34j6Tf9u+z51U1tlqllHllHAAEEOiMwYsQIex5ZvU5rPVz1IdS8efPs22lwVhuVfvTRR1JaWmpvZwEBBPxPYP7EDNEA7WurdjerV3y08bD8262zzfqDlvpaY1qV9gZnE2Ij5T/unmcEfzNMOJ2m5ZXlO6WgpFIijZ6zIwclyYQhaTJ1RD/z9cKH28zg7JL/uMmslzhq66hBjiMHaV2G4KyjEMvtEYhIGmgfXplzWGrLmqYBsHc4LPSecpUMe+gZCYmIloa6WiMQu0Qqzx6TqLQhkjT7Jkm/5Scy4Lrvy67/vsb4m3+Lw5mNiylG/WTI1/5ojtJTfnKfUT95ReqNukO40Vi017hLzPqH1kGOvvRj44TG+n11/kmpzM0yGqNeI7FDppgX6jXhMgnvlWqO0qP1EA3shpqj/4SZdZcR331B9Pq7f3Gd1BSeuSAfbEAg0AQef/xxWblypVnshQsXthmcfeGFF2Tp0qXm8ePGjWszOPvuu+/Ko48+ah4fHR1NcLaHvmAEZ3sIntsi4CsCISEhcsUVV9jZzc/Pl48//theb7lQWVkpGzZskIsuusjepS18HnzwQTl8+LBs375d9u/fL3ocCQEE/Evg6hmNrXDfX3+wWcG0Aa22YH/g+mnm9uReMTJz1AD5fPeJZse5WjGeX8jP7pkn2uNW0zrjvP98YZXkGD1eHZMOc6wPYm6/dJy9+afPrhB9XTl9qDz6jcvM7TuPnJG7f/mGfQwLCHRYwOhp2mdGUwvWor2fS21pvsvLhcUny+gfv2k8mEiT+poq2fvrm5vNG5X91u9k3CMfiz50GfL1/zWDr4U7VtjXCzKGIRv+nefMwGzZsR2y678WSV1Fib1fFwbe+P/Ml+NG7VF7etlfpXDXp3ZwVoOwh57+F9FW7NUFp82W7np98yHNTT+SiOR0CY3tbQSCX5VtP5zNQxJHUJYRQKDbBPThkAZYrXTo0CHZutV1o5cTJ06INg4d7NCQdMKECTJmzBjZs2eP6NQpeo26OnriWKa8I+AvAtaQxi3rHtqjdc3OEzLP6DGrac7YdHNe2rziplG+WjPQgO+T37tGRqYnmYdpUFZ7xuooPY5Jj3v8u4tk8vC+9uYv/bTxwbjWSbT3rKalK3fJ//x9jX0MCwh0VCA8MU3iR8yyT88zeppavVXtjQ4L2mhzxL/8zQishknVuWwj8LlYtLeslU5/9BcZ89P3JDgiSkb9YKns/Nnlxv6j1m6zTjL0G38yR9fJXfsPOfTE/c16tuoIQiP+9SXRALBjqjp3Qk6+/ZhZxxh2f2NwVnvE7v3tbVJ6ZJtdX9IhktMWfkMGLP5Xc4hjbcQ66vuvyM6fX2EEkpv/f3O8PssIIICAvwg0jrHhL6WhHAgg0GUC2lt2ypQp8vWvf918uKEX1gcczz77rJSXtz7E4YoVK8yHJI6Z0fmghg4dKjfeeKM8/PDDcsstt8iwYcOMuVqMqAsJAQR8XkD/Ky+aNdzs0brcybBh77cYumux0YLd3XT9nFF2YLawtFJ+/PSKCwKzei3tiWvNN+XutTkOgc4IJM28wXho0dRz9qQRXG0tZdz9qBmY1WOyjWGHi3avbnZ4dUGOZP3jv81tGihNv/knzfbrUGI6pLGmk+/+7wWBWd2uPWkbaqt1sdVUX10h59a/0di79vwQZFZr+p2PXGk8NGlsha89ejO+/ItWr8VOBBBAoDMCWh9ITU2V2bNnm632dVmDqevWrZNXXjF66NTXu7x8g/HzS4d9KyoqanaMjuIzfvx4s9eA1j2uueYa6devccSCZgeyggACPimgI+HMHjNQjp4uEB0Fp2V61xhhx0ragNMK5FrbWnt/+LY5dmB2x+Ez8tuln18QmNXzK6pqZfnWpkBXa9dkHwJdIdDv6oeMQGnjMzRtfJljNL50mYzjhj3wpBmY1WMOP/XtZoFZ3VZyaLOcWfG8Lpqj6fS7+kFz2fpHp1Gxpj3RukvLIYd1/dT7j1uHt/quQV9tdOrYkLWusswM4u793e12kDl2yGRz6pdWL8ZOBBBAwE8EQv2kHBQDAQQ6KKBj1uvDCivpwxGdKzY8PNzaZM4vq0Mp7N27197W2oIOefziiy/KokVGK9LJky84VB+WjB071nzl5eWZwzRoq3Z9uEJCAAHfFBibkWrO6bryi6NSXF51QSF0+OGtB0/LZGPeJk0LJmVIbFS4OfzXBQe32HDz/KYhiJ94a5MxnJjrVu97jp2Tsoq2A1MtbsEqAu0S0IBs8kW3yEBj2F8rnXj9V1K4c6W1esF7cFiE0ap8kb09d80Se9lxQXu3WkkfTui8TjVFZ81NsRkTrV3m/LT2isOCDhG2/w93m8OVOWxu16LOVavlybj7f8zzNN+af+3tS0IAAQQ6I3D//fc3C7YGG9MWaN1DX1bSnq46Us/p00avfjdSYWGh/PWvf5WbbjKGE83IuOCMqKgomT59uvk6fvy4LF++/IKGpBecxAYEEPBqgYXTh4gGXbXXrLPHCKt3HDfrGVrf0KQj8Ly4bEebZYqJDJOrZzWOBqQHP/r3z5oNmdzyAhv2nGy5iXUEulbAeEYX3X+E9L3yW8Z8rfeY19ag6ME/f7P1eWLTx0ikMXSxJq1LONYxzI3n/yky6h461YmmxElXiDz/g6ZAqTHXrJVqii9sBKH7ivd9Ljodi3GSdWi734v3rpXcz18161d6cp/p1xlB3z+3+zqcgIA/Cehz8+rqxmdbOiJMW0mnBbFGsRwypPH/fmvnDBo0yD4+MjKytUPZ140CBGe7EZdLI+ALAjq8cEVFU6BDg7PWQxLtPavraWlpcvvtt0tWVpY555MGUquqWn9AqwHaN998U9avX2+2gtdfKtY8tI4uOkm5PkgZNWqU/POf/zTnq3XczzICCPiGwKLzDzFaDivmmPt3Pz9gB2fDw0Jk4bQh8vrq1ht99O0TJ6MGJduX2bi39QcgdUbvml8aD1EOnMizz2EBgY4KDH/waXNuJvN8I4CgAcqovkPNOVmta2ow9Miz35O8DW9Zm5y+x42YaQ4Zpjur80+7fJiicyzVFBtzyBpDIGtKNOZnsuaebTDmtLVSH6MXrT5McZbyt37obHO7thVs+cAOzupQZ3HDpknRnjXtugYHI4AAAi0FdB5YxyGGrbqHPhSKj483D9fRdjIzM+XAgQNm3UOnRHE8p+U1db2kpESef/550YdRs2bNMkfs0Wu3TPog6r777hMd6WfVqlU0Dm0JxDoCPiKwaOZwM6euRs2prqmTjzYdkhvnNjbyHDagj4wYmOS0l61jkS8aly7hoSHmJp1n1lmvXMfjj5zOl9//Y52U0DjUkYXlDgho40+dd9VKOmSw/g0eM3i8XS/QfTovqwZDSw+7HvJfj+s1/jJ9M1PJwU12wNXaZr2XHd9lLZrDGGsguDx7n7nNsQOFjuBzdtXf7WPtBaN1hFVXsbd1YEHrHtr4VVPs0KnGfLQJonPTkhAIVIH//d//bVfRH3nkEdGXu+mb3/ym6IvUswIEZ3vWn7sj0OMCH374oTlcsbOMRERESP/+/c2HHDNnzhSdO1Zf8+bNM4cYO3XqlLPTmm3Lyckxhxp77733zACsDi+mD01aPizRVkAavH355Zd5SNJMkBUEvF8gNCRYrpg21Oyxqq3UXaWPNx+W/3fnHPuBhw5t3FZwdsGkwfbltEfu8TOF9rqrhdYCxK7OYTsCzgT0IURr6dS7f5Lst39vDwHc2rG9xl9q7644td9edragc0JZwVmdf9ZKFdlN56Ut/LrUlhUYQ4H9QeqqWp9uwDq/Pe+V57LMeWbDeqWap2n+Cc62R5BjEUDAmYCOrtNyCGLruNjYWBkwYIBZZ5g4caKMHDnSfGkD0SVLlogGdltL+hBZe93qKy4uzhylR+efdTac8SWXXGLWR3R0IBICCPiWQHpqgozNSJHth3NER+dxld5bd8AOzuoxWvf4zRLnvf+sa1wyOdNalJ1HzjjtlWsfYCxor90Xl2133MQyAh0SCImMlT4zrmv13EN/fUjOrX3VrdFsehkNPK2kAV1XqSovu9kurXtYwdmKk/ul17gF5v7Me39nDmus9285vHGzC3RwpfjABvtMDUzHj7lY8nVOXRICCCDgxwIEZ/34w6VoCHRWQHvHHjlyxHxt2bJF7r77bundu7ckJCTIPffcI3/5y19EhyV2J+m1tm3bZr70YYkOd6zzSukwY1bSIRj0IYy7wydb5/GOAAI9KzDLmO+pV2yk2Vv1GmPe2dbSydxiyeibaB4yYUia6MOVrDOuW8QOH9jHvtyhk/ltPiCxD2YBgS4Q2PTACKl3CHxGpmXK+P/8xJ67KW7kLKlb6vqhoGMWos4PK6bbQmJ6GUMcX+24u9lySFSsva4t5q2Ut/FtSb/1p2ardt024IaHpe9VD5i9dnPX/kOKjZ6tXfawxHjaWJa1W3qdD85GpjY9rLTywzsCCCDQlQIafN23b5/50nrDnXfeaU61oo1D7733XnnyySdFR+dxJ2lPWp2zVl8pKSkybdo0mTp1qoSENPaI02vMnTvXrJsUFDTOse3OdTkGAQR6XuDqGcPMTBSWVMqX5o5ymaEgCZKq6lqJCG989Knzzv7htXVSU9s0EknLk0e0qHu03M86At0lUHHqgOz4aVNjTr1P78lXybAHm+aVjc2cLGc/fcmtLDjWPSKTB7Va9zAqEPbcssHhTXWP08uekrTLv2rWfYLDI2XY/U/KoNt+bgaIte5RlmX0utUWCl2QdFqV2rJCCTXqSZqiqHt0gSqXQAABbxcgOOvtnxD5Q8BLBPLz82XZsmVy2223mTnSocfmzJkjb731VrtzqA9LdBixTZs2yQ033CAalLXSjBkzCM5aGLwj4CMCi4wHHZo0kPrTu+e1K9c6/9P/vbHR5TlR4WH2vkrj4QoJAU8K1FUUG8HZpqH/y45ul+w3fycDb/x/ZjbijCG3+l/zHcl+6/dtZsvxQYfOHTvy+y+3eY55QFCwfVx9TaXse+zLMurflkp4Ypq5XVvZp8y703xVF+RIjvEQ5fSHT3RJb1rHwLQ+kCEhgAACnhI4evSobNiwQS6++GLzlsnJyaIj8GiD0fams2fPio7io9fTuowGazVpoHbKlCnyySeftPeSHI8AAj0koKOVLzrfGHTexMGiL3eTNia9aGy6fLrtmMtTIs8HcvUADeySEPCUQIMxPU9dRfNGnzoPa9LsLxlzwV5pZiPtsvskf/N7UrhjeZvZcmzgmTT7JuM6N7V5jnmAQ92jMuewObft0G/92ZjepbEuoHWQftd823yVn9gjp957XHLXLO2SBqJm3eN8cNax7uRexjkKAQQQ8D0BgrO+95mRYwR6TEBbshcWFhq9aBpbsukwYR999JHovLUdSeXl5bJ06VJzjPvU1MZhEwcPHmwOMeY4t0VHrs05CCDgGYGYyDCZbzwUKTXmWfr585+22XJWhzT/r69eItaDD+1p+8Sbm6TeRYvbkGDjCYyVuqZRrnU13hHokMDJtx+TpJk3SJQxH5OmgTf9uxRsWyaO8zU5u3CQMWetlQq3fyLHlzxirbb6rvPPOiYNEG/7wQzpt+ghSbroZolMGWzv1ocl2rM2Zf6XZfcvr5Oq3Cx7X0cWHOe4FYcHNR25FucggAAC7RXQXq9WcFbP1blkOxKcte577tw50WGVH3roIdHpWzRlZGRYu3lHAAEfEBibkSoDkuNl7/Fceea91ufc1OKkJMbKw7dfZJdMG4a2Fpx1rHtQ9bDZWOgpAaOOfOTZ78vE31wsIZExZi6GfuNx2fbwzDbnYw1y+NvdCqC6U4yW9Ydz614357ftf+13zWGXQ2N725eJHjhaNHCbPOdW2fu725o1arUPasdCs+eATuaNb8elOBQBBBDwCQGCsz7xMZFJBLxDoN5oyac9aK3grM4Rq0MUdzQ4q6Wqra2V5cuXyx133GEWMth4eB0WFibV1dXeUWhygQACrQosmJxhDhX2wZp9snzLkVaPtXZeNnWIMUftEHM1rXesTB3RTzbuO2ntbvbu2Fs2PKxpKMJmB7GCgAcF6muq5NBT35VxP//QvGtQSJgxxNdfZPtPFkhDrevfXdrr1Ur11ZVGMHentdru99ryIsl69ReS9dovJTZzkhEs/pIkz71dwuIahwGPTM2QEd99wRga7ZI2G0y0dvPgsHB7t2MvWnsjCwgggEA3Cugwxzo1ihVITUxM7PTddL7b9evXy7x5jSN96GhAJAQQ8B0BHZpY0+ur98ryrUfbzLjGd+64bJwZ0NWD504YZE7HUlja9HeZ40Wa1T1CqXs42rDcMwJVeScly2jUmXHPr80MhPfuKxlf+ZUcfOJbrWZI6x7GzK3mMTUleZ2qe1SePSaHn/lXOfL8w5JgzAWbNPtmSZp1g92bNmHsPBl8x3/Jkef+rdU8tbWzWd2jumn0orbOYz8CCCDgqwJNTfh9tQTkGwEEPCqgvd4ckwZsHdOgQYPkxhtvNHu/Om5vbVmHGnNMLa/puI9lBBDwLoFFMxvnmP1gw0G3M/bRxkPNjl18UdPQ5s12GCv5xlxSVhqYkmD8bLHWeEeg5wRKDqyXnI+ftjMQnT5GBn7pYXvd2UJtWZG9OTgi2l7u1ILRmr708FY59vefyJaHxkjuZ0vsy2nQNm7YdHu9IwsRfQbap+k8UCQEEEDA0wKOdQ9ndYSJEyfKggUL2pWt3NymEQmcXbNdF+NgBBDwmEBoSLBcOX2o1BnPID52s1GoDs7jWPcwrzFjqMs8FzSrezQGtlwezA4EPCSQ88kzUnKwaSqg5Itvk95TF7V6d23MaaWQLqp7NNTVGEMqr5BDT95v1j20HmIlHblHp1vpaNI8Wg1N9RpV1D06Ssl5CCDgQwIEZ33owyKrCPS0gD4ccWyxrkOO6NDEjikhIUF0uGPtUduRpD1mtTctCQEEvF8guVeMTB/VX/KKymXzfvcDN2t3ZpnDIFslvHRKpujwyM7SJocetamJMTIotXFYdWfHsg0BTwrosMTV+aftWw647nuic9C6SsX71tm7ogeMkq5uaaCt4w/95cFmD250btuOJp1jNmrASPv0gu0f28ssIIAAAp4QiI2NlfDwph78ZWVlF9w2LS1Npk2bdsF2dze0rMu4ex7HIYCA5wVmjRlo9npdu+uEFLno+eosVx+2bBhqDG3sKjmO5jN91AAJdpxixdVJbEegmwUa6uvk8FPfEQ2OWmnI1/4gYfFJ1uoF7xfUPS44onMbtDfunkdvkNqyQvNCOidttEPdob1Xj04fa9SPzocpjGeNRUYQmIQAAgj4uwDBWX//hCkfAl0oMHjwYHtIY73swYMHpaLC+VAjffv2dfvOAwYMsI/dtWuXvcwCAgh4t8CV04dIsNFoQx941Ne7PytTdW2drHAYhkznn9UArbO0xQj61tY19dCfOabp54Wz49mGgKcE6ipKjKG7vtd0O+NhwtD7n5TgiKimbQ5LRbs+tdd0OLKYQePsdXcXtJW8vlwlfXCTt+Gtpt0O89w2bXRvKcZ4QBIU3DicX31VhRTv/dy9EzkKAQQQ6CIB7RXrmHbs2OG4ai9rELc9DUOpe9h0LCDgUwKLzg9p/MF690fs0QIeOplvvqzCjhqULEP7N82baW3X93W7T9irCTERMio92V5nAYGeFCjP3ifZb/3ezkJYfLJk3mesuxhayrHukTBuvmjDy/am9Fv/Q2KHTHF5mvbO1Z60djpfd7DX27Hg2Ki05PBm0eAvCQEEEPB3AYKz/v4JUz4EukggJCRE5s6d2+xqGzZsaLbuuOL40MNxe8tl7Y3r2Np9y5YtLQ9hHQEEvFTgqhnDzZx9sOFQu3Pobgv28qoa+WhT0/Xvu2qS9I5zHvxqLROOoeOIsNDWDmUfAm4L5G/5wAiGvmkfH9V3qAy69Wf2uuOCztVUlZtlb0q/5adNrcPtra0vRKYMln5XP9TqQToHrpWqjTmqOprSLv+afWrhzhXiOGeuvYMFBBBAoJsEYmKM0TmmNw3NriPrbN682eXd3K17hIWFiRX0rampEVcBX5c3YgcCCPSIgI6yM3/iYNE5YVdtP9buPLSse1zrovfs9kNn5FhOY09Avcm/3DxTQjrQ2I26R7s/Ik5wQ+CkEZytOHXAPrLP9MXG/K832euOC0W7V9urOtywjvLT3qSNSVMX3N3qaUGhDnWPDg5FrPWX1Evvte9TYNSxSAgggEAgCBCcDYRPmTIi0EmBPn36yF133SVDhgyxr7Ru3Tqz56y9ocXCnDlzZPbs2cYwQK5/zGhgVueI0nlqNR0+fFiys7NbXIlVBBDwRoHMfokyalCSZJ0pkj3Hm88b7U5+N+49KQUlTT3vp4zoJ/2T4pye+vjrG6WkvMrcp0MpP/3wYpk8rK/Za9fpCcbGlkOQFRQ33WtQaoL07eP8Xq6ux3YEXAkceeGH4jinU98rvyUJoy92evjJd/5gb0+ceLkM+ervXfa0NQ900hI+ZtBYY/hk50N4ak/XpJk3mKfWFJ+Tgi8+su/XnoX4kbPsHrp1VeXGnLZGIJmEAAIIeEBA6wdaN/ja177WbMSet956S4qLi13mYPHixTJqlDFkfCtJA7M33XSTREY29h7S+kxVVePfF62cxi4EEPACgQWTMyTCGG1n5RdHpaKqtt05cpx3Vk/WXrjOgq46n+1vXlkrOletpqlGHeX3D10hGX0TGze4+Ldl3SPfoe6h9Zbw0MbRSFyczmYE3BKor6kyhzd2PDjz3t9KeGKa4yZzWesCZz99yd4+4IaHpe8V32y9caiTuofWLUJjnX//Q2N6Sa/xl5r3KNq1SqrONfU8t2/sxkLfK74u0QMbf4dXnD4kpz58wo2zOAQBBBDwfQG6jvj+Z0gJEHBLQOdrSkpKkoEDBzY7Xnut6gMQbTleV1dnVEIazJf2lNUW6xqQ7d27acgfbbW+fPly+fzz1oc31POvvPJKmTFjhtkiff/+/VJYWGjeJyoqSoYNGybjx4+3A7NHjhyRV155xbx3swyyggACXiUQHREmQ4xhwG6/1JgTxkjZucUyYUia2cK8qKzSfpDhKtNa39Oer4PSeknW2SJJdOgF+63F0+T1z/bIkdOFzeaRyskvlW//8QN5/F+ultiocPPhyDM/vE50++rtx+0grz6w0fxp0LjCaFX/zd++Y2dj2+Ecs6W9DqGsxz31g8XywYaD5s+cFGMu28+NuauWbTpsH89CYAmERicY86uOksjUzGYF7z1lkdk6XXu91pU7DwrUFBo9LF76sQz9xuP2uUO/9WdjyON/k7Jj26Xa2G/9x8hZ/pzEGvPSpsy9wzw29ZJ7RFu8F+1ZI+XZe415pOokJDJGInr3k4jkdNG5m7b/eJ59XWth9I/ekJxlT8m5df808tfYszw2c6L0X/yvEpMxwTws+43fGL1dXQcdgkLDJW7YdKk8c0TqKkqlob5WQmMSjYc2X5d+i75jXkPPP/h/XzePse7NOwIIIOCOQHR0tCQnJ0tKSkqzwy+//HIpLS216x66U+sfoaGhZjBW6x46TLGV9Ni3335b9u3bZ21y+q71lttvv11OnjwpOk2KTr+ic9Rq/UaHPNbArfaY1fqQpvXr15t1GqcXYyMCCHiNgNYVhhl1j1sXNNY9Co25ZnVY4qwzhVJW2TT/pqsM6xQsqb2NXoPJ8ZJXVC59EqLNQ/X9vqsnydpdWXLMqHvoaD1W+twY2vgXL66WH335YrPB59zxg0Rf+0+cE21gqvfV62q9It4Y+nhcZop8tiNL/vjaeusSsn7PCfnOjTPM9XSjYajWPXTI5NCQYOlv5OX5D7aZ17NPYCFwBIzvTnhCqlH3GNks6KmBzsRJV0qVUe+ozD0m9dWVTk2K96+XnE+elbTL7jP3az1m2INPSfbrv5ay4zvtOWB159G//dAMelpDE2d85VfGKDwPStHeteZ99JgQ4/yIPv1FR+gpy9olh/7yoG62U0h0vEz45Wqz7qHTp1QVnDbqKBESP3K2DLr9EQmJiJaG2mrJfvM39jnOFsLi+kjM4PHmSEL11RVG3aNeIpIGiAaNU85P26L1qv2PfVl0ShUSAgggEAgCBGcD4VOmjG4LaEDRH5P2TtWXs6QPQBx7xDo7prKyUo4dOyZHjx6VvXv3mkFWZ8fpNu39umzZMhkxYoTovLOJiYkyb94889XyHH0Qk5WVZT5A2bhxozFnZdO8ki2P9Zd1f/2O+cvn423lsL4voUZA0xvSjXNHy0/ubj68+eyxA0VfmvYcy5U7//ufrWb1vf+502Wv1WtmDxd9aVq7M0se+uP79rW2G8HVL/10qXz7S9PlkkkZEmMEadOMBy23LBhjH6ML2tp9y4HT8qd/bmi2vbqmTn6zZK385K555rQ82kv3a4smm8doD94POzA0c7MbdNFKsDQNgmZ9/l106R65jGMZ6o1end6YdC4lV8N8DX/oaTvLZcd2yPYfNf/+WzvPrvq7JM+51e4xG5E0UEb9YKm5W4c93v/HexoPNX7v6QOPYuOBiAZSdRjk0NjeZoBWg7RWaqirER2K7PiSR6xN5rsOY1ZTdFbCElLM8/UaLVN1/mk5vvQRyf1sSctdzdb1Qcq4R5Y122avGPks3LlSjr/8U+Mhj3fPBV8X1PS9am20DrtsLCDgRQKOPyO9KFudzsqXv/xlGT688fd5y4tp48y2UkmJMae30XBTXxqUrahw/ZB206ZNZgBWG35qILh///7m64orrrjgNtrI9MCBA7J161bZs2fPBfv9bYO/fr/87XPylvI4fl9Cgpr+Hu3J/Gm9Q+sfjun2S8cZjUTHmZte+3SP/OKl1Y67my3rSDlv/vdtEh7W9LeC4wEPXD9N9KXpqXe3yJ/f3GTv/ufqPeboQA9eP11mjB5gBlVHDEwSfTkm7cX76bajsnRF87+X9h4/J6+t2iM3zWvM//ghqaIvTTpsclllteNlemw5+PxnrQ1k/CE5fo+9te4x5t/fkISx8y/g1t6vo37Q9Pf7mRUvyOGnv3vBcbrh+JKfS+8pV9s9ZnXUHmvknuOv/ExOvvNH87y6yjLZ9Z9Xy4Drvy/aKFTrENoANMV4Oaa6ihLJ3/pBszltdX/Z0S8kftRFRhB1oAy64z/Nl+N5ulyetdtolPp90aBxa0kDsxrkdZZ0+pS89W/KsZf/w6zrODvGW7bVn697OH7XvCVv5AMBBHxPIMgIjnjHX12+Z0eO/VDg448/ls8++8zvSqY9VbXnrGPSIcP0Iaarl+7XBxjV1dWSl5fXocCpXluDs9pqXnvf6h/8+geM9tLVVvCHDh0y3x3z5c/LWv4f//jHpoE/l5OydZ3AM888I8ePH5eahmD56HSaQ9iu6+7RnitpQDTOeLlKdfUNkltY5mq3uT3V6KWqP1/aSjW1dZLnMByY4/E6LNi0Uf1FhyeOiQw3e9xoa/dDJ/PNAHFpheuHHf2MoOylxrBokeFhUlBaIbuPnjVarevPOO/4c2hKYr70i6o0jX70ox9JRESEY9F9cvnRRx81H6z3qsqRmSf+4XVl0FbqIVFNvbRcZbChzvidWJDjarfZ49XZkF/11VVSU5x74XnG/4Noo8V8wqiLG+8fHGq0Ei81WqzvkdJDRrDBeJjiNBnnhSf2NVrBjzaDuyFR8cZhDVJbWiDF2gP31H67p27L86P6DZdJv91obq7OPyV7f3Or6FxSYfFJEhweJfVGq/faskIp3LZMqjoxX23L+3bn+qE+M+VQ4nTzFvfff7/ZMKw778e1EehKgXfeeUc0uOhvSXu+tnxo6azuocdYdRE10DqCBmJ1tJ2OPKbQv7W1Z6y+EhISzLqH3levW1BQYNY9tG4TKElHMFq0aFGgFJdydlIgNzdX/vSnP5lXOVIWK7uL9O+Lnk29YiPN3qmucqHzz2pPWldJhy1O7hXtanez7eVGb9ji89OoNNthrMRFR5iNUVOMa0UZo/TU1tWbx+7PypP9Week2qi3uEray3fWmAHmEMo66o82OD1hjB7kDU9iw4PrZWFajmjNbNy4cXLzzTe7KobPbNfREn7xi1+Yz7KSyo7J1NNve13ew3qlSnBo0zytrjKovUdrSvJc7TZ73eqIOy1TrTHaj7MRf3T6k7jhM8xRdnQOWqMCLLUVxVJ2ZJuUGqP9aO9XZ0nPizB61eqww5GpGUZP2Rjj2BqjXnRaCnd9KlqncJWSjR6xw+5/0txd8MWHRvD3MbMOE2r0xtURgrQHbbXR8FTnmHWcJsbV9Xp6e4Pxv+XTwfdJVWiMWefQugcJAQQQ6IyAfzSN6owA5yLgIJCZmemXwVl9yNFai3MHgi5d1J6wGtjVF0kkPT39ggdVuCDQmkBGRoYZnA0Lqpf+URWSXRHV2uHdvq/MCHrqqzPpTIGLgFM7LqoPQLRn7dqd7Tjp/KGnzpXIi8t2tP9ED5wRGVInaVGNw9D262cMa+sHgVll0++x9lAqjEiT4sgUia886wFN92+hwUh9dTZpMNVlQNXZxY2ncuUn9povZ7tdbjPO04cg+irc/onLw9raEWQMR6ZDn+nLV1N9UKhkx44ys68N0VoOn+qr5SLfgSOgdQ9/DM5qI8yeSNqwNCcnx3z1xP297Z76+5eEgLsC2qhBhwDXnuv9o8plf3Gc1Da03aDS3et35LjWAq/uXE9H1NGAaGdTiRG0bTlnrbvX3Hs8V/TljSk9uswINTUm/X3kD0kb/ehzFx15IS9mkJSHJUh0TZFXFU2H7u2KpA1RvwN+AABAAElEQVQz9eVuaqivk+J9n5svd8/R4/S8ypzD5qs9511wrNHjtOTABvN1wT4f2XA2JsMMzGp2+R3rIx8a2UTAywWCvTx/ZA8BjwroH6R9+vTx6D25WeAITJ/e2LMncEpMSTsrMGXKFLNHiV5nTEKRRIfUdvaSnO+lAjqc8aTEQrGGNfannxeOZdmZcrnUBvt+b2Av/Rq1K1vaCt7X056kuVIZFmcWY/LkyTSA8vUPNADzP3LkSDMYEoBFp8jdLKBBNp1mhoSAuwLay3zatGnm4RFGj8rxvQrtwJ271+A43xFICKuWEXGNgevIyEiz56zv5L71nFrfY+3luCN1odQZo9OQel4gKMS3P4fK0FjZm7zAhNSfl1OnTu15VHKAAAI+L0Bw1uc/QgrQlQL6C5ahn7pSlGtZAjqv7+jRo61V3hFwS0CH5Js7t3GOSx12ak5yXuOQt26dzUG+IhAfViOzk/IkKbyx1+zAgQNlwoQJvpL9NvOpDZ+sn38l4X1k/YCbJT+yX5vncQACrgQ0IPtF2tWSnTDWPCQ+Pt7p3Pauzmc7At4ioD18rrrqKm/JDvnwIwGt07YcWtqPikdRuklg9uzZ5nREenkdtWd6n3yJCXU9ZG83ZYPLdqNAkNEgdFBMuczukyfWfLM6R3fLabC6MQvdfmmtd1g9gQsj+8qGfjdJUWTjfL/dfnNu4HcCOvlRrtFjdv2AW6TSGM5Y06xZs8wpFMwV/kEAAQQ6IeDbzVY6UXBORcCVwNChQ2XhwoWybNkyV4ewHYF2CeicuzfddFO7zuFgBCyB+fPny9mzZ81hYSOC60TnJK2MD5bCmnCp7+Ghxqw88t4xAX04og+8NDhrJZ2n+5ZbbrF7TFvbff39+uuvN+cQPHXqlJSG95aNA24yhhgrlLiqPKNNe72vF89n8p+Y1E8mnc9tXXCYbEvzvaBQRWicMTx2qvG/p3EgPu3tcfvtt4u+kxDwRYGxY8eaw/CuXr3aF7NPnr1QYN68eXajKC/MHlnyYgEN0N1xxx3y3HPPSVlZmaREVMolKVVSVBMq5XWhXjFPqhfzeX3WQo2pcnqF14g2+rWSzk2tozX5U9JOF/r8Rb/HOpeyTquybsCtElttNDaoKZCghqby+1O5vbEsIxImyLDzGSuJSPK5uodOoVIckWwEZY05es+nYcOGyWWXXWat8o4AAgh0SiCowUidugInI+CnAjt27JB3331XKisr/bSEFMsTAtpqc/HixRIdHe2J23EPPxXQ+ZtXrFgha9asEV0m+aeAVvQ0iKlDEfpj0t+n7733nmzfvt0fi+cTZdL55L7zne+YedW56B999FGfyLerTKalpcmNN94oqan0hnBlxHbfEdC5Zz/66COpru7c3O6+U2Jy2tUCGljTHnDWkJ5dfX2uFzgCeXl58vrrr8uJEycCp9ABVtLQ0FAzwKS9pf01aQODt99+W/bu3euvRfT6ck2cOFG+9KUvmfk8fPiwvPDCC16fZ1cZ1KD/zJkz5fLLLxf9/0NCAAEEukKA4GxXKHINvxXQB5dbt26Vffv2mT3XeFjitx91lxUsODhYdHjF9PR0swWqvpMQ6CqB/Px80Ye3WrEpKCiQ2lrmoO0q2564jlbwNBCrwxhri/WMjIyeyIbH73ny5EnZvHmzHD9+XIqKimhw4MFPQIOzDz74oHlHDZb/+te/NnrB+E47Tf0/o42d+vXrZw79PWrUKL/rZe7BrwO38kIBfZCsPx8PHDhg9vapqWkaWcELs0uWvEAgLCxMdJSe4cOHm/PfxcQ0DrnoBVkjCz4uoH8f6M+ibdu2SXZ2ttmT1pf+ZvBx/m7Jvv680L8FdbQ4bcThrw1CW+JpnWPLli1mY4Pi4mLqHi2BunFdp+rRxseajhw5Ii+++GI33q3rL63TA+jIVlpP1/8z+vuWhAACCHSlAMHZrtTkWggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggIALgWAX29mMAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIINCFAgRnuxCTSyGAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAKuBAjOupJhOwIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIINCFAgRnuxCTSyGAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAKuBAjOupJhOwIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIINCFAgRnuxCTSyGAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAKuBAjOupJhOwIIuCVQXFzs1nEchAACCDgT4GeIMxW2BYpAWVmZ1NXVBUpxKScCCCDQKYH6+nopLS3t1DU4GQEEAluAukdgf/6BXnq+/4H+DaD8CCDgbQIEZ73tEyE/CPiQwL59++QPf/iDbN261YdyTVYRQMBbBPbv32/+DNmyZYu3ZIl8IOAxgZKSEnn66adl6dKlBGg9ps6NEEDAVwU0MPvqq6/KU089JUVFRb5aDPKNAAI9KLBixQp5/PHHJTs7uwdzwa0R6BmBHTt2mHXv3bt390wGuCsCCCCAwAUCIT830gVb2YAAAgi0IaCBWeuBsgZY4uPjpV+/fm2cxW4EEECgUUB/bixZssQMSh04cICfIXwxAkpAA7PPPvus5OXlyblz5+TMmTMyevRoCQ6m3WRAfREoLAIIuCVgBWb1gXJlZaVoPUR/ZkZGRrp1PgchgAACy5cvl1WrVkltba3s2rVLMjMzzfoHMggEgsD27dvl9ddfF/19unfvXklOTpaUlJRAKDplRAABBLxagOCsV388ZA4B7xRwDMxaOSS4YknwjgACbQk4BmatY/kZYknw7u8CjoFZq6wEaC0J3hFAAIHmAo6BWWsPAVpLgncEEHBHwArMWsfqlBIEaC0N3v1dwArMNjQ0mEXVdwK0/v6pUz4EEPAVAYKzvvJJkU8EvETAWWDWypoGV+Li4uhBa4HwjgACFwg4C8xaB/EzxJLg3V8FnAVmrbISoLUkeEcAAQQaBZwFZi0bArSWBO8IINCaQMvArHUsAVpLgnd/FmgZmLXKSoDWkuAdAQQQ6FkBgrM968/dEfApgdYCs1ZBNLgSGxsr/fv3tzbxjgACCJgCrQVmLSICtJYE7/4mUFxcLM8995w5lLGrshGgdSXDdgQQCDSB1gKzlgUBWkuCdwQQcCbwySefyOrVq53tMrdZAdqMjAxJSEhweRw7EPBFAVeBWassBGgtCd4RQACBnhMgONtz9twZAZ8S0GFPXnnlFbGGQmkt8xpciYmJIUDbGhL7EAgwAW3c8fLLL7v1M0SDuPwMCbAviJ8XVwOzTz31lBQUFLRZUg3Qnjp1SsaOHcsctG1qcQACCPijgAZmly5dKnv27GmzeBqg1eFJx4wZwxy0bWpxAAKBI/D+++/L2rVr2yywBmi3bt0qQ4cOJUDbphYH+IrAF198Yc4x21Z+9fmezufOHLRtSbEfAQQQ6B4BgrPd48pVEfArAQ2qaGC2PYkAbXu0OBYB/xbQYKsGZtuT6IXfHi2O9WYBDcw+++yzUlhY6HY28/Pz5cyZMzJ69GgCtG6rcSACCPiDgNVj1p3ArFXe6upqc/68UaNGEaC1UHhHIIAFtMesO4FZRyINUNGD1lGEZV8V2LZtm7zxxhvtyr5+/1NSUsxXu07kYAQQQACBTgkQnO0UHycj4P8C1lDG7vSYbalx8OBBer+1RGEdgQATsIYy7ujPEIZJD7AvjJ8V1wrMarC1vYkhjtsrxvEIIODrAlZgVh8StzdpD1od6YcAbXvlOB4B/xJoayhjV6XVHrQEaF3psN1XBDoSmLXKpr9D6UFrafCOAAIIeEYgyHhY2uCZW3EXBBDwNYHy8nJZuXKl1NbWmlnX9+zs7Fbny4uLi5PMzEwJDQ01zwkKCpKLL75YEhMTfa345BcBBDopUFFRIStWrGj2M+TEiRPSWqBKg7FDhgyxf4YEBwfLnDlz+BnSyc+C03tGYM2aNc1+Z+bl5cmxY8dazUzLoTlHjBghI0eObPUcdiKAAAL+IKCjZujDYStpj9idO3daq07f09PTzYfJ1k6tc2jdQ+sgJAQQCCwBbRTXsnGH9qDV7a6S/o2lPWatFBERIZMmTeJniAXCu88I6Pd81apVog2dNOnzu6NHj7b6/e/Vq5cMGjTIrnuHhITIggULzE4WPlNwMooAAgj4sEBj9MSHC0DWEUCg+wSio6Nl0aJFzW7wz3/+s9mD5mY7jZU+ffrIjTfe2HIz6wggEIACUVFRF/wMee2111oNzvIzJAC/KH5cZG1Y4Jg06NBWcFZ/72ojBRICCCAQaALDhw8XfVnJneDs9OnTZfz48dYpvCOAQAALxMfHy6xZs5oJaE/C1oKzGphqeU6zC7CCgI8I6Pf/2muvbZbbv/3tb61+/9PS0nh+10yMFQQQQMCzAsGevR13QwABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBAJTgOBsYH7ulBoBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBDwsQHDWw+DcDgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEAlOA4Gxgfu6UGgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEPCxAcNbD4NwOAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQCU4DgbGB+7pQaAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQ8LEBw1sPg3A4BBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBAJTgOBsYH7ulBoBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBDwsQHDWw+DcDgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEAlOA4Gxgfu6UGgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEPCxAcNbD4NwOAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQCU4DgbGB+7pQaAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQ8LBDUYCQP35PbIYCADwucPXtWSkpKXJYgMjJS+vfv73I/OxBAILAF+BkS2J9/oJe+rKxMcnJyWmUYPHiwhISEtHoMOxFAAIFAENBHFUeOHGm1qKmpqRIbG9vqMexEAIHAFcjOzpaqqiqXAElJSZKQkOByPzsQ8GWB06dPS3l5ucsixMTESFpamsv97EAAAQQQ6F4BgrPd68vVEUAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAVOAYY35IiCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIeECA46wFkboEAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggQnOU7gAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCHhAINQD9/DaWzQ0iBSWVUt5dZ3U1RsrJAQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8JhAcJBIeFiw9I6JkNAQY8XPU0AFZzUYuyOrQFbtzZVdWYWyO7tIispr/PwjpngIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIeLdARGiwjOwfL2MGJMjUIX3kkjGpfhmsDWowknd/FJ3PXVVtvbz/xSl5Zc1x2XOyqPMX5AoIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIINBtAsnxEXLzzHS5dVa69ImL6Lb7ePrCfh+c3WH0kP3RK9vlWG6Zp225HwIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIdEIgJiJUfnjdKLlh2kAJ8oNRj/02OKtzyD7+4QF5ZuURqff/zsGd+EpzKgIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAALeLTB3VLL81y3jfb4XrV8GZzUw+8OXt8mH205797eI3CGAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAgFsCg5Nj5Ln7Z4oOeeyrKdhXM+4q3wRmXcmwHQEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAHfFdBpTO99Yr3kFlf5bCH8Ljj72Hv76DHrs19HMo4AAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIICAawEN0D74zGaprWtwfZAX7/Gr4OyWo/nywuqjXsxN1hBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAoDMCe04WyVMrDnXmEj12rt8EZyuq6+QnS3ZIg28GyXvsC8CNEUAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEPA1gSc/PiT7ThX7WrbFb4Kzb27KlhN55T73AZBhBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBon0BdfYM8sexg+07ygqP9IjirvWVfXnvcCzjJAgIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIeEJg5e6zcqqgwhO36rJ7+EVwdsOhPDl6trTLULgQAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgh4t0C90YNz6edZ3p3JFrnzi+Dsyt1nWhSLVQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8HeBlXt8K07oF8HZXSeK/P17RfkQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQKCFgK+NruvzwdnaugbZd5LgbIvvIasIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAII+L2AMbKxTyWfD85m55dLVW29T6GTWQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQCDwBnw/OVlTXBd6nRokRQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQMDnBHw+OFtX72N9lX3uK0KGEUAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEECgKwR8PjjbFQhcAwEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEOhuAYKz3S3M9RFAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAFDgOAsXwMEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEDAAwIEZz2AzC0QQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABgrN8BxBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEPCBCc9QAyt0AAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQIzvIdQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBDwgQHDWA8jcAgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECA4y3cAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8IAAwVkPIHMLBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAgOAs3wEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEDAAwIEZz2AzC0QQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABgrN8BxBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEPCBCc9QAyt0AAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQIzvIdQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBDwgQHDWA8jcAgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECA4y3cAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8IBAqAfuwS0QQAABBBBAAAEEEPArgYtGJEtGSozLMn2w7bTklVS53M8OBHxB4M45gyUoyHlOyypr5Y1N2c53shUBBBDwAYGJgxNl7MAElzldsy9XjuWWudzfVTsyU2Nl9vAkl5fbfrxQdmYVutzPDgQQQAABBBBAAAHfEyA463ufGTlGAAEEEEAAAQQQ6GGBzJRYWTy1vwxKipHoiJALcvPFsQKCsxeosMHXBC4akSSDkmNkQO9oCQluHqU9mV9OcNbXPlDyiwACzQQG9I6Saybr7/JoiYsKa7ZPV75f/IVHgrO9Y8Pligl9Jb1PtPSJi7ggH3/68ADB2QtU2IAAAggggAACCPi2AMFZ3/78yD0CCCCAAAIIIOBSQB/w3XHRIJf7ne04U1Qp/1iX5WxXs23zRqfI+PRezba5WtGeJxqs9HSKDAuR710zUuaNSpasvHL5/bv7ZO/J4i7JxoufHRV9aa/CJMP5hQdmSbrxcNfddMmYVPnmZUPNwO7rG7Pl+VVHpKHB3bM5ri2BMUZPqO8tGmkEFaNk5e6z8tj7+6Wqpq6t09jfQuCBZzabW0JDgsz/7397cFaLIzq/mpoQKbfMSm/3hXafKJIVu8+0+zxOQMBfBJLjI+S22W3/js/WhhTG75muTtOH9pEZxqu1VFZVK8+uPNLaIT26792tp0Rf+ru8V3S4PPaVyTI1s7fH87T5cL7c9fg6876xkaHyL1ePcOuz9XhGuSECCCCAAAIIIIBAlwkQnO0ySi6EAAIIIIAAAgh4l4AGJxeOTzOChjEX9HpzltOSihoz2OFOcDbD6E13w/QBkhIf6exS9racwko5fKa0R4KzDy8eZQd9+hs9//769ely2S9WdmmQTgOqucVV7brmsLQ4+cM9kyX4/Hix3zcCyIXl1d3y8Nz+IAJoISYiVJ76xnSJP98L6ssXNw7N++ibewJIoWuLWlvXIKcLKrv2ouevFm18XtpjbKDRY6xl71xnN6w3/tNpI5KK6jqCs86A2BYwAtHh7v3f0f+/n+3NlXNdONS+/v76jxvHymDjbwFXqbyqTrYdL/Dq4KyVd/1dXlBWLTpce0+nUiMP+aXVPZ0N7o8AAggggAACCCDQzQIEZ7sZmMsjgAACCCCAAAI9JaDDjl7769Wivd50WNIhxpxmf7xnSrPsaG8NHS7vaG6p+WDS3d6bz686avT2PGr2/NShfeePTpUHrxhmXvuvnxySZTty5Pi5MjOA0uyGHly5ZGxqs7slGsMGTs5IlHUHzjXb7umV+WNS7MCsde/LxqYRnLUwOvk+xej1ZAVmrUtdavgSnLU0vOv96NlSueZXq+yfUyP6xcvv7prULJObj+TLkx8flOz8CskprBANNpEQCHQB/R3r+H9nSGqc8Tt+8gUs+jeA9k7/87KDF+zr6IYFxu8xZ4FZbYz1yzd2G39TlBkNlyoZEaKjwJyHAAIIIIAAAggg4PcCBGf9/iOmgAgggAACCCAQ6AIayDhmPCg9YQzt2zKdMAK4W47mt9zs9rr2jNGhgmMjm+Zq0yGM953qmuGD3c6IkwO1164OOeyYdFtPp9NO8nDaCDiRukZAg3ctE74tRbxv3fo5pb1iW6Zs42fX+oN5LTezjgAChoD1f+ek0XjBVdLg7FPLD0tNXb2rQ9zeroM+fO2SIU6P15EkNhzi/6pTHDYigAACCCCAAAIIIOAgEOywzCICCCCAAAIIIIAAAn4j8Nt39ooGj630t9VHRXvp9XRatv10s0DTqYIKeXrF4Z7OltfeXwMBOoSmvruTDpwukVfWHrcP1SEif2fMN0xCAAEEAlVAGyrpNAddkaZm9pFxbs453xX34xoIIIAAAggggAACCPijAD1n/fFTpUwIIIAAAggggAACokOhXv6LFaIPkrPyyuSgEbTzhlRdWy/f+OtGmTQ4UWIiQ8xArW4jORfQHlrfvWqEfOe5LW7PMfoLY1jN1zacMIbzjpKNxtDdOp8yCQEEEAgUgV0nimTswIRmxb1jzmB574tTzbZ1ZOWrCzLN03RY5Xhj1AydMoCEAAIIIIAAAggggAAC7ROg52z7vDgaAQQQQAABBBBAwIcEisprZPmuHK8JzFp09cbkvjqc9Oq9uUJg1lK58D0iLETuvjjjwh1ubNlvDK29fNcZArNuWHEIAgj4l4DOra7TGTimCYN6XRCwddzvzvJIY07oOSOTzUOf//So1Lk7Ub07F+cYBBBAAAEEEEAAAQQCSIDgbAB92BQVAQQQQAABBBBAoHsEeEDdPa43TBtAr6zuoeWqCNgCGSmxZk9+ewMLtkBsZKhcMaGv28Oq2yf28II2AHIc3t3Kzp1G79nOpK9e0thrNq+kSt7enN2ZS3nlufwu98qPhUwhgAACCCCAAAJ+KUBw1i8/VgqFAAIIIIAAAggg4EkBfRBO6lqBkOAguXd+YyCga6/M1RBAQAUyU2Pl13dOlLd/MFd+Zbzr/zlSo4AGZb9x2VBZ9uMF8ru7Jsn49ESfo3nLCJ46zruuBbhyYl/pY8w/25E0oE+0GajWc1/67JhU+eFw/A38Lu/IV4NzEEAAAQQQQAABBDogwJyzHUDjFAQQQAABBBBAAIHuFxiUFCMTBveSxJhwqf3/7J0FfBVX9scP8QRIgktwCO7uBVpKKfVSp0a9W29X/t1tt7vb7na3vpWtu7tAKbQUirs7wQkuIUSI8j+/gXmZN3mW5CV58jt8wsybuXPnznfmzby5v3vOKTopmzRn7FLNI1vVQmichtZt17iWpDauLUnaFtiJ/CLJyS+UnYdyJG3fcTmWHRg5TSNUXOnWPFm6tUiShZsPy2ZtmzuroTpMc+1sb98kUVI0N2sNXZCTVyj7j52QBRoSsyId76i7aZ0EaaviD/YRGx1hhG8GtxMFRbqPPIPb0ex8d80zRAC0qzqtruZSbKfnvXXDmpIQEyWQ4I9pm7fsz5K0/cdLCR8Vbat5/lL1eqsVHy25eo0tSjsi2w5klbnqCD0JzerFS5PkeImPidS/KImMrCF5yh8hn3Ht0koTiI2KkLZ6zpMTog1m8bGRcrL4pBw/USDLt2f4PUw2hLK+bepKg8RYqaH/9mbkarjzAxUKdw4RDXXWjI2SGD0e3KvSj+TqOc+W3Tpt1aCm3K7C49ieTQ2P0Gz93j/4wXIp0uOsbsN3oLW2DzxwzSbotVtDl+XmFcmaXRnG/akibYToOqBdfaP+OK27SJ8t+4+BTY4ghyoEauRmveGM1pKo30HYy1M3y8odR33aLa6f1Ca1jb/khFPPLvBfvztT1u85JlWp/WWdKBQItFcNaeloe3RkhFw2sLm8+nOaY5mvMzee0UZwX8H18un8nb5u5lM5PDNwr8Izo4U++/HMKNbr8WCmPiv0frtV/wqKKpabHd+FNuol3k7PT329t+OZh+cRnku7juizfG+WZFTgWY5jwO+WDhr6uUlynODLhWfq3qO5Rm75irbfJ5AsRAIkQAIkQAIkQAIkEDQEKM4GzaliQ0mABEiABEiABEgguAmc3ydFfnd2qsuDmPDSfDmkYRJb1E+QC/o0k3N7NTXm7YV3Hc6Rl37aJJOX77GvMj5DdHj15n4u12HhlS/Mk4wc94KgdUOIsdcOb2W0BQKtJ/NF1Hjy6p7SU3P+ubJP5u2Q937bVmrVo5d2lcHt65danqudyRc/M9uxvLF2BE/Q3KwX9UtRUemUgPx/n6x0Kc6ic/0iDRd87bBWhoDgqMQyg873XzVf68vTNstuZe6rQfi4cnBLo36cC2+G0Jg/r94nj3+91igKYWNAan0Z1bWRnN29sdPmfx3fVf5wQSenZR+o99ZHc7Yby24Z1VYuHdDcab35YUHaYXnsi9XmR49T8Dm3VxMZP7CFIXC5K4xcwb9o2+FBtmpnhrti8ueLu8iw0zkarYXStcP+plcXGuJYr1Z15MK+zeTMbo0c589adtm2o/Lkt+tkXfox62KX89erqAR+yA0Jcc6d4buEnLz/03OM8x3OVltFuIcv6iwdUxKlrYo3EAhdGcSiFTsy5PP5O4x7kCuhDfeNF2/s42pzeeKbtTJ7w0HjHOMaO0/vid1blL4nIFc2QtK+oufG18EojZLijO/eiC4NjYEkLhugC3HvwP1ML3PDIE7d+eYSWW25hnu3riP/vLKHyypwzT89aUOpdUM6NJBHLulSarm5YPxzcwRioTu7f1wH6de2nnRQ4Qy5pt0ZBpz8tGKvvDVjizFox1056/IoHZRwSf/mcla3xtJf94HPrgys8b223u+xn1d/2eyquNMyDOS4We9BCMWO68mVQQD+ZtEueVfv9VUl1H2s15FVnEW7rhjUUt781Xd+2AaDCC7u3wyz8oUKs8dz/TMgCef6cr3X4t6NQVDuLFP3N2nZHvlg1jbBvasshkE+1+rz8cK+KW7PjVkfBoKV1SDoXz6ohSBktLvnHnjhWfeSCv0HdAAUjQRIgARIgARIgARIgATcv62TDQmQAAmQAAmQAAmQAAn4kcCybUcMb5seKkYM7lDfSTjC5zM6NXSETHS3W3hhIvwmOranrdpXqtiBzBPy2i9phucKxKm+bes6lanhQ1IPiBbXD28t953b0akTH+IFRDiIG/A+ghcOvLuQrxGeld4MHdrwgES7+qhXGzrzTTM9tMzP5vS7JemGaN1NmQ1MrWfs11yHNiTXjJbb1AMOHdv47M3ghQx+VsF3+8Fso9MYHpU4lrEa9hKiHsR0iBm//3C5zFx3wFvVMkhF5H9c3l0gFJsGb6ff1AvwuIoyUdqB3VIFW3j0QUiCocMfPEz7z4SecmZXZ1HWXOcqFKeV24x1+43rAvX1UnEppW6CuanU13b4Yg0T4+SJq7rLIBWITUNH+k8r9woEM4jH8Io6o3ND4/xjEAG8D9/9bau8MGWjS7Fo0rJ0Y1t4gQ9Ujz1TFMO1g/Nw79gO0qV5krk7l1OIZW/d3l8ue36uV7Ecwmyf1iXXPTzOF285LIez8g0P2nM0fya+R/iDd+AQ/e5NVJH4qK4PV6t1+no3jx/fhaUqiK/ddUxydb65XktjVUyFaIdzgb/OzZLkqR/Wl/KEhOfr2zO3GqIrvhPmtY66IcIPaFdPrlIRB9eSO0tSr93bR7fT/UW4FELt2+He8LSG3jUHZmCwCLxvF+qghLyCYsMDFMI/hGB4UZsGIfLut5fqsR4xFxnTbQeyDQER12U//b7CE9e0urVch8SFN/ZbM7YaAz7aq0ANRlaRG16KnmxcrxTHvQMiKdjjHECUS1Kxc1zvplJf7xcQv1PPqW1w/N3bS4z7sad6cZ/97w19pKeyNw1txfMDnvu4b+K84HuDe7pVmMXAj+d/3FjqHJv1mFOcU4SHtt6jMPAB+8Fh437ROSXJGHB077kdZIzeY295fVGVfOfwzJmvkRBwLZqGe8/obk1kygrXg5zMctbpBB3MAz4QL9/XASn+MNyrwQ3htU3D/Wr2hgM6YKTI8Jrur2xx3eJef7V6AF+iAvG/v1svXyzwzXMXAyAwaMA6UAXPJXw3MCgl6vSzvJ5eJxBW2zaqbTbFpymeGfju4Xo3Dccwc+1+ydffKYhMgYFGEOwxQGC0PlPve2+ZsX+zPKckQAIkQAIkQAIkQALhSYDibHiedx41CZAACZAACZAACVQ5AYTVfFdFC9gLN/R2EuEev6K7CpfZhjfadp0ihChEUISSHdW1oeFRZW3wI5d0lV+08xuebFZDfr1vFu92LFr0xBhJ0LCkZTGIZfCAMg3eTs+op9iva/e57aT/5sFhbr1QzXoggJgiCPZxy5kl+zDL2KcIpWmG07TvA15MqAdCji8GDyV4FZtCIAQceKx+rZ5cVs/fN6an6fnpY4RnhJDzzHW95Vr1bF63273X5hgV/J5SYRVep7A96hX6gooaU1TUtJ8jFEGH9d8u6yZdbaLkE9+sk+cmbzTqwDVhFVTQ1gWbDxnrzP8gmJqWtk9DDesfDHkVn57Qy1zl0xTC9bt3DnR4bKPjHm2Bp5s9xHMn9bB8YFxHQ/DA8SA3LkS4P368otQ1slI9LfEHm/+P0Q7PLZyPFyf2lU0q4Hyuwj1CX2Kf2BfEk3Eq/EIsNw2d+9jnA+8vMxd5nMLD8V/qbYsw1VZ7UT3P4RV5nopdMIhd8Gg3vZetZcNx/u9frZFv9R4C4dJqL03dJJ/dN8QQCLH8Oh3AAcbwarYavENxPvEHL+bfn1/i7Y0cpvhuzN14UEOoajh0FR6ztTxETHh0QkC1DrK4Qa+rLxfuEgygcGcY7PDKTf0M4QxlUPbm1xbKvgzn8w4PUAw6ePmmvo6yKH9EBUq7QbT8dN4OYzGuE3j9ezNEPrAKZr8+Mkoanh6E4W1b63pwwzVqD3mOwQ+v39LfMeAGot3VQ1oZHrTW7a3z8OR/+/aBDo9M3Ofw/YFwajWIsBBx37ptgNN9HG1w5R1t3baHRkN4Re+rptiOkOcPf7rSEJfNcv+blmZ4bf5dB6/gfgFR8mX97l/38nyXAzrM7fw1xfFZxVnUe83Qlj6Ls+B4lUZEgP2wLN0vnp8Y5PLe7wY6RFP8PsBgB0RssHqLg9dQ9cp+4LyOxr0K4jmiKNSO13OrgwE8GbxZEX3CNFyj/52yyQj1bH3mmesxxXmBmO6Lgcubt/V3iMsY1PHXL9fIj+rhaz0GCPMvTuxjPCNwH8fz9eoX5xmhmn3ZD8uQAAmQAAmQAAmQAAmEJgH3w3VD83h5VCRAAiRAAiRAAiRAAgFI4P73lsuFT82SP360Qv7382YjVOxXKkp8MHub3Pi/hfLI56ucWl1HO9LhgetvG9mlkZMwC4F44qsLtDPfvTDr7zb4Wt9f1BsIORghWtyqXlij/vGr9P2/qS5DGaPOu8ekOoRZfEYnNcQUeyc1BM6HP1mJIoZBdPjThZ0dHp/mcnOK/IpPXt3DIcyivrvUow2hp+3CLLaB2AGPMoRttRu8VCEu4Q/srWZdZ5axCzjW8mWd//vl3RzCLLZ99PPVhkBlF2axbn16psBrzxoKFl60COlcFhvxt+ly1X/nCQTBNzTMKEKQ4rpHPsgLn5ot36vntNUgUsB7zReDwGUXZrEdzsl/vl/nVAWEYFNccloRZh/g8QZx0C7MAgNY2sWgi/u5DqPtDtv7GpL17CdmyD3vLJX/6vlBKHOIr9jnQ+qhfs2L842cwNbtcU/yZHePbe90TWCAgF2YNbefr4Mb3pi+xfxobHfTyDaOz9U9A0ELnqquvtc4J09PWu/URAxQgXjnzhAq3BoqFx7udmHW3PaIeo4/qOfAahik4+l7ge/iUzoIxCwDce7ud5Y4CbOoD8eFQUMIX28avEHhSVkVNktDaUP8tBoGvsD72xe7TMMOQ4iEvXN6gJUv27krA27PX9/bIczi3OKZgcEOVlET2+N5gVDgt7y2SBAG37T7NbKF1VvVXG5OIYgipLxpqAf3bPtgJHN9eaYP6cALq9cvBsNMWppe6hgQjv6vlrD6YPmQis00EiABEiABEiABEiCB8Cbg25t9eDPi0ZMACZAACZAACZAACVQyAXuHrH133yzaLcu3O3uoDdcwyP40eK89oHkPrYacq+6EDmu56pi/8ZWFctsbiw2xZZ6GrYR4ifyRL/1UOj8iPDERStU0eO+9N8u915HV2xPboBO8i4uOfAgjyANr9fZ7XT1vIXJ5s0UaVjJQDMdnFcGWbD2iYU/3emweBIW/qZcUOv1Nu3N0qkNwMJd5mroSr83y+E4gR6lVKIQXuCdBwtwW53erhjN1ZxCiTC9jlIE3V+/WJaGQ3W0X6ssRCtiT2T23kd+1LObpfKMeeKdDoLcawr27MwyMsIawhoDsycMd9fyq4VathnzI7nKwWstVxfyK7RlGGGN3+1qngyKseWsRhrZlfdch5XFvunJwC6eqpq92PnanlfoB4W4x8MM0eGnCQ9edQVxtWifesfqjOTsEkRbc2Yc62Mhq8L6uCsN1ZxWGzX3Ce9abQUg12zlDrx0wqqghDD/Cqpv2meZwRi5hTwav12dPR1VAOZxfRBLA1JXdp89y5II1DZ7gCJXtL0Oo70s0v7BpuJ9+rb9T3Bm85a3XFn6/uMtP664OLicBEiABEiABEiABEggtAhRnQ+t88mhIgARIgARIgARIIGQJoGPYasg/6E8b2bmRUxhZCB32znR/7q+idSH0sytD+GV4xCL3pWl2z0jkIETuQE9mF3mseVjN7eD9ZV2OsLxWzzyznKsp2udJyHC1TWUtQ/hZq/2oXr9W0dW6zjq/QT2AzbDTWA6PbnjQ+svAE7kRrebtuv9g1nbDE9db+yHgWi3Rx/DY1m1CZR55TSGEw2vZk5VipqK2vw1hXa1WXwdWuDN4hlpt8ZYjXq/bzTpwAsdrGoR5fI+r2+Dp/fyPGzw2AyKj9b6Gwta809aNcUzWsODwxIcHozfDwAyrDVXx2p2NH+js+TpLc6V6sl2Hc0qJy9Yc3Z62reg6Izy7Dt6xGu5V1tzn1nXmPHKPY3APDDmF/WHI6W61yRoG2BdDSGXcE02D96+rQUMIG23Nq44BLvBU96dd2DfFKafyHBVfPQ0yw/14rS01wEBLbnN/to11kQAJkAAJkAAJkAAJBAcBirPBcZ7YShIgARIgARIgARIIewLwDLVaTLR/f8oO6VDfWr0s11yS3gRMpw0C5AM6geHRuUSFGtP6aj5Lq63QXLbeDKGHrdY/1bkOrBvdrbG1iKzameHk6em00sUHhBlFyM3qNHhXDWjnfO7tx+6pfWts3lgDXXDytL23dQczbdd9VKTHTRAadI4PTIuKi53qsXqZOa0Igw8QfBBmG+KZJ7OHAK8MZvZQ1LEe7nONbTld7fdIV8cCAelQZkl4WJSpW8u9AOyqjspY9t2S3aXy97raj6/nwJ7v9khWnk/3c/v3zZ14iVzfHTR3tmkQjr15ZuLebI8qYL/3mPX5e4r83D/YRFBEPBivIYvdGaJJTDwd9hqRK1bYole4287Tcnic4s80XI+bT+cKN5e5m4Lx+t3OzyXr4CBzuyEa/t1qGERz3DIgwbquvPP92jg/D62DdNzVaX+u+PtZ4W6/XE4CJEACJEACJEACJBCYBPzboxWYx8hWkQAJkAAJkAAJkAAJhACBImctye9HZA9fiXyuoWDoYO/bxjlkrV1QdHWc2w6UhPfE+naNapcqNswWcnXVjrIxQ67N139JK1VvVS5A3kUznyL260pA8dQe+3UCb6gId7E2PVXkZp1djHJTrMyLPftNl7m6sNgA10ZlW1nOt124LfKxgYUqclktUMIaW9vkbt7HQ5RYDUlsNV+fH3b+URGuu0wQTtr6NU/TcL/2PNnW/Zvz2w86hwW25sQ1y1TW9OM520tVfeWgFm7DWp/VtZEjbPRbmhPbHzbQFiZ6uz5nkKvXV1tj9z5t7zywBvUMsA2Q8eV55+v+UQ45hru3dPY29ybMY7tSz9TGpZ+pKEcjARIgARIgARIgARIIDwKu3zTC49h5lCRAAiRAAiRAAiRAAiRgEICXlT3/m70TOFhRtWlY00l8xHFkqheVN8u1dZjHxziLHRB97fke4TkbbGYXruE96YvIYh6nveMfHnVtG9cyV3NKApVGwH6dxqgnpC9mF3Vz8nwXx3ypPxDK5NqOCblTfTG7qJuTXxJG17p9j1bO4pwv91Rsbz9n9v1Z9+HveXjt2sM249l3ZlfnCAjYL4Tnm0a2NZqwRYXnWev9E+HAHsXB7kns7ZjtIihycFs92OENbM/Lbd/G2z68re+YkiT268kaKtzd9ic0tLbV7N9D6zrOkwAJkAAJkAAJkAAJhD6BqNA/RB4hCZAACZAACZAACZAACXgm0CQ5vlQBePSEgtWKK50XEzlW7SKB/Vjr2XL62juSEVLV7nFnz8lprzMQP9vDlh7Nzi9TMw8ddw4Ri42TE2LKVEdlFcb5aV6vpjStEy8Q1+HxFa1/EBaa1y0JLVpZ+w/WeuFJjXyldWrGGNwMZir6JMQ6D1Co7uNDuFZrjuPUJt498WrGRpU693Zvzuo+LlyfLRvUFNxj4szrVvnjPNi/r+7aCjZWa6R1IT+tNxGtvc2b0d1zoLbtvtpEv2N3n9PeukuX8/b8vnEewla7rKCCC+E9ax+Qcs3QVjJ15V6nmgeoh2uX5knGsndmbvWYT9VpQy8f6tR0fh5V9H4LMRb3tqwTp0T0ZK0/zuY1ve2As7eylyZ6XV07vnQ32i1ntvUaNjtFrxGrxXkJUW8ty3kSIAESIAESIAESIIHQI1D6V2XoHSOPiARIgARIgARIgARIgAQ8EnDVQX7C5jnqsYIAXhlv66hGU68b3rrMLa4h6kplMXiI2i2/oJJjT9t36IfP9o78Al/jn57ed0Fh6WOOj/HNS88PzS9VBUT1yzWP5NndGxsCo11AL7UBFxgEIJpdriFeB2uYVHu+0kBFZM8B2qtVHWPAhKdc2T00HCu83k1brd7uOw95zrVrlq3MaUrdeLliUEsZpaF0W9RPqHBo8D1HcwQDJ+qfHmQCT9Be6mX527oDbg8DXPrYQsD/uGKPy/L2wSpo/21ntXNZ1tNC+33VU1l/rJu+Zr8grzHEatPgadopJVHWp5cI2jeNOuU1i7KTbblqze3KM4XYbrWy3m/zXdxvcQ83xVn7/Rz78vezPNaFqGp6GVuPzet8ydfQa1EWIAESIAESIAESIAESCD0CFGdD75zyiEiABEiABEiABEjAZwL+6hu05t7zeecBVNBVaMmCIue8jAHU3DI1xZU499QP68VVJ7eniottyR6tAo+5na85L83ygTCFN57Vim35OK3rXM3b83eiTIyLzntX2/p72fBODeQ/1/RyCmO9Lv2YzN90SPZlnJDsvELjvENQhvhi9+Lzd3uCoT58Px46r5NMGNbK0VyIOQs3HxaInxkaAjxXQ9ua35dnr+vtKFfdM8u1fRBXu6mwDIMH4Tk9m8qkpekum4b7NARoqyHvc3XbRf2ayaOXdnWEisW9BseGc3Dw+AlB2GXwx3X7hws6SbN63r2+cbt697dtem47Og5v/IDmGp73gJFX2rHQMjOmexOn787sDQcl/UiupUTJbJR6bFpto3rqfrlwl3WRT/NbNWRwVRpy6n46b4fcO7aD026vVu/ZRz5bZSxDHu5Bmjsb9v6sbVJWAdXY0M1/8HS1Wlnvt0XFpQfDWIVyV+Ksv5/l9mcqrrV/frvWelg+zecF4WAmnw6MhUiABEiABEiABEiABHwiQHHWJ0wsRAIkQAIkQAIkQALBT8CmrRkHZM+bVt6jtNbjyWurvPVX9nbosLZblHpRlQ5Yay8V+J/zXHgafaUigulpVN4jcHWeETY32CzP5iEdqWJdWcwu0mBbV9dTWeosT9n+Gob05Yn9jFyR2B6iz13vLHHrFXmhCmI0kYcv6uIkWEKs/Pd368TV98aV8FOdDHFPf1Lb+vYdA42Q1WjLX1XkhIj5y+p9TtchPN1vVkH+rG4l+UV/Xbtfvl2yuzoPQc7vkyKPX9Hd0YZl247KA+8vM7xeHQstM3ecnWr55Hn2Ew3he4HW3/50uOeRXRoZ4u5rP29R0b0kfDlylmLdY5d1dVR4NCtfHvtiteOzfcYU683lu1XE/WTuDvNjQE9x/79TOVqF0nG9msqzkzYIwgzfNLKN0f7juQXy5YKyC86eDt4eXaGs91trm839WJ9FrgbL4FnuT3MVLSFYzr0/ObAuEiABEiABEiABEiCBihGgOFsxftyaBEiABEiABEiABIKGALyRIDhYBTR4WvnDkMfQtGPqaRZsZhfo0H4IztkhoM66OjZrjr7ynquDmSdKbWoPWVmqQAAusOfejbF5dnlrsnVgglnWVR5ac11lTR86v6NDmMU+/vTJSrfCbGW1IRDrhZhzo4pN63cfE3hCWq1No1pOwuwWFbQf/3qt3/JrWvdVWfMrd2TIra8tkhcn9jFyquK7/cy1vQxxc8mWI3rPLzJyIA/ScM3WaxUiNIRdV4N2Kqut9nrhgfigxbMVz6j7VZg97CKPs31bXz7jeXfDKwvkpYl9BaF7YdcOay1XDm6p3uSHVYjM0+dhpLHOGsoa+WohECOkrzuz31f99Sx1tz9/Lj+iwvOU5Xvlgr4pjmpxbcCzeNqqfRoSvYmx/NN5Ow1ve0chP8zk2gbDxESW7TeIPdIBrl9r3lr7eUGTY/yc19ceJhke6fhd5WpAhx+QsQoSIAESIAESIAESIIEQJVDSixaiB8jDIgESIAESIAESIAESKCGw42C2w4sIS7s0TzJy+9lD1pZs4dtcFw2DaNq2g1UbptHcb0Wmx08UltoceQ+tnb6lCgTJAnTE261pnQQ5mFkx5RlsIGxaRQl4qK3ddcy+u4D+vDfDOWypNRejLw1vprkm7ZZ+pGpzeNatFSOdU0q+gxCV1ms4Y5pIR82lec857Y3wrHZxdkj7Bk6IZqgnaUXvhU4VVtGHpduOyM2vLpJP7xssCJUKoQi5Vs/peUpkM5uxee9xmak5V+Exi3DI1W3tmyQ6csKiLSu2Z/hNmDWPLVO9P296daF8+cBQzcFc07hnYTARQoBbDZ6yv2nIY/CZrVNvQpv9vtq0Tun7gLX+QJv/aK56FVvEWbTvyiEtNd9vTWOQBzyDP1TPY3/b3qPO99uGSbFl2gVy+1oNg4SsXsyuIkK0qFfTr88lV78LmtZNkG0Hgu+3j5Ul50mABEiABEiABEiABKqWAMXZquXNvZEACZAACZAACZBAtRLYvO+4kzjbMDHOyFe4csfRcrcLXiOmCABRyu6JWO6Kq3DDLfuPC7xhrGFLe7aqI/BKC3bbrecEQgIEPNOaq/BckXOOeuCxhNC5EPhN69umrnyzaLf5MSimyGtpNXjQJSfEOIU9ta63z1sHJmAdxCBXnff27fz5GTkirbZOvUSr0yPS2pbqnu/R8lQ+VuTbtZv12sU6cAtGq1MzRv4zoacx0OZPH69UD+EDRl5WDJxASFfckzOyC+SAC2/36jzerpZ7B9pRGQMK8Hz6v4s6S1v1kp68fI/86eMV0iQ5XhDmGbmhEaI2R3MK7zyUUyZhHvmIrdZMxTmER66OkObWdvg6j0E0eL6Z3w9sh4EpF/c/Fe7828W7/S6UYx8L0w47eat3aJqIxT6b/X67TQecWQ333x2HsqWlisym9WyVLFNW7DE/Vni6SQc52AcmYTAXxdkKo2UFJEACJEACJEACJBBWBIIvKVRYnR4eLAmQAAmQAAmQAAn4lwA8w+w2untJDkL7Ol8+D2xXX0yvoZ81JGIwGnLWwWvLaj1bngqDaV0WjPMQ6ZZuPeLU9N4qPPvD4GlmNeQ9jfBzfj9r/ZUxj458u2jVvmltn3dlF0bnaOjcqhZGrYMK0PACvZ5ppwiY4lN2XlEpJFavb6wMRm449y9pSONWDWoagyVwj4cnIQZOmAIcxCT7NV4Kho8L/Pn1TrCF1XeVy9PHZrkthjy7lw9qYax/89ctxndzj3pvrk/PNAaorFMP8+0q8JXVYxrCpnUbhGju2vzUQAC3jQmwFZ+o96wrw3G9+9tWV6sqvGzRFufBMPjtUDs+2ud67eKs3RseFS3b6iyc9/LT885sJAT4FbYBbWbYbLMMpyRAAiRAAiRAAiRAAiTgjQDFWW+EuJ4ESIAESIAESIAEQojAjDX7xZ4T9qJ+KRrusVa5jrJWXJQ8dllXx7bfLU13zAfbjF3AhNBozaUbbMdjbS/Cnlrt3F5NBeeuojZdryerwSPtnB7OoVSt6ysyDw+4yjAIqRBUrdaxDN5c3U97Zprbz1jrLFibyytzWlBU7FQ9dH+ZMwAAQABJREFUvAJppwj0OD3IIseF56xdDAw2bvDUfEo9Zs1jfGvGVifB0B/XgDVkLOqrr9EW/GX2uhP9fN1e1K+Z3Du2g9Hc3zRcMcI6+8vgib1BBV6rXXFaBLYuC+T5qSv3ufSOxSAreBJXhiF8tD30fQcfB8MgXLfd03amiwFnS2yDkXA/x7PJn2b/vXBh32ZGOHF/7oN1kQAJkAAJkAAJkAAJhDYBirOhfX55dCRAAiRAAiRAAiTgRAB59F6YstFpGUK4vnV7f801l+C03NsHhH9949b+kqLhHGHfaBhEf3Z+e9u/v9cjjKM1JCWEmjvPTvX3bqqlvsnL9jiFm4Yw++ilXY3cghVp0MY9mWIXaH9/fidpnFxxAadQvZOslqTXaWXZ2zZR62IVdSJ8UIMh4FuFXHjgTVu1t7Ka6bZeeAJaracKxjEqZIS7NUiMdXj1Z7vIK23nhrDcwWR3aS7dkV0aOZoMkaue5pr1pyF/sdXAyB8DO1CnnX+/tvWsu6rQfLcWyfK3y7o56kjW0M/Iy+zD19qxjbeZz+fvdCpyfp8UzWXb0GlZIH/AoI7PFzgfA9r79szK8Zo1Wbw5Y4s5a0wv7d/c6bO7D1cMbul0X5umIjLuuXb7ZfU+p0FoiObwhws6+fXcf7c4XRBxwzTkeP697oNGAiRAAiRAAiRAAiRAAr4SqPhweV/3xHIkQAIkQAIkQAIkQAIBQeAL7Ywd0bmhUycycs++c8dAefXnNEHHpqecmQifCUFg4og2Uud0HlN0sv/n+/UBcXzlbcTejFz5auEuRwhM1HP9Ga0FOVvRCW8Vbq37gIiHkJaBbDifH83ZLgjxaRq8Z5GL9tnJG4wwqObysk6fnbRBBqaWeBlDEPvo7sHy6OerZe5GZ49Ud3VDMLGHAt6n58NqZ+g1+/WiXdZFfptHBz/O8ZXa+Q9LbVJbrhrS0mDmbiemwG1d//QP691eJ9Zy/p5P01zSB1REw4AJWKyGur1DBxa88KPzQAysg1AxUllCqAp1Mz1KcZyucs7O0etz4sg2Dgxj9Tvx8dwdLnPPJmro1auHnro+HBtU8ww8Ca123fDWcu2w1nI4K0/yCuBNXSIeWcshfDPEangYzli3X5ZvO+r2ul2tuUkRbcH0Ksa97smre8p97y11Eqes9WPAT3Skc9us6815hLiFQGiWbV4vQS4b2ELwjLIbBhsgt3mzur55QOK+DM9i0xDe+vP7hxg5oXEtFNsGf5jlsPiE5uhN0zzk8IJH2NzjmsfUlX23JF1uObOtY4AS7mP/uaanPPTh8lLe+K62D4RlX8zfJbee2c7BCjlh7Z6t/m7n9NX7NY3AUUFed9i43k3lK723L9niHOHBul/87vjdmJLBUsj5+l/bQDOzPM7vuyow33vuKa9pLEf6hnvUixq/cfI0v7wrw/nzdVCL+XvhisEtHFXh+YFn6mu/pLn9PjkKc4YESIAESIAESIAESCDsCVCcDftLgABIgARIgARIgATCjQBEsAfeXy7PXNtLIHiZ1kiFnb+O7yqPXNJF5m8+JMu18zRH8zQWaud5fEyU1FUhdninBqVCIEPYuu2NxW47sM36g2H6lIpryE8Hcc60P1/cRW44o428P2ubwFM0Vzt241X8atOoluE1Obh9fWmmokKg2yvTNksf9Xqz5t+bMKyVIbRDkMf5hpcc8g3GRUVKg6RY9YCNl+bqGd2pWaI8qNcMOqTthpytd729VF67pZ+jYxvXEj4j7yXqXqZ1Z6kYBCEmKiJC6um1BI/rZvXiBR5ub2kuSLsH7uz1Bw2hxtzfmV0bGd6+k5ala8e3qFgUZVyLM1VAQRsqak/9sEHaN0kUM3cgPICbqAfwO79tcwr9CdEHgxPuOifVyPNp7vcZFalnaujU6jB8p3/V8J6muIw23KJCfOeURIMrwscivyq8fOHZBwE9/Ujpc1kdbbfv054/F+vhDWq9bs1tIKaYgyOiVAyEGAdxBXksIeZYPTFd5ZyFKJmRky8QE2EQO9+5Y4BAdFu/+5h62tWQOupxiWticIf6huCZqUIdhNpAMHNAgVVQAhN48fliXZonGQNQth3IknveXSaY2g0i5kwVcBG21TQM7pn+l1HGtXXoeJ7BHZxSVDjF97pl/Zo+eSni+TJ/0yGngUJ4/gzSeyoGdmDfCbFR0k3bOVz3Ce64B/mSo3TVzqOyameGdNf7i9VQhy/nD8+AsT2bCqJNPPbFavnBRch+3M8e+nCFcc2Y1y0Gbbx6cz+ByIljW61tyFKxENcp1jXWe2MTvT5b6vVZpCL5w5+utDavQvOx0SWCOK5lXAv2QS/2HSAXMcIYQ/iGIYqAr4brLhI7sVh0VA1jAIg78RtF8Yx58IPlxiAeRFkAmxeu7yOvT08zBslAeDUN+7i4fzO57ax2jjQDqPv+95e59Jo1t3tLxVlENsC1ZBruifDS/UCf5Ss0Z3BOfqHE6H0D94qOeq9E+dTGJc9+czt302cmr1eBOdkp1DKibUBsNgXog/r9wPHiN0MDHQTXpE6cYBACcudOfHWh8Vx0Vz+XkwAJkAAJkAAJkAAJhDaBGifVgvkQ1+hI3itfmBvMh8C2kwAJkAAJkAAJkEC1EIAH3UXa4X6XeqOYHndlaQg6rSFYQliD8ObN0HE/unsT6aGd5egwtYbGhLgGoWS95vD7VfOYmiIgRKTR3RpLZ+3IRDhNqwiKfaLzGx3wP6sAuPtwjtG5ik7PNppDt3XDmg7PHLNt69KPyZZ9WbJVRYg0nc5QQctu2Ce8n6zCjr2M9TM64ZHP0hpedJPmNoRXGto2STv1wXdohwbarpqGoAjBwPQ6Rl3wtkEnPsSRrQeyBR598ISE598QFYUgYMI71fQwwzZo/+Z9mbpNtvEHERRigScD8z9e0Nno7PZUzr4Ov7lveX2RRwEenpj/uKKbU0e1vR5XnyEO3K7iPphZDd6fX6qnm7d8yDe/tlDQvhEqmLbRTnaUx/WC6800iEArdxw1zjt4LVaPvS0qHNsNAstD53d0EjnhZYXzkZlTKNEqFCB0rLUDH151T3y7TiYvS3cphMBjb1jHhtJDO/EHtqvvJFqhTWg7cldOWbFXTpz26OqkQgGuF4hnuA5Nr0W0F6IqPA5xvSBctekRivzIH9w1SAVmz+ICvjfwcsd2GKBh2i79/oDL6p3H5Mflp+qFt9mp71ItPeZaTucWnuQrtP3guV2vWxyHPdejWbenKbz2b1bvw9Z67lrp98PfuSHNfV/+/FyXHrF929aVN28d4NX7HfeOP3+ySr3vnEMJm8xwTpC7GN8xeKXjO477HAYqmIZ8m4v1vgDhd+7GQ4I6YRDS4LWLexe2w3k3DUIUBsvgXoIBC5haDYMW/nVVTxUxI62Lyzx/WEWkK/S9el+GcxhjVITrD97wELG8GY7p1WlpcvvZ7Zy8s+EpuVIFMVwzEANNw/n/5N7BTpzMddYp2vfYl2uMNjx4XkfHqs3qNb5s6ykhdvJy51CzYP+qDhKxfl8dG5Zx5k8fr5BJ+n1zZbjf/OuqHtJWB+z4ajivr6qHJQbNlNcwaOFWfHf0nodzg++OVSuFyIln6w4dQAX2eF67Mgx8wL1jgw4+uuy5OS7vY9gOPOFpjn1BgG+q91gIq3bDcwg5a7cfPHVfeGO6cxhjszyetY9f0V2fcQ3MRYLviPFMVeE0QQeF9dKBEdbvEO5Tf/l0ldjzqDsqsMzgu4jQ1mN8zIOO3x+b9mY6DUTAceC7vUr54R6P3z1Ww8CORy7t4vM+zG3R/ltfW+Soz/4bAc98DIgzDdc/7rH4jbBNzyfuNfaQ42ZZTkmABEiABEiABEggnAmsefrcoDl8irNBc6rYUBIgARIgARIgARKoHAIQwUZ2aShnqEcdxARPXlfomFypnezw0pu2cp9AWPPVIFggBKQ3+592ViOXHGxoxwZi7Yh3ty28feDZBLEJHq2+GATVa16c77IoPPDQ3isHt1CxpI5Thzc2gPccPD2nrdxriCwX9k2R6zQEsiu7+OnZAgEF+SF9tRenbDIY/99FnQ3B2ZftbnhlgVOePU/boDP+EvUgOrNbI7deZOgEhoCE41yy9bDbDnvrfhDydJSKpMi9iGvJ9CazlsE8wqTCU/c39TQ1hUB7GXyGwPrY+G5GXdb1EEyXb88w8rt+qaGoIew9bREarWVdzX+vnpHveMiriI7x8QOaG6EwXXnZYXgrQgl/r9fcd0t2G+K6q/1gGTxw4XXpze58c4ljUAK8ryCMerP733P2HkMH/916nY1VLzg7e5zPySq6fr1otyH8w3vrvzf2cbkLs96vHhhqeMG5LGRbuFGFjT99UnYvQAhn/5nQ01ab/z/CuztdQ5S7MtxnEPLbnnMWAto8HQACblNW7DHC+F4ztJWMH9i8VDU7VQS7V71PIdD5ci1iEMqLP20y6oFI9brm7/Zm+D4idLbd4Cn8yk39pJ0K6LhHo92eDIM8XIVif0tzgT43uXQobNSFfeDaOluvSzwzTMN3Ya2KzbPWHzC+jxg0AkNOawhrdsPggGtfcr7v4p79O73mz+zauFS71mnd4I+c4Lhv4N71qEZ4cGWo1z5QCJ6XGJCC8LMYUAAvck+GQUv2cNEoD1Fw3JO/GV6QrrbHNueqx+SFfZoZx20NqWyWx7lZrSLfTGUFgdrd9WiW9zaFKPvsdSUDLDyVx6COu95e4rII9NUXb+xr3Muswrm9MDzv/3V1D/tij58x8Ogm9RB1Z9g3Bq9cqvfboXqftF5b5jbmQBBcA1OWlwxiMdd7mqL+EZ0bydUaph6esfbzggExiL4wVXOFYzpAB0G5+81xqw5QOpiZ53J3A7Ru5Cofqc96DJRxZfhuYKDPdB1IhUEKVjcJDO7CwA9f7eWpm42oFL6WZzkSIAESIAESIAESCBcCFGer8Exj9CA9Z6sQOHdFAiRAAiRAAiQQ8gTggZVSJ0Hqq2AQrWIb+vkhZO4+nCsIYYkQfeFkELkgeiBsZ6HCQOc/PB29eakGAyN0XNerFWt4QuG8FxSe1FCPRer1lO0x77AvxwaPqoYaGrmFeljFabhN5LlELkd4RcPjpyyXEUTaVg1qGaLTIc2niXDJ7nIA+9I2X8uADwYrIARpgoYFhmPyEd0/vEVNL1df66rKcvAY66SezOAOr2HkIIUHXVmYV2V7A2Vf8MxHGG8IdFknCmSnCnLw5Atkg5iFUMAXqTA0VQeL/OGjFV6/G7iuOzVNMsQgCNOmITf1qL//6vHeZl5bCKELr3GEtYdo6g/DPRbhhCGoIicuBv9UNPz2eTpQ5G8q5h7NLpDrXp4vyI/uzTDIAeFvrTlOsQ2iByBagzeD8N1MryOEL8YzNDe/2GAFb/1QeG54O/7yrsczA964+A4iSgFSKuxVT+5dh7Pd5jcuy75wXSESAH7b4F6IZzlSFfjzXo7vFsIXw7M4PibCGAwAD2Y8MzCoi0YCJEACJEACJEACJFC5BCjOVi5fp9opzjrh4AcSIAESIAESIAESIAESIAESIIEwIACx6c3bBhg5ceHRd+6TM1UILMnX6Q0BhN3pj4x05NxFeYizZYmI4G0f1bn+hhFt5KHTIZDh9Qgv6LLYE1d2dwpx++R36+TD2dvLUgXLkgAJkAAJkAAJkAAJkAAJVCGBYBJnI6qQC3dFAiRAAiRAAiRAAiRAAiRAAiRAAiTgBwIPX9zFEGZRFUISl0WYxTYID75w82HMOiw6Sl3/QsCMkPjjTuWmRb7bsgqzQDBLc/xaLUrDQdNIgARIgARIgARIgARIgARIwB8E+HbhD4qsgwRIgARIgARIgARIgARIgARIgASqiADCpiIvsmnIo1wes+aeRahwhDYOBbv/3A6OXOHflZONPT/pQQ3HTiMBEiABEiABEiABEiABEiABfxCgOOsPiqyDBEiABEiABEiABEiABEiABEiABKqIwOAO9R17ytac4OXJZ4mwyF2aJTvqWbD5kJGj2LEgSGfqaZ7oDk0THa33Jc+so7BlpmerOo5PxSpcz9no7EnrWMkZEiABEiABEiABEiABEiABEigjAYqzZQTG4iRAAiRAAiRAAiRAAiRAAiRAAiRQnQQiapSEH46NihQIrWW1S9XztnFynGOzH5fvdcwH80xECRrjMGrFRZX5cJokx8tlA0s8kyHMHsspKHM93IAESIAESIAESIAESIAESIAEXBEo+xucq1q4jARIgARIgARIgARIgARIgARIgARIoEoIrN6Z4dgPQhNf0r9ESHSs8DDTvUWy/G5MqqPEwrTD8sPS8oVGdlQSIDNHsvIl/UiOozVXDGrhmPdlJjE+Wh6/srtEn84xC8/kJ79b58umLEMCJEACJEACJEACJEACJEACPhGgOOsTJhYiARIgARIgARIgARIgARIgARIggcAgsErF2UnL9jgac5/mWD23V1NHnlXHCtsMPELv1bLv/26QJCfEGGtX7jgqD76/XIpPnrSVDs6PyJ377+/WOxrfv109+dOFncWbB23N2CgZpwy/fnCYDNBtYBk5+XLPO0tl56ESsddRMWdIgARIgARIgARIgARIgARIoJwEapxUK+e2AbHZml3H5MoX5gZEW9gIEiABEiABEiABEiABEiABEiABEqgKAnVrxcjTE3oJxEfTDhw7Iav1HXnjnkyBxycsNjpSWtWvKX3b1pWmdeLNogIR85Vpm+XNX7cY844VITCDqM+/O7u93HpWWzFDQOfmF8laZbNm9zE5dDxPiopOKpsIqVc7Rnq3qiudmiU6ygLBnA0H5ZHPV8nBzLwQIMJDIAESIAESIAESIAESIIHQJ7Dm6XOD5iApzgbNqWJDSYAESIAESIAESIAESIAESIAESKCEAITH0d0by1VDWkrX5kkSp0KsN0vblyU/rdwjXy/cLQcyT3grHtTrU5vUlptHtTU8YevXjvV6LPCUnb56v3y1cJeK3BkS3EPZvR4uC5AACZAACZAACZAACZBASBGgOFuFp5Oes1UIm7siARIgARIgARIgARIgARIgARIISAIRETWkbcNa0kVF2uSaMRITFaGeoCL5hcUCj9qtB7Jlx8Fsh0dtQB5EJTaqYWKcIWA3r58gMZpPNkr/CoqKJTO3QLYpm20HsuRwVh4F2Uo8B6yaBEiABEiABEiABEiABCqTQDCJs1GVCYJ1kwAJkAAJkAAJkAAJkAAJkAAJkAAJVD6BYg1TvHnfceOv8vcWfHuAl/Cva0PbUzj4zgpbTAIkQAIkQAIkQAIkQALhSSAiPA+bR00CJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACVUuA4mzV8ubeSIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEwpQAxdkwPfE8bBIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIggaolQHG2anlzbyRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAmFKIOjF2YigP4IwvfJ42CRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiQQZgSCXtqMi4oMs1PGwyUBEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEghGAkEvzqbUS5DIiBrByJ5tJgESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESCCMCQS/OxkZFSPsmtcPolPFQSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEgpFA0IuzgN6lWVIwsmebSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEKkAgpW5CBbau+k1DQpwd2rFB1ZPjHkmABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABKqVwJAO9at1/2XdeUiIsyO7NJJGSXFlPXaWJwESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESCGICVw1pGVStDwlxNjKihlwxqEVQgWdjSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEyk+gX9t6ktq4dvkrqIYtQ0KcBbfLB7eQOrViqgEhd0kCJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJFDVBG47q11V77LC+wsZcTY5IUb+emnXCgNhBSRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAoFN4HKNqjswtV5gN9JF60JGnMWxndWtsYzr1dTFYXIRCZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZBAKBBIqZsgD53XKSgPJaTEWZyBR8d3ld6t6wTlyWCjSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAE3BNAmtOXJvaRhNhI94UCeE3IibM1Y6Pkfzf3o0AbwBcdm0YCJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACZSUAYfad2wdKauPaZd00YMrXOKkWMK3xY0Oy8wrl0c9Xy9SVe/1YK6siARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARKoagKpTWrLMxN6SZtGtap6137dX8iKsyYliLP/+GqtZOTkm4s4JQESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESCAICERE15OZRbeWO0e0kOjL4gwKHvDiLa+pIVr58PHe7fD5/pzEfBNcZm0gCJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACYUsgJipCzu3VVK4d1ko6NE0MGQ5hIc6aZyu/sFh+Xr1PZq8/IGt2HZPtB7PNVZySAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAlUI4F6tWOla/Mk6dumrlzUt5kgx2yoWViJs/aTl3WiULYdyJK8gmIpKC62r+ZnEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiCBSiQQUaOGxEVHSOPkeGmYGCf6MaQtrMXZkD6zPDgSIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIIGAIhD8WXMDCicbQwIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAKuCVCcdc2FS0mABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABHwkkJ2dLe+9954cOnTIxy1YjATCkwDF2TA575mZmWFypDxMEiCBYCZQWFgoOTk5wXwIbDsJkAAJkAAJhA0BdLwUFRWFzfHyQEmABIKTAN8xgvO8sdUkQAIkQALBRwDvB++8845s2bJF3n77bQq0wXcK2eIqJEBxtgphV9eu9u3bJ6+88opMmTKluprA/ZIACZCAVwLoNPn444+NH2/4MUcjARIgARIgARIIXAJZWVny1ltvyaeffkqBNnBPE1tGAmFPgO8YYX8JEAAJkAAJkEAVETCF2QMHDhh7xPsCBdoqgs/dBCWByMfUgrLlbLRPBCDMvvvuu4Yn2u7du+XEiROSmprq07YsRAIkQAJVRcDsNElLSxP8mNu8ebN06dJFYmJiqqoJ3A8JkAAJkAAJkICPBKwdLYcPHxa8c+C5HRHBsb8+ImQxEiCBKiDAd4wqgMxdkAAJkAAJkIASQF8eNAhTmDWh5Ofny9q1a6VDhw6SkJBgLuaUBEhACVCcDeHLwCrMmodJgdYkwSkJkECgELB2mphtokBrkuCUBEiABEiABAKLgFWYNVtGgdYkwSkJkECgEOA7RqCcCbaDBEiABEgg1AkgPRmE2f3797s8VAq0LrFwIQlQnA3Va8CVMGseKwVakwSnJEAC1U3AVaeJ2SYKtCYJTkmABEiABEggMAi4EmbNllGgNUlwSgIkUN0E+I5R3WeA+ycBEiABEggXAhBmkWPWnTBrcoBAu27dOnrQmkA4JQElQM/ZELwMPAmz5uFSoDVJcEoCJFBdBDx1mphtokBrkuCUBEiABEiABKqXgCdh1mwZBVqTBKckQALVRYDvGNVFnvslARIgARIINwIQZl977TU5dOiQT4cOgXbhwoXSrVs3hjj2iRgLhToBirMhdoYhzL711ltGbllvhwaBNjc3lzlovYHiehIgAb8TQKfJhx9+KFu2bPFaNwTaDRs2GD/emIPWKy4WIAESIAESIAG/E4Aw++yzzwqm3gwCLZ7vPXv2ZA5ab7C4ngRIwK8EyvqOsXHjRunatavwHcOvp4GVkQAJkAAJhAEB02PWV2HWigQetB07dqRAa4XC+bAkEBGWRx2iB216zObl5fl8hAsWLJApU6b4XJ4FSYAESKCiBMzR7Fu3bvW5KnT0IkwKhFoaCZAACZAACZBA1REwPWbx/PbVMAj0008/laKiIl83YTkSIAESqBCB8rxjoEMZOfL4jlEh9NyYBEiABEggzAjA2QsOF+iri4qKKtPRozy2f//99+Xo0aNl2paFSSDUCNQ4qRZqBxWOx2MKsxi1Uh4bNGiQjB07tjybchsSIAES8JmA2WmSlpbm8zbWgg0bNpQbb7xRatasaV3MeRIgARIgARIggUogYAqz5RkRj+Z06NBBrrzySomMjKyE1rFKEiABEjhFoKLvGI0aNZIbbriB7xi8oEiABEiABEignAS++eYbWb58udutmzVrJrfeeqvb9VxBAuFIoGxDG8KRUBAcM0akr1692ggHgObi88GDB2XPnj1uW49RKp07d3aMboG37a5du6R58+Zut+EKEiABEqgoAYQnTkxMlN69extVYUAJlnmyVq1aSd26dR1FVq1aJRhQQiMBEiABEiABEqhcAnif6Nu3r2MnGRkZgsg7nmzMmDFSo0YNR5H9+/dL06ZNHZ85QwIkQAL+JuCPdwz0qQwcONDfTWN9JEACJEACJEACJEACJOCSAMVZl1iCayFGoo8ePdqp0bNnz/YoziKnyvjx45224QcSIAESqGwCyOmEP9PQyetNnB0+fLi0a9fO3IRTEiABEiABEiCBKiLQvn17wZ9pGMzpTZwdPHiwkzhrbsspCZAACVQWAfs7xrFjx7y+YwwbNkxSU1Mrq0mslwRIgARIgARIgARIgAQ8EmDOWY94uJIESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAE/EOA4qx/OLIWEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEvBIgOKsRzxcSQIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAL+IUBx1j8cWQsJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJeCRAcdYjHq4kARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgAf8QoDjrH46shQRIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgAQ8EqA46xEPV5IACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZCAfwhQnPUPR9ZCAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAh4JUJz1iIcrSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESMA/BCjO+ocjayEBEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABjwSiPK7lShIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARJwQaBFixZSVFTkYs2pRfXq1XO7jitIIFwJUJwN1zPP4yYBEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiCBChDo06eP4I9GAiTgOwGKs76zCqqSjRs3lt69e7ttc2xsrNt1XEECJEACVUUgJibG470K7UhKSqqq5nA/JEACJEACJEACHgjUqlXL63Pbw+ZcRQIkQAJVQoDvGFWCmTshARIgARIgARIgARKoAIEaJ9UqsD03JQESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAES8IEAPWd9gMQiJEACJBDIBDDG5sSJE1JcXBzIzQy7ttWoUUPi4+MFUxoJkAAJkEDoEsAz2FN+pdA9ch5ZWQhERkZKXFxcWTZhWRIISAK43+G+RwssAlFRUcIIaYF1TtgaEiCB8CFQUFAg+fn54XPAPNJyEeCzulzYQnojirMhfXp5cCRAAqFKAELs+vXrZenSpbJz507+CAzQE40fXk2bNpWePXtKjx49JDo6OkBbymaRAAmQAAmUhcD27dtl0aJFsnXrVsnJySnLpiwbxgQSEhKkbdu20r9/f2nZsmUYk+ChBxuBrKwsWbx4saxdu1YOHjwoDMAWmGcQoefNe0zz5s0Ds5FsFQmQAAmECIEjR47IwoULZcOGDXL06NEQOSoeRmUTwLM6NTVVBgwYYPQXVvb+WH9gEwjJsMZ5eXkcyRnY111AtA7ebOgggXhCI4FgIoAfgF9++aXs3r07mJod9m2tW7euXHrppcKOkrC/FAiABEggiAlgRPz3338vq1atCuKjYNMDgQAGbp1//vkcuBUIJ4Nt8Ehg+fLl8uOPPwr6WWjBQwD3mHHjxtGbNnhOGVtKAiQQRATmzJkj06dPZ/ScIDpngdZU6BIYsHnOOecIIuzQwpNAyIizmZmZxmiVNWvWcLRKeF7L5Tpq3AgbNWpkeLX16dOHLy7losiNqpIARqq//fbbkp2dbewWScN3aEfJwcJCKWIK8ao8FV73FaH3lzr6A6t1bKxE6TwMnrPXXHONtGnTxuv2LEACJEACJBBYBBCu7L333jMiVpgtiy7IkJiCY1KjuNBcxCkJuCRwskaU5MckS0F0kmN9q1at5Nprr6VA6yDCmUAjMGvWLPnll18czcrR6D3b9N0jS6f0nnVgCYiZmIgISdF3jUaWSD0YFHrdddexnyMgzhAbQQIkECoEpk6dKnPnznUcTmRhjsTmH5aIYoY1dkDhjEsCxRH6PhBdRwqjEx3rO3bsKFdeeaVE6HOcFn4EQkKcXbFihUyePJkjOcPv+vXrEScnJxtebQwx5lesrMyPBNAp/Morr8jhw4eNWtM019MUHZiSocIsLXAJxOsPrDMTE6WXeurDkG/u7rvvltq1awduo9kyEiABEiCBUgQmTZpkhDLGiqjCLEnOWCEx+QxhVgoUF3gkkB9TV44m95SiqJpGuUGDBsnYsWM9bsOVJFAdBDZt2iQffvihsesi/f9Xfe9YrANEOSC0Os6G7/tsHhMj52vfRr3TEcKQWgXRe2gkQAIkQAIVJ4Dw/p999plRUcTJQkk8tloSctL1M1wnaCTgG4G82AaSkdxDiiLjjQ1GjRolI0aM8G1jlgopAkEvzi5YsMAIsWOelciiExKbd0Aiihhyx2TCqRsC6smGm+CJ2EZyUkeuwODVNmHCBGndurWbjbiYBKqPwIwZMwR/sHW5ufK15rTgz7/qOx9l3fNIFWOHnhZku3fvLuPHjy9rFSxPAiRAAiRQTQQOHTokL774ouEpFlWUI/UPzubo+Go6F6Gw26KIWDnUYJjxLoJIPvfee68g/QGNBAKFQLF6xj7//POSkZFhvG98qmlVMDCUFhwEMDj0hvr1pf5pgXbixIkCT30aCZAACZBA+Qng2fjcc8/JsWMaNUefjvUOzdOBmkfKXyG3DGsChZEJxvtAcUSMkXLxwQcflJo1Tw3eDGswYXbwQZ1sMz09XX766SfjlOGmWPvYWqmVvV0/U64Is+u4QodbrCHGjid1luyElgLPxC+++ELuuusuIx9thSrmxiTgRwL4Ebh48WKjxmydn6Q/Bnmn8yPgKqhq5vHj0k69ZhvrIBCMtoSXDH94VQF47oIESIAE/EBg6dKljhCeiRlrKMz6gWk4VxFZnKeeFmvlaN2+xnW1bNkyOeuss8IZCY89wAhs3LjREGbRrCXqLUthNsBOkJfm5Or74vcqrE9UgRa2cOFCirNemHE1CZAACXgjsGXLFkOYRbma2dsozHoDxvUeCWDAb2LmesODtlAjIiIy7JAhQzxuw5WhRyCog1kjxjsEC1jy0WUqzG7TOcoVBhD+5zMBhKFIylil189WY5usrCxBYncaCQQSgf379wuuTdjKnBzJO33vC6Q2si2eCeDphFBwsKKiItm2Dc8sGgmQAAmQQDAQ2LFjh9FMROeJzdsfDE1mGwOcQPyJfSryFxitNK+vAG8ymxdGBNLS0hxHu/D071fHAs4EBYH0/HxJ18HnMAgKzBEcFKeNjSQBEghgAtbfawk5uwK4pWxasBCIz92tXtintC3r9RUs7Wc7K04gaMVZ5Fzcvn27QSBOX2zjc/dUnAZrCGsCtXW0SmRRrsEAo9f58hLWl0PAHTzCKZq2S1+0acFJYLfl3FnPaXAeDVtNAiRAAuFDAOHLYNFFx/UFmkYC/iBwUqILjxsVmdeXP2plHSTgDwLm71RE7Dmq3hy04CRgvnuc0JDU2RTZg/MkstUkQAIBQ6Dk99pJiSo49RsuYBrHhgQlgRoniyWy4JQTR8n1FZSHwkaXk0DQirM7d+50HHKCjjKgkUBFCeCGGJ+LJO4iOeqZaL6QVrRebk8C/iCAkNum0WvWJBF807yTJdEdrOc0+I6ELSYBEiCB8CJgRusR/b1IIwF/ETh5+npCRA0aCQQSAfN3Kt87AumslL0t1vNnntOy18ItSIAESIAEQMDxPmDgKOnbIR0SqAgBpOqE8X2gIhSDd9ugFWetowk4WiV4L8BAa7k5eh3tsl5jgdZOtocESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESCD4CEQFX5NPtdg6WsUcYRCsx8J2BxABizeE9RoLoBayKSRAAiRAAiRAAiRAAiRQisC5974oNSIiSy23L5j+xsNyIivDvtj4nNKxn/Qce6PLdebCeZ89LUf3bDU/ckoCJEACJEACJEACJEACJBAABIZNeFgSGzTz2pKFX78oh3asd1kO26MeT7b6l49k5+q5nopwHQmQgA8Eglac9eHYWIQESIAESMCPBIalpspfL7jA5xqRtzlfw/Ttz8yUbQcPyjbNmzsnLU22W/LneqrsL+edJyM6dPBUxGldkebEytOcWPs0L+ASzUm+WP/WpKdLAUMFOnHiBxIgARIggdAkkLZ4qiQ3aiVJjVpI8y6DJSa+pnGgxUWFsmPVbDm2f6f+7ZDC/Fy3AI4d2CU7Vs4yOnWSGjWXlt2GidSooZ0vcyRj33ajjtzMw2635woSIAES8BeBu0aNkot69fK5umJ99zihqWB2Hz1qvHds3r9fZm7cKBmassgX+/rOOyUxPt6XokYZvGNgf1v1PQfvHYu3bZOt+p6DdyAaCZAACZAACVQHgZ2r5kiDVp0kqWELaaqDLmsmN3A0Y9fa+ZKxd5vxez776AHHcvsMBnFuWzpdEhs203qaG+8VUbHxsi9thRzatdHYPmPfDvtm/EwCJFAOAhRnywHN102GXvVHaZza02vxxd+9qp0gv7ksF5uQKOc/9JrLdebCdTO/lHWzvjI/ckoCJEAClUJgi3Y8fL1smaQ2bChdU1LkrM6dHftBJ0ShiqOmRUZESIR25rqyVbt3y+eLF8t/p0+X4ydOuCpiLPt1/XrJV7G1faNGMrx9e0nVqWkQYtEBY5p9f7cMH26sysrLk2enTZN//fij0XlilueUBEiABEiABEKNwKZ5kxyHdOEf3pIm7XsbnyGmTn35Acc6TzNZR/bJml8/dRS55X8LJTI6Rn566X6ByEsjARIggaoiMG/LFomNipL2jRvL4LZtjfcPc9/Wd4Ea+s4Rpe8ergzvJ7M2bZK358yRTxYudHp/sJf/ZNEi6dK0qbG/MV26SN2apwa4oJx1sKen/WEQ6p+/+UZQF0VaO2F+JgESIAESqGwCO1bN0kGZs4zdDJ/wZ+k8Yrxjlz+/+nvJz812fHY3k5+b5aQzXPrIx9KgZSeZ9cETciQ9zd1mXE4CJFAOAhRnywHN1002zPveuGklNW6po0wGSd2Udo5Nt6/4TQ7tXK8j0HfIwe1rHMvtMwV5OQLxFiNV8Ne695mSkFRP0jcslv1bVkqGjn7fu2m5fTN+JgESIAG/E9iTkSGvzJhh1Ns0OVnSn37asY9nVAD9/RdfOD5jpnZcnLQ7LeRe2LOnjOveXeKio6V7s2bG3z1nnSUPf/210VniqvMCHTL4gz149tny9OWXG/P4b9A//2mMUHcs0Bl03vRs0UKGtGsnEGc7akdOrdhYefT88419D33ySQq0VmCcJwESIAESIAESIAESIIEAJbBsxw7BH+y6wYPlvYkTHS0994UXZNratY7PEEzr16plDOrs07KlXNa3rwzVqD8QbUd17Gj84X3idx99JPNPv184Nj4988WSJWK+zUy65x7j/QGrjuXmSvLddzsVx/6S1Mu2f+vWxiDS340cKckJCdKqfn356JZbZECbNnLvJ584bcMPJEACJEACJEACJEACJGAlQHHWSsPP84jdbsZvzz/vFidxdsGXzxuhwbzt0ghDZvGqrVmnkbTqeYasmPKOIBwBjQRIgAQClQC8Ypfv3Gn8fTB/vtTTDpPHL7pIbjvjDI2QWEMa1q4tb15/vSHU3vfppxUeXY6Qxgu3bjX+Xv71V/ns9tsFojAMnTQ3DxsmL+lyGgmQAAmQAAmQAAmQAAmQQOgQwEDPg8ePG39zNY0KIvQg0s9LV18tZ5xOk9JLB3HO/P3v5eo33pCvli6t0MFjfwiXDIEYf6/OnCm//eEP0qbBqfCRd2tI5td/+03W7tlTof1wYxIgARIgARIgARIIBALpmjZu165djqYMHDjQMe9q5vDhw7J582bHqt69e0tMTIzjs30mKytL1qwpcWDsqIPrktUxKNTNdeyXUD9qHh8JkAAJkECVEzisD9o7PvxQxjz3nGRruGHT7jnzTHlozBjzo1+mEGrhyWv1yP3T2LFuQ575ZaeshARIgARIgARIgARIgARIICAIrNFOxJEa6eeRb791tCdGI+18ettthmerY6EfZpDn9t9TpjhqwkDUh8eNc3zmDAmQAAmQAAmQAAkEM4H3339fBg0a5PjzdizTdaCctfyRI0c8brJhwwan8gs1HUU4GMXZcDjLPEYSIAESCCACP69bJxe//LJTi/5+4YVGGDKnhRX8sHn/fll6OhQaqkqpU8cpd1QFq+fmJEACJEACJEACJEACJEACAUwAAzUfnzRJ/vXjj45WItTxWzfcIBBq/WmfLV7sVF2P5s2dPvMDCZAACZAACZAACZAACVgJ+PfXqLVmzpMACZAACZCAGwIQaF/TUF8IcQxDLtp/aMjjK157zc0W5VucqaGVrYb90EiABEiABEiABPxLAKlXWvceKfVbdJTIyGgpyMuRvOxMObJnixzZvVmO7tkqxcVF7neqXmYpHfpq+pYREpuQaJTLzToqW5f+Ivu3rhYNheG0bWRUjCQ1bum0DB+OpKeVlNU6m3XqLy26DZVDuzbKpnmTSpXnAhIggfAg8Nj338sFmu6kS9OmxgFjeoPmsX191iy/AUBKF6vxvcNKg/MkQAIkQAKhTiAqNl5adB0sKZ0GSEx8LSkqyJP83Cw5dmCXvg+kyeHdG/VztkcMdVPaSerAcyU+sa5EREYZ2+9aM192rZkrSP1ot6RGLSQyOtZpcfaRfZKXc9yxLKlhc2k3YKy+Y9SWeZ8941jOGRIIBAIUZwPhLLANJEACJBCGBB7VEGPIAxupo9dhF/bqJckJCUb+Jn/hSIqPd1SVk58vBzW0Mo0ESIAESIAESMA/BCCSDpvwf9Jx6EVGhUUF+dqJclw7ZGprR0lJTqFv/nW97N+yyuVO0QHT+9ybpE7TNsb6vJxMzU0fqXXUlB5nXysZ+7bLrA8elz0bS3JExtVKkrNufkLqpLTVsiXBoN65Z7gKw7lGp06vsTdKcuNWRp2rp39CcdYlfS4kgfAgkK8pT/763Xfy5R13OA74Oj+Ls4mW9w7sZNvBg459cYYESIAESIAEQplA+0HjZPiEPwsEWgyqzD1+RCL0PQGCqGmrfv7QrTjaJLW39LvoTmnaoY9RvOBEthTpsxu/+buOvEJyMw/Lwm9ekg2zS1IVoGDvcTdL2z5nndrv6R3NfOcx2TD3O2nUppv0Pu9madltmOgLg2Qf3e92/6c35cQDAeSLrV275Hx6KGqsitIIJdbySPngySK0b9haPjIy0lPxkFlHcTZkTqXzgcTWTJKWPYZJ8y6DJU7ni4sKJP9EjmQe3G2MXj+wbY0cP7zXeSPbp4atukiHIRdIzToNBR0vGAG/Z8MS2bzoJx0Jf8xWWqR+8w6C/Vrt2IGdkqUjVkyr2yxVOgw+XzBq5aeX7jcXc0oCJBCGBA4cPy5T166Vc7t1M44+Vh/cGNH+/rx5fqGBMMa9WrRw1PX1smWSqwItjQRIgARIgARIwD8ERt30d2nbb4yOjM83BNS0xVONeYx0b9Smuwwcf680atvd7c7Q2TL0mj8Z6w9uXydzP3ta9m1ebgiubfudLcN0HQTWc+56Xr598sZTnrFaOjvjoHz+2OUSHZsgE56a4vC2babvPv0vukOSGpX2qnXbCK4gARIICwKTV62SY7m5Yg7eHNKunZH2JF3zxfrDzuvufK/7OExypfmDHesgARIgARIIXgJ4Fxh10+PGAaz77UtZ8v1rknPskCGIJjVsIV1GXCbdR1/j9gCbavSccfe9bAzsNATUz5+TrUt+Vo23WBqn9pIzrntE6jRpLSOue1T1iOOybdl0R10z3n5UZr77mIy84W/SfvB5xvJE1RxG3/4fadt3tKMcZypO4MEHHxT8+WqXXHKJ4M9X6927t2RmZvpaPGTKUZwNmVNZciC4GQ2f8BeJiok1XP4RPgCjE3BDrHHaQ23zwiky/Y2HSzayzGG0yqArHhCIs7DsowcMYTaxQTNpo6NRBl/xoKz65WNZ8NULJWHDtFyrXiOk/aDzBOVMW/ztK7J08ptGmDKMVknp2N9YhRssjQRIgASmrlnjEGdBo3uzkvtHRencriGTI06PzDpRUCBP/fRTRavk9iRAAiRAAiRAAqcJQDRFZwxsxdT3ZOO8H06vEeMdZO/mZTL74ydl/CMfO5ZbZ/CuMfiq3xuL8L4x6dk7NATZqRdyvCuk6YBQ2Fm3/ssIjTbmd8/KJ3++0On9A4NHi4tKwiUPuuw+DYU8XQ7ueN0YkBoRFS0X/997Rj38jwRIILwJ4H1g5saNcqEOBjWtW0qK+EOcRSQgM10L6l6/d698vmSJuRtOSYAESIAESCA0CWifW59xNxnHhnQmsz78Z8lvdfWgPbZ/h8z//Flp3Wuky+OPr11HRt/2pCHMniwulh//e48c3rXJURaDNqe+/IBc8fevDE3jrFv/KR/98VwVfw87ymA7vBOYBiF42/IZMuejJ43UJhB8R9/2b8P5zCzDKQkECgGKs4FyJvzUjpbdh8uoif8wals/62tZ8OULjk4OxHtvN+AcI8yAu91hZPu4+18xhF3cvOZ8/G/ZtmKmcWOtXT9FRlz/qMaO7y89z7neCFm2bPJbjqowMgZ/CEPQ57xbjOUYtT7mjqc0B9WZjnKcIQESIAGTwIZ9JZ71WNapSRNzVbmnCRpq48Gzz5a/nHdq1FymjpC/+o03ZNXu3eWukxuSAAmQAAmQAAk4E8CgTdP2blpqzjpND+1Yr6Pbf5XCfOdcjCjUTTtOIiJOhataqWHOTGHWWsGWxdN0tPxfJDquphF5p57moTqsOWzd2c+v/bFU+OR1M790V5zLSYAEwowARFOrOIt3j590sGhFrElSkrwyYYIMTU01qtmo7zfnvvCCZOflVaRabksCJEACJEACAU8AjmCIkgnbu2lZiTBraTkGXa6c9oFG5Yy2LD01eyq/bD3jwxb1lrUKs2ZhpDjZo+8aKR37GZE9W2iY4g1znMMbm2UxXTvjc5n/xXPWRTL/y+flrFtUOKaRQIARoDgbYCekos2BaApDKOHZOlqluLhkJDmScKNzot3pEe72fSGvE0aSwOMW8eGnvvKQIPyxaccPpcu0//1ern5ykhEzvv/Fd2lC7nk6Mn29WcSYWhN0t1ZvWsNL980/y+GdG41QysOv/bMhEjttxA8kQAJhSWCrLRdTi7p1feJwy/Dhcq4ldBhGq8dHR0ufli0FIcpiNETy0ZwceW/uXHli8mQ5xFyzPnFlIRIgARIgARLwlQDeHUxDx8zudQvNj07Tqa886PQZH5AntnmXQY7l7sRddOYcTk+Txm17GGVb9hjuUZx1VGiZWfD1fyUmLsGyhLMkQALhSqA87x5x+o7x1wsucEIWpe8eyQkJMkwF2R7NmxvrUPcz06bJG7NmSYHFo99pQ34gARIgARIggRAiYH8fcHdoa3791OWq5l0HO5a7ex9AAYi2EGdhLbsP9SjOGoVs/0E4/u4/N9uW8iMJVD8BirPVfw781gIk2W7SvrdR36GdG5yEWetOVk593zGqxbq8VY8RUqtuY2PRrnULnIRZsxxGtO9YNUvaDxxnLGrVc0QpcdYsi+n2lbM0/9QT1kWySBN405PWCQk/kEDYEsjV8GJWg6jqi0Gc9Wb7jh2TpsnJMrx9e5mkOabyCwu9bcL1JEACJEACJEACPhI4kXXMUbL3uJslfcNiwch2X6x+y44SVyvZKIqBna5GyZv1HN2z1SHONmx9Kk+9uc6XaWFeruCPRgIkQAK5+flOEHx594jV95PHbOKsUyX6AWLs4exsQZjkvq1ayYKtW3W8+0l7MX4mARIgARIggZAiYH0faNZ5gJH3ddO8ST4dI5zDmrbv6yhrdRBzLDw9k7F3m2NRed4HsHHmwd2OOjhDAoFCwLde8EBpLdvhkUBUTJxjfZ2mbY3E2/CAtduOVbNVYJ1tXyzW0SqI6e7OjM6T0+Jsy+7DZPF3/3NX1OXy44f3yvsPnuVyHReSAAmEF4HoyFPhDM2jLtJcEb7YxHfekXUalsy0GjqDumrFxUlPHb0+qG1bYyQ7QpVd3q+fHFbP2X/++KM89/PP7CgxoXFKAiRAAiRAAhUgsHXpLzLgkruN/E816zSUS//yoWyY+71s0tyzB3WgqKv3EHN3jduV5HwsOJFt5Kg119mnBSdKckhFx8bbV/MzCZAACfhMoDzvHghPfOYzzzjtA+8eEHbr16ol/Vu3loFt2shgjd7TT4XZ20eMkE3798udH34o09c7RxlzqoQfSIAESIAESCDICSDK5oHta6Vhqy7GkSDVYltNfYJ3gp2r5khRofOgKOvh1m/Rycg1ay47kZ1pzpaaFuSXDLSM4vtAKT5cELwEKM4G77kr1fK8nOOGtyxyNyU1VHFi/H1GzlmEA/NmRmgxSygBe6hi6/bW0Sr1W3YywpL5sg9rHQixTCMBEiCBOJun7PETpXPSuaK0Jj1dFm/f7mqVTFm92lieGB8v/xk/Xm474wyppx0nz1x+uZHT9rb335diFwNXXFbGhSRAAiRAAiRAAi4JYPT5sslvSp/zbzXWIy9stzOvMv7gQbt5wY+yfvbXknPscKntY7SsabEJiXLtU1PNj6WmiA5kWoTtd4O5nFMSIAES8IUAQhRbzZd3j82l1SkAAB4uSURBVEIdPLpQPWHd2dfLNMeeWruGDeXdiRONFCvtGzWSaQ88IDe8/bZ8MH++u025nARIgARIgASCnsC8T56S8x587VSaRD2alj3OMP6gU2xbOl3WzfrKZXROa0hkQLjk4fd0wGZJekYrmKjYEoc0V7lrrWU5TwLBRIDibDCdLS9tLcw/IVs1eXa7/ucYJXuMuU6aaS6nDbO/kbTF0yQ3s3THiFll7fpNHaHFsMyTeIr9WC1SwxAwVJiVCOdJgAR8JYBcTVbbrKPM/WWZubly+wcfCETaq/r3N6q9edgw+W75ciPMsb/2w3pIgARIgARIIGQJwD3Mgy3+/lUjRNjAy+6T+NoleeOTG7eSfhfdKb3Pu1kF3Ldk6aQ3nDxpI6JKBBKMhN+tKVV8sYz9O3wpxjIkQAIk4JKA/d0j7cABl+XKsxB1nfn007Lpn/+UFnXrSkSNGvL6ddfJT2vWyMHjx8tTJbchARIgARIggWon4OV1QPZtWSnf/Os6GT7hz9KobXdHezHAsuOwi4y/navnyoy3H5Hc40cd6yOjYhzzmEnfsESKCvKclrn6UFTknB7NVRkuI4FgIUBxNljOlI/tXPj1i9IktbcgtBisXrNUGXLVH2TwlQ9J+vpFOnr9G2PUSnGx80iU6DhngeSCh153G/qzRkSEU2vgqUsjARIggfIQ6KdhwKy22Y8dJGa9T2o4Y1OcxbIHx4yhOGvC4ZQESIAESCAkCVh/61uF0DIdrAoLkZHR+k5QbETncbmtRqLYqGGMNy+cIimd+huDRFv3Gikx8bWM4uh06XfhHYIR7ou+edlRBfLMmpanIcxmvPNX8yOnJEACJFBpBBCC2Gr+fvfIKyyUZ6ZOlReuusrYDTx179Awx3//4QfrbjlPAiRAAiRAApVOoPikc99/hP6uL4+Z7xKehFOkQPzmX9dL3ZR20rbf2dKu3xhJatTCsbsW3YbIeQ+8aoi4hfmnBFjr+wAKzvvsKZcRdxyVcIYEQpAAxdlqOqk1tLOjPOZts+OH9shXj18tg694SNr2PdvIAYX9IGxxs84Djb8je7bIz6/+QY7uKQnNYx+tsmn+ZMnNKhnN4qmtnuLHe9qO60iABEhgiOZmstq8tDTrR7/Mr9q9W9KPHpWUOnWM+kZ06GDkpy1wEy7FLztlJSRAAiRAAiRQjQSseVpj4moJBlNaBVtfmhYLgVVfPgpys528Xl1ti86VXWvmGX+zomM019RoGXjZ/ZKQVM8o3nPsjbLm108dHS7Wzp2oGOaRdcWUy0iABPxLAH0w1neP3Px8WbFzp393orVNUU/ZFyy1ju7cmeKshQdnSYAESIAEqoaA9X0Ae4ytmSgnsjLKtnN9dsbVSja2KcjL8brtkfQ0wd/i7/4njdt0l34X3ykpHU9FsqvXvL20H3ieEeYYFZkirVlpZHRJ6GJzGackEOoEKM5W0Rm2d4ZERJYPfcRpl3/7Dcx6GMjr9Mvr/ydzPv63tOl9prTpO1pHsvczBFqUq9u0rcAz9ou/XakdJIeMTYsLnUMCoPPk0K6N1mo5TwIkQAJ+JYBQX0Mt4izCfc3YsMGv+zAryy1wvsfFas46irMmHU5JgARIgARCjUDmgRLBIVLF0ibte2uosMVlOsxWPUcY5Y/s3lym7YoK8mXTgslyYPtauehP72qHTpIhDrfoOlQ2zP3uVJ3aaWNaTEItQRuxHY0ESIAEKosA8sA2qF2Sw3ry6tWSlec9fGJZ2wPR12r2PLfWdZwnARIgARIggcoicMzyPoB9NO88SI7tL3lH8GW/DVt10cGW9Y1UidaQxF631eg6CHf84/N3y9h7XjCcxbBNq14jHOIsnMesFp9YV44fSrcu4jwJhDyBiJA/wgA5QPtolbhapzy4ytq8+NqntkNuJm+G0TBIuj3p2dvloz+O09yz3zo2iU+sJ93OutrxudAW0z0qhqNVHHA4QwIkUCkExnXv7vBmxQ4+WrBACouL/b4vjJJP0ryzphXrj8RsW6eJuY5TEiABEiABEggFAmmLpzkdRuveo5w++/Khw5ALjGJblvzssjjyyiY2aOZyHRZm7Pv/9u4Euqr6TuD4j6xkgUACgYQliIBsgoiyyV7RYoWpWre6nFHH8TCjTvWcdmx7Zmo9bed4pp3TjsVabWu1gNWyDC6oiIIQUKISCEvYt0ACCRISIAshYX6/m9yXF0hCAsntu8n3f8419713l//9vOfj/e/v///998v29CWB1623vlvydmYGRvLaqN7UQde5L/EXAQQQaBUBSy8cXF5buzb4YYutnz+vbXFZWYsdmwMhgAACCCDQVIF9G1YGfm/bPpfTHti74RM5V8/9uqiYOOk5cFSDVbJsm1/834uB16PjEgLrpcVfO6Ns3Sd66xQpFATamwDBWY/e8cK82hTCdkrLtd7ckpDcR7r1HazD/svkRN6+Zu1+6vgRWfX6c7Jt1cLAfmkjJgXWiwsOSXB6gtgu3QOvsYIAAgi0tECEzl39k9nVN33t2DZqtrXmYhqrc0sF95Jfq6mTz2mAloIAAggggEBbFbBRq8E94weNnyWWSqyppf/ob0jqVdeJpSveu+HjencbMOZmGXv7E/W+5j556vhRd1WCO5dau+Oo9qZ3y5DJt7mr/EUAAQRaXKBPYqI8Only4LjLt24VGznbGmX2NdfUOWz6ruZlH6izMw8QQAABBBC4RIHy00VyMGtNYO/UwddJ2ojafwsDLzSwknzFcLlq/K3Oq7vXf1DvVvFde8q3/u0FaWyQ16nC2vbA2fMGmx3atj5w3KsmzBJ3ftvAk6wg0MYFCM569Abn7vjSSQHgns7praKjuZpT3N7rBzatdgK05+9r6cBs0u0GiwYjvnznpcDLUTaPVE2xGy9WR7f0GTbeXeUvAggg0KICNpL1l3fdJaPT0gLHfWLBAiksufj8FYEdmrHyT5NqO6LYbnM/+aQZe7MpAggggAACPhTQ3/2r//LzQMWtV/utT78kib3qzvUe2KBmxUaxjphxn9z42PPOM1+9+4qcLsw/f7PAY0tNFh1bmyY08ELNSvd+Q6vXtD45Wz6r8/LeL1cEHvcffaMMnfqdwGNWEEAAgZYSiI+OloVz5khsVJRzyKLSUnlc2x6t0VnTOqA+PHFioOqVOsro5dWrA49ZQQABBBBAwEuBdW/+Ss6WV2ff7NAhTG76l/+WPsMmNF4FvWdnv82t7RARHSO71r8vebs2NLhPZMc4cadDqW+j7mk17QF98UBWep1NgjuBJvRIk8n3/SgwLWOdDXmAQBsVuLSJT9soRmtelg393/n5Mhl50wPOaSwF2DU3PygbP3itSaftljZEhk+/x9l21/pl9e4TGR0rM/RGSv7ezXLy67x6tyktLgw8f6bsdGDdVg5nZwR60FiQd/3iF5o/UXidI/IAAQQQqCvQMyFBXn7wQZk1cmTghSffeEPe/KJ58+AFdm5kxYLA37vxRnkkKDj76Y4dsnhDwz8qGzkcLyGAAAIIIOArgcPbMyR9wfMy8d4fiN7lEJse5a5n35LcnV/J7owPxeaStalNwiMiJa5rD0kbOdlpC9gcsVZsm8xlf2r0msMjomTaw88557FMPcHFsvTY6ForBzavuWAOqS0r39T0ajZCd7SzzeT7fywpA0aJpWQ+djBbbySVaS/8aGeeq07dUqVrSn/J+mh+nWw/zo78BwEEEGhA4Pp+/WTeo4+KzTdrpVgDs7f8+tey62jtKJ4Gdm3203EaBH71oYfkyu61Wcj+a9kyOVxYew+m2QdlBwQQQAABBC5DwDJlLn/p+3LTnF86o1vtt/u3npqrv7W362/9D3Re2CwneBsWHuG0FfoMn+AEWuMTezpntXlj18z7xUVrMPaOJ6WkqEDbGXq/LShTnbUxxtz+uLO/ZQLdsfbtOsc6sitTNq9YEJh6cfCkb0tCz76ydeXf5KjGN86UnBSnbglJ0ikxRbr0TJODW9ZKYe7eOsfhAQJ+FSA46+E799U7v3d6nnRKSnHOOu4739Pc71WyRb+Eqqoq662J9V6/QnurTHnwP8R6vB/Y9Kns15GzjZWB426RDe/9sd5NegyoDYjs03zxwWVf5ipNTfak2AhcG1VrX9zL/veJQA+b4G1ZRwABBIIFenftKtfpzY9TOqfSmcpKsXld7fstIjxcenTuLMNSU2XG0KFy27XXSqQ+Z2VPQYE8Pn++fLBlS/ChmrQ+UG+wnNCbKyU6d2zF2bPO+SxRcZQeO7VLF7l5+HCZrQHgsf37B463eudOueN3v5MKrR8FAQQQQACBSxNoXuabSztHy+215ZO/6lxOu2T8nU+LM4pVg7SWrtiWhkrpyUL5YumLkv3pYr23UtXQZoHnrae8LXbzxp16pXu/YZLUe6CzzVG96bPq1WcD27sr1nn1w7lPyeQHfqzZf6qDuNaOsaW+kr9vi2xa/hfnJeuhP2zqndKj/3DpGDR31cTvPiPHDmRL4ZH9sls7tJYUfV3foULmOetIZsX9GzIVoyIIhLiAtS1sWpTT5eXOb3trB9hI2KiICOmXlCTDe/WSO0aPlhsG1GYL+Dg7W/5V2x47jtTtSHKxS7XRsFf37i0leq7Sigo5W9PWsXN2jIyUISkpMlPbHna+ND23W174+GP5ydKl7kP+IoAAAgi0EYEO4q/2wMHNa2XRz+7T9sBTOs1idXYHmzbRloaKBVI3fvBnyXz/VamsONPQZoHnLdYx+/t/0M6YuTrKNlP3KZd4fc46YVpAuOxUkXz00g/EUi2fX9a99SvtfFkqo255yBk1mzLwWrGlvmKD0Q4EpWq+8roZ0luzfwZP3Thg7Eyn4+mJI/s0c886J8hb37FC5Tn7PWGF9kC1Q3v7L8FZD9/xM6Wn5f3fPKE9VF7UL4lk58wT7npaRs38R9nz5UeSt+MrsdGsYWEREh3XSXoPGSd9tce523vdeox88sf/rNMDpb7qj5r5kJw+USB7tNe5fZm6xXq9TLrvh85Dm5B726eL3JecvyePHZb0N553AsH2hH2B3v3ThbJ11VtyZNdGKT1VqHULl5jOiWLHSkjuK9YjP1frTUEAgbYvYDcenpoxw7kBYevB5Z4xY8SWphSb8/WvGRnyhzVrpExvcDRU7HjfHjXKOd9VPat77bnbztce8E0pdpNmY06O/Ck9XV5cudIJ4jZlP7ZBAAEEEEAgWCA2NlZOnz4tleExwU/7Yt1+qy/6+f3Src9V0nvoOF3GSqekVKf3fJgGMmyEannpScnTnu452hPdtg9uQzR0kZuWz9PA7x4n6JvY60rp0qOfJIzo7XSWtxsvW1e+JTlbP3PmumqoI2q59ob/6PfP6I2fPzudWPuOmOgEW62zqNWr+NghrVemHNycLgX7twWCxTba19pIRfk5GrB9/YIqxmp7xQK4EuLBWffzZJ8vCgII1BW4dcQIp30xVAOxg89rC/zP3XfX3biBR9bWeF/nln1t3Tp5e9MmJ4DbwKby7OzZMlo7m1o7p3+3boHNbERs1rPPBh43tlKunUatrfOL994TCwZTEEAAAQTajoD7e82CaZVh0RJeVe6bi7ORpss0JpHQo29Ne2CcJKb21wFaeh26WDDVYhL5ezY7I1MPZ68X+51+sVKYt0+WPv+I9NTBYEl9BjmZbnoNGaOjXcO1PVGug8xWO+2B/ZkrxTqA1lesw2bGkt9K9pol2h74hlwxaprEdUnWlModncBwicY4juzeqPVa52QAqjpbex8xtkt3J+C787N3Lzi0zYNrr4d6qQrv6FTR/XyFen2pX8sK+DY4G9yboDXmCmlZ5tqjHc/dIwufu0dGz/pnnVfpzupgZ6dEGT7tbmep3bJ2rUK/HDdoSrGsj+ZdpLdKdV8LuxEx7aGfysTv/rvz5WW55WP1Sy35imFOD5SyUydkxcvP6PxRF6byyV69WCr1y3Pifc84o2cthZiN8K2vWGqE4NG3A8Z8U1OTTXeCuu72/a6Zoj1nXtFe9PslZ9tndbZ3twmtv7W9n4I/Y6FVR2rTHgXCa0ab2rVH1Iyy8NohTM977NQpWbNrl7PY/yP2XLj2Jre/9a3bPEvWw3zfsWNO+rCsQ4ck98SJJld9r46uteW9rCxnH/ccNvrWRuVaT3Zn0XV7zupUqqNpbS4puzGyStMYf611DpVSPWa4ujZhWncKAggggIA/BFJtlJb+e1QZES9ndYk4Gzr/tjRJUDsrWfoyW6wXfEsUa6Ps/WqFs1zu8dy62Y2ZphRrz3y+8DdN2TRkt6mI7Cxnw6uDsinndXoL2UpTsXYj4LY9wvW39d+r2O/6g8ePO4tl2nHbG/Yb2uplf922gfvaWW17WDtgd36+0/bIPHhQTmpmn6YU23eztlVscYsd19ocTtvD2h017Q97bIvtY22Po8XFsnL7dlm3Z0+jnU/d43r1N/j9o+3hlTrnQQCBtipg7QG3lHXsKXElB9yHvvlbdPSg2GKdKFuiWJYdm4+2sTlpm3oeGzS26cPXnaWp+1hKZD+XM9FJUhUW5VxC8OfLz9dE3Zsn4NvgbHBvgqoIbdRWljTvyv+OW1tPEZv/KWPJXCeQaSNkrXeJ9egI13mVLF2ABVQLDmzT3uuf6Rec9V6/eG8cSxGw4IezJGXQtZLUa4B00XmZrNd4mAZm7ZiW1/3Qts+dY5aXFDcosPPz92Sf9mixnutXjJouFqCN1AnArWeKjZ49qr1ocrauE0stZr1b3HJOUzMX7N/qLO5zfvvr9l63egd/xvx2HdS37Ql01tTAbumu6bP2aFotr8vW3Fz50eLFnp3WRtfa0pZKsr53bumi6ZcpCCCAAAL+ELj66qtlk4260uoWdR4qicczfJbQzB/O7aWW9jkq1s+RW0bqVAwUBEJJwG17dNYAZLQGKMu1g4fX5R39zrXFq/Kzdy8cdePVuVvrPN01O4IVC7bHx8e31mk4LgIIINAuBIYMGSLv6r8VFToA4WSnQRJTlidhVRdP+dsucLjISxDoQHvgEtTa2i6+Dc4G9yYo1d4qUeXHfPfenCk9Jfs36hyyurRUsdGstlxuqSgvcdIiW2rkphZLzez3UhZTnarVGi89dE5LCgKhItBL502KsNSDmiprREyMfB5Co0FDxcgP9bD3zi19+/Z1V/mLAAIIIBDiAgMHDpTeOufgIR1RVd6xhxR1GSkJRZulQxPmZA3xS6N6Hguc6xAuRQkjpDy6Os1aWlqa9O/f3+NacDoEGhewz+VmTQds42ZHaNrtLzStO8VfAvF6T+NKTclsxf79ckdD++sqqC0CCCAQOgIdO3aUcePGyRqdostS0R5PGiddj38h4ZWloVNJauILgaoOEXKi6yg5E1k9aGPw4MHS87wpHHxxIVTysgV8G5y1H5eJiYlyXNPclMSlaSqBgxJR0fBo0MuW4gBtXqA0plfgS3Ho0KFOIKzNXzQX6BuBqKgoGT58uGzcuFF66OjLsdrzeT0BWt+8f1bRAXpzZEhNcNYCs927h/7cF74CprIIIIBAKwpYes3bb79dXnnlFSnVlJklsX2lPKqbxJXmSOQZTdd/7mwrnp1DtwUBC8pW6A0Ya7u62XosU89tt93WFi6Pa2hjAiN0vtfly5fLGU3ZO6VTJ9mhqYGLKyvb2FW27cuZqZmX3LTGo0ePbtsXy9UhgAACHglMmzZN9u7dK4cPH9Z7yAlSkDxVYksO6aCxAunAKFqP3gX/nuacBmUrohOdtqTNW2zFspXMmjXLvxdFzS9LoIPO1+p9fprLqnLtzpZabNGiRc4TYZVlkqTpxSIrimo3YA2BJgqUxqQ6PVbOic5foz1M58yZI8nJyU3cm80Q8EagsLBQ5s6d69wksS/uFTq3ESNovbG/3LMM0R6W/9C1q0TqzX27wf/www+LjUigIIAAAgj4SyAvL0/mz58vxfpvMAWByxFISEiQ+++/n2w9l4PIvq0qsHr1almxYoVzjkINzP5NO8Yf1VSOlNAWiNK2xi36/XK1dv6wYiNxHnvsMUbOhvbbRu0QQMBHAtZRc8GCBXLggP/mnPURc7uoalJSkjzwwAPOAMR2ccFc5AUCvg7O2tUsXLhQsrKyai7snMSU5knH8nyxYC0FgcYELLhVGR4rZbG9dORDUmDTmTNnyvjx4wOPWUEglAQsvZh977n9avL1Bslm/WFYoOmOK/3b1yaUiFusLmF6Y6SrdvYYpqNl++jIZ7dMnz5dpk6d6j7kLwIIIICAzwTKdATZqlWrZMOGDWLrFASaIxCjvwtsFNuUKVMkuiblaHP2Z1sEvBKoqqqSefPmye7du51TVul/t2u7Y095uZzUYK1ve/l7BejxeSwo20vbHCM1KBsXFuac3UbnP/LII2Ts8fi94HQIIND2BeyeXEZGhqxbt05sIAUFgeYIxMXFyZgxY2TixIkSqdkRKe1XwPfBWWswLF26VDIzM9vvu8iVt4iAjZidMWOGTJgwoUWOx0EQaC0BC9C+/fbbUq43Rij+EbDvGAvMTpo0yT+VpqYIIIAAAg0K2Dzwubm5zg0ZW6cg0JhARESEdNUsGqmpqUyf0hgUr4WUgH23LVmyxJl/NqQqRmUuKmDTgN17772Mzr+oFBsggAACly5gQdr8/HxnsakAKAg0JmDtARstm5KSQkaLxqDa0Wu+D86679WOHTvE0u7k5OS4T/EXgSYJhGmv0kGDBonNG2BfjhQE/CBw4sQJZ9SOBWorSC8W0m+ZfccMHjxYpupoWUsrRkEAAQQQQAABBBBAwE8C2dnZkp6ezv0WH7xp8fHxcv3118sNN9wgUTqSloIAAggggAACCCAQmgJtJjgbmrzUCgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEKgWqJ6IAg0EEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAgVYVIDjbqrwcHAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEKgWIDjLJwEBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBDwQIDgrAfInAIBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBAgOMtnAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEPBAgOCsB8icAgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECA4y2cAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8ECA4KwHyJwCAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQIDjLZwABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBDwQIDgrAfInAIBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBAgOMtnAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEPBAgOCsB8icAgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECA4y2cAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8ECA4KwHyJwCAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQIDjLZwABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBDwQIDgrAfInAIBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBAgOMtnAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEPBAgOCsB8icAgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECA4y2cAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8ECA4KwHyJwCAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQIDjLZwABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBDwQIDgrAfInAIBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBAgOMtnAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEPBAgOCsB8icAgEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECA4y2cAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQ8ECA4KwHyJwCAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQIDjLZwABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBDwQOD/ARm1jEn2v65uAAAAAElFTkSuQmCC)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Hivr6J-3WuV_"},"outputs":[],"source":["%%capture\n","!pip install --upgrade openai\n","!pip install jsonlines"]},{"cell_type":"markdown","metadata":{"id":"Z8RFaAaEW-oI"},"source":["You can find your OpenAI API key [here](https://beta.openai.com/account/api-keys).\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2142,"status":"ok","timestamp":1651342007058,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"},"user_tz":240},"id":"OdtifP-QW8GK","outputId":"831384e0-3692-4e70-fc99-999cd017a679"},"outputs":[{"name":"stdout","output_type":"stream","text":["Enter OpenAI API key: ··········\n"]}],"source":["import os\n","from getpass import getpass\n","import openai\n","\n","openai.api_key = getpass('Enter OpenAI API key: ')\n","\n","os.environ['OPENAI_API_KEY'] = openai.api_key"]},{"cell_type":"markdown","metadata":{"id":"Z0Qd3JFuvEnE"},"source":["The OpenAI API documentation has a [guide to fine-tuning models](https://beta.openai.com/docs/guides/fine-tuning) that you should read.   The basic format of fine-tuning data is a JSONL file (one JSON object per line) with two key-value pairs: `prompt:` and `completion:`.\n","\n","```\n","{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n","{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n","{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n","...\n","```"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"4kIr8uDwvHB_"},"outputs":[],"source":["from pathlib import Path\n","from tqdm.auto import tqdm\n","import itertools\n","import random\n","import json\n","import re\n","\n","\n","data_dir = Path(root_dir) / 'processed_multiwoz'\n","db = MultiWozDB(Path(root_dir) / 'MultiWOZ_2.1' / 'db') # for simulating database query\n","\n","\n","def constraint_to_dict(constraint):\n","    constraint_dict = {}\n","    domain = None\n","    last_match = None\n","    for match in re.finditer(r\"\\[(\\w+)\\]\", constraint):\n","        if domain and last_match and re.match(r\"\\[value_[a-z]+\\]\", last_match.group(0)):\n","            slot = last_match.group(1)[6:]\n","            value = constraint[last_match.end():match.start()].strip()\n","            constraint_dict[domain][slot] = value\n","\n","        if not re.match(r\"\\[value_[a-z]+\\]\", match.group(0)):\n","            if match.group(1) in ALL_DOMAINS:\n","                domain = match.group(1)\n","                constraint_dict[domain] = {}\n","            else:\n","                print('Invalid constraint token:', match.group(1))\n","                print('Constraint string:', constraint)\n","\n","        last_match = match\n","\n","    if domain and last_match and re.match(r\"\\[value_[a-z]+\\]\", last_match.group(0)):\n","        slot = last_match.group(1)[6:] # [value_name] -> name\n","        value = constraint[last_match.end():].strip()\n","        constraint_dict[domain][slot] = value\n","\n","    return constraint_dict\n","\n","\n","def constraint_to_db_result(constraint, turn_domain):\n","    constraint_dict = constraint_to_dict(constraint)\n","\n","    match_nums = db.get_match_num(constraint_dict)\n","    match_domain = turn_domain[0] if len(turn_domain) == 1 else turn_domain[1]\n","    match_domain = match_domain[1:-1] if match_domain.startswith(\"[\") else match_domain\n","    match = match_nums[match_domain]\n","    return db.addDBIndicator(match_domain, match)[4:-1]\n","\n","\n","def create_finetuning_data(filename, max_dialogs=8433):\n","    fine_tuning_data = []\n","\n","    with open(Path(data_dir) / 'train_data.json') as inp:\n","        data = json.load(inp)\n","    \n","    print(\"# dialogs in dataset:\", len(data))\n","\n","    for dialog in tqdm(data.values()):\n","        data = {}\n","        turn_strs = []\n","        for turn in dialog['log']:\n","            db_res = constraint_to_db_result(turn['constraint'], turn['turn_domain'].split())\n","            turn_strs.append(\n","                f\"user: {turn['user']}\\n\" \n","                f\"state: {turn['constraint']}\\n\"\n","                f\"match: {db_res}\\n\"\n","                f\"action: {turn['sys_act']}\\n\"\n","                f\"response: {turn['resp']}\\n\"\n","            )\n","        \n","        data['prompt'] = ''\n","        data['completion'] = '###\\n'.join(turn_strs)\n","        fine_tuning_data.append(data)\n","    \n","    fine_tuning_data = random.sample(fine_tuning_data, k=max_dialogs)\n","    with open(filename, 'w') as out:\n","        for data in fine_tuning_data:\n","            out.write(json.dumps(data))\n","            out.write('\\n')"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":277,"referenced_widgets":["e65507122c8546a7b775cf2ac303d830","44bcad28d38c4d90a0f647ba5eb7cf15","113f485c51b94274baf77c15cb9abbb1","44737be04f924cdc9a0b8e68516070e7","483a3a4b77ce4b16a73e0e03ba120a94","d796229fee5b415fba8bdc7ed67f0030","c052b264d10a437eb741c8d6b09e4883","aee2fd94b8d84b818bb455ded2c40bcd","c10d3b18fe8245c5a7e18144210dd18d","5549fb8db17d477c9b31114ef001196e","9576e77f50b94f5ca953da58a63a75f9"]},"executionInfo":{"elapsed":231863,"status":"ok","timestamp":1651097496070,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"id":"4TJ2vnq_9jI7","outputId":"003d8ebe-b8ce-494e-c83e-d9212cd8de4b"},"outputs":[{"output_type":"stream","name":"stdout","text":["# dialogs in dataset: 8433\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/8433 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e65507122c8546a7b775cf2ac303d830"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["{\"prompt\": \"\", \"completion\": \"user: i need a place to eat that is cheap .\\nstate: [restaurant] [value_pricerange] cheap\\nmatch: 3\\naction: [restaurant] [request] area\\nresponse: does location matter ?\\n###\\nuser: yes , in the south , i am also looking for romanian food .\\nstate: [restaurant] [value_pricerange] cheap [value_area] south [value_food] romanian\\nmatch: 0\\naction: [restaurant] [nooffer] [general] [reqmore]\\nresponse: there are no restaurants that meet what you 're looking for . is there anything else you 'd like to try ?\\n###\\nuser: same area and price , but portuguese , anything for that ?\\nstate: [restaurant] [value_pricerange] cheap [value_area] south [value_food] portuguese\\nmatch: 1\\naction: [restaurant] [nooffer]\\nresponse: sorry i do n't have anything for that either .\\n###\\nuser: can you try chinese ?\\nstate: [restaurant] [value_pricerange] cheap [value_area] south [value_food] chinese\\nmatch: 1\\naction: [restaurant] [nooffer]\\nresponse: sorry i do n't have anything for that either .\\n###\\nuser: you choose as long as it is cheap . could you book a table for 7 people at 15:45 on wednesday ?\\nstate: [restaurant] [value_pricerange] cheap [value_area] south [value_food] dontcare [value_day] wednesday [value_people] 7 [value_time] 15:45\\nmatch: 2\\naction: [restaurant] [offerbooked] area food pricerange\\nresponse: you are booked at [value_name] , a [value_pricerange] [value_food] restaurant in the [value_area] . your reference number is [value_reference] .\\n###\\nuser: are there any museums close to restaurant ?\\nstate: [attraction] [value_area] south [value_type] museum [restaurant] [value_pricerange] cheap [value_area] south [value_food] dontcare [value_day] wednesday [value_people] 7 [value_time] 15:45 [value_name] nando's\\nmatch: 1\\naction: [attraction] [inform] area price name phone\\nresponse: i found only [value_name] in the [value_area] . there 's [value_price] . phone number , [value_phone] .\\n###\\nuser: what is the address for the museum ?\\nstate: [attraction] [value_area] south [value_type] museum [restaurant] [value_pricerange] cheap [value_area] south [value_food] dontcare [value_day] wednesday [value_people] 7 [value_time] 15:45 [value_name] nandos\\nmatch: 1\\naction: [attraction] [inform] address area price\\nresponse: it is in the [value_area] address is [value_address] and [value_price] admittance\\n###\\nuser: great ! thanks . have a good night . bye .\\nstate: [attraction] [value_area] south [value_type] museum [restaurant] [value_pricerange] cheap [value_area] south [value_food] dontcare [value_day] wednesday [value_people] 7 [value_time] 15:45 [value_name] nandos\\nmatch: null\\naction: [general] [welcome] [bye]\\nresponse: you 're welcome , let me know if i can assist with anything else .\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: i need a place to eat that is expensive .\\nstate: [restaurant] [value_pricerange] expensive\\nmatch: 3\\naction: [restaurant] [inform] choice pricerange\\nresponse: there are [value_choice] options for [value_pricerange] dining .\\n###\\nuser: specifically , i would like an indian restaurant .\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian\\nmatch: 3\\naction: [restaurant] [inform] choice food pricerange [request] area\\nresponse: there are [value_choice] of [value_pricerange] [value_food] options . is there a specific part of town you were looking for ?\\n###\\nuser: not really , as long as it is open on monday and has room for 5 people .\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian [value_area] dontcare [value_day] monday [value_people] 5\\nmatch: 3\\naction: [restaurant] [offerbook] name\\nresponse: it appears that the [value_name] meets your needs ! would you like me to book that for you ?\\n###\\nuser: yes , please make reservations for 5 people at 16:45 on monday .\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian [value_area] dontcare [value_day] monday [value_people] 5 [value_name] curry garden [value_time] 16:45\\nmatch: 1\\naction: [restaurant] [offerbooked] reference [general] [reqmore]\\nresponse: booking was successful . the table will be reserved for 15 minutes . reference number is : [value_reference] . anything else i can help you with ?\\n###\\nuser: thanks . i also need a train from bishops stortford to cambridge .\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian [value_area] dontcare [value_day] monday [value_people] 5 [value_name] curry garden [value_time] 16:45 [train] [value_departure] bishops stortford [value_destination] cambridge\\nmatch: 3\\naction: [train] [request] day\\nresponse: what day will you travel ?\\n###\\nuser: i will be traveling on monday .\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian [value_area] dontcare [value_day] monday [value_people] 5 [value_name] curry garden [value_time] 16:45 [train] [value_departure] bishops stortford [value_destination] cambridge [value_day] monday\\nmatch: 2\\naction: [train] [request] leave\\nresponse: okay , is there a certain time you need to leave ?\\n###\\nuser: i would like to arrive by 11:00 . could you give me the departure time and train id ?\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian [value_area] dontcare [value_day] monday [value_people] 5 [value_name] curry garden [value_time] 16:45 [train] [value_departure] bishops stortford [value_destination] cambridge [value_day] monday [value_arrive] 11:00\\nmatch: 1\\naction: [train] [inform] arrive id\\nresponse: i can suggest the [value_id] arriving by [value_arrive] .\\n###\\nuser: is there a departure time ?\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian [value_area] dontcare [value_day] monday [value_people] 5 [value_name] curry garden [value_time] 16:45 [train] [value_departure] bishops stortford [value_destination] cambridge [value_day] monday [value_arrive] 11:00\\nmatch: 1\\naction: [train] [inform] leave\\nresponse: that train departs at [value_leave] .\\n###\\nuser: thank you . that is all that i need .\\nstate: [restaurant] [value_pricerange] expensive [value_food] indian [value_area] dontcare [value_day] monday [value_people] 5 [value_name] curry garden [value_time] 16:45 [train] [value_departure] bishops stortford [value_destination] cambridge [value_day] monday [value_arrive] 11:00\\nmatch: null\\naction: [general] [bye]\\nresponse: thank you for contacting us and have a nice day .\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: i need a train out of peterborough on wednesday .\\nstate: [train] [value_day] wednesday [value_departure] peterborough\\nmatch: 3\\naction: [train] [inform] choice [request] leave\\nresponse: we have [value_choice] so if we can narrow down by giving departure time good with you\\n###\\nuser: yes , i need to arrive in cambridge at or a little before 21:45 .\\nstate: [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge\\nmatch: 3\\naction: [train] [inform] choice [request] leave\\nresponse: i have [value_choice] trains available . what time would you like to depart ?\\n###\\nuser: whatever the closest train to arriving before 21:45 is will be fine with me .\\nstate: [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge [value_leave] dontcare\\nmatch: 3\\naction: [train] [offerbooked] arrive people reference id [general] [reqmore]\\nresponse: i got [value_people] ticket for you on [value_id] it will arrive by [value_arrive] . your reference number is [value_reference] . anything else today ?\\n###\\nuser: can you cancel that and get me 5 tickets instead of 1 please ?\\nstate: [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge [value_leave] dontcare [value_people] 5\\nmatch: 3\\naction: [train] [offerbooked] price reference\\nresponse: sure , this is the new information : booking was successful , the total fee is [value_price] payable at the station . reference number is : [value_reference] .\\n###\\nuser: thank you so much ! i also need some entertainment in the centre . what do you recommend ?\\nstate: [attraction] [value_area] centre [value_type] entertainment [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge [value_leave] dontcare [value_people] 5\\nmatch: 0\\naction: [attraction] [inform] address price name type\\nresponse: [value_name] is nice if you like [value_type] ! it is on [value_address] and the entrance fee is [value_price] !\\n###\\nuser: i would prefer a lively college in the centre , please .\\nstate: [attraction] [value_area] centre [value_type] college [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge [value_leave] dontcare [value_people] 5\\nmatch: 3\\naction: [attraction] [inform] name\\nresponse: okay ! how about [value_name] ?\\n###\\nuser: sure , i just need the address , postcode , and entrance fee .\\nstate: [attraction] [value_area] centre [value_type] college [value_name] christ college [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge [value_leave] dontcare [value_people] 5\\nmatch: 1\\naction: [attraction] [inform] address price name postcode\\nresponse: [value_name] is [value_price] to enter , and they 're located on [value_address] in postcode [value_postcode] .\\n###\\nuser: ok . thank you . that 's all for now .\\nstate: [attraction] [value_area] centre [value_type] college [value_name] christ college [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge [value_leave] dontcare [value_people] 5\\nmatch: null\\naction: [general] [welcome]\\nresponse: thank you for using our services\\n###\\nuser: have a great day !\\nstate: [attraction] [value_area] centre [value_type] college [value_name] christ college [train] [value_day] wednesday [value_departure] peterborough [value_arrive] 21:45 [value_destination] cambridge [value_leave] dontcare [value_people] 5\\nmatch: null\\naction: [general] [bye]\\nresponse: have a nice day .\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: i am looking for an expensive restaurant in the centre of town .\\nstate: [restaurant] [value_area] centre [value_pricerange] expensive\\nmatch: 3\\naction: [restaurant] [request] food\\nresponse: we can narrow down our selection for you if we know the restaurant chain and the kind of food you prefer ?\\n###\\nuser: i am open to any kind of food . i am looking for something in the centre and on the expensive side .\\nstate: [restaurant] [value_area] centre [value_pricerange] expensive\\nmatch: 3\\naction: [restaurant] [inform] choice [recommend] food name [offerbook]\\nresponse: there are [value_choice] of good choices like that . i can recommend [value_name] if [value_food] appeals to you . would you like me to book you a table ?\\n###\\nuser: that sounds lovely . there will be 2 of us at 16:45 on tuesday please .\\nstate: [restaurant] [value_area] centre [value_pricerange] expensive [value_food] british [value_name] cambridge chop house [value_day] tuesday [value_people] 2 [value_time] 16:45\\nmatch: 1\\naction: [restaurant] [offerbooked] reference [general] [reqmore]\\nresponse: i was able to get a reservation at that time . your reference number for your reservation is [value_reference] . is there anything else i can help you with ?\\n###\\nuser: i would also like to find somewhere to go in town .\\nstate: [restaurant] [value_area] centre [value_pricerange] expensive [value_food] british [value_name] cambridge chop house [value_day] tuesday [value_people] 2 [value_time] 16:45\\nmatch: null\\naction: [attraction] [inform] choice [request] type\\nresponse: we have [value_choice] attractions to choose from , do you have a preference as to the type ?\\n###\\nuser: is there a cinema ?\\nstate: [attraction] [value_type] cinema [restaurant] [value_area] centre [value_pricerange] expensive [value_food] british [value_name] cambridge chop house [value_day] tuesday [value_people] 2 [value_time] 16:45\\nmatch: 2\\naction: [attraction] [recommend] name\\nresponse: you would love [value_name]\\n###\\nuser: thank you , i just need their postal code for now and i can take it from there .\\nstate: [attraction] [value_type] cinema [value_name] cineworld cinema [restaurant] [value_area] centre [value_pricerange] expensive [value_food] british [value_name] cambridge chop house [value_day] tuesday [value_people] 2 [value_time] 16:45\\nmatch: 1\\naction: [attraction] [inform] postcode\\nresponse: the postcode for cineworld cinema is [value_postcode]\\n###\\nuser: okay , thank you so much for your help .\\nstate: [attraction] [value_type] cinema [value_name] cineworld cinema [restaurant] [value_area] centre [value_pricerange] expensive [value_food] british [value_name] cambridge chop house [value_day] tuesday [value_people] 2 [value_time] 16:45\\nmatch: null\\naction: [general] [welcome]\\nresponse: you are very welcome . is there anything else i can assist you with ?\\n###\\nuser: that is all for now . thank you for your help .\\nstate: [attraction] [value_type] cinema [value_name] cineworld cinema [restaurant] [value_area] centre [value_pricerange] expensive [value_food] british [value_name] cambridge chop house [value_day] tuesday [value_people] 2 [value_time] 16:45\\nmatch: null\\naction: [general] [welcome]\\nresponse: you are welcome enjoy .\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: i need a place to stay that is cheap .\\nstate: [hotel] [value_pricerange] cheap\\nmatch: 3\\naction: [hotel] [request] area\\nresponse: okay and what area will you be staying in ?\\n###\\nuser: i need a place in the center of town , and i would prefer a hotel over a guesthouse .\\nstate: [hotel] [value_pricerange] cheap [value_area] centre [value_type] hotel\\nmatch: 0\\naction: [hotel] [inform] choice area pricerange type [offerbook]\\nresponse: i have [value_choice] [value_type] in the [value_area] , one is [value_pricerange] and then there are [value_choice] [value_pricerange] ones . would you like to book one of them ?\\n###\\nuser: i want to book one of the expensive ones .\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel\\nmatch: 2\\naction: [hotel] [inform] internet name parking stars\\nresponse: the expensive ones are [value_name] and [value_name] . both offer free wifi and parking . the gonville is [value_stars] star rated , and the university arms is [value_stars] star .\\n###\\nuser: great . can i book the university arms for 2 nights for 4 people starting sunday please ? i 'll need a reference number .\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2\\nmatch: 1\\naction: [hotel] [offerbooked] reference [general] [reqmore]\\nresponse: ok , you 're all set . reference # megfyqha . is there anything else i can help you with today ?\\n###\\nuser: can you help me find a hungarian restaurant , also in the centre of town ?\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2 [restaurant] [value_area] centre [value_food] hungarian\\nmatch: 0\\naction: [restaurant] [nooffer] food [request] area\\nresponse: i am sorry , there are no [value_food] restaurants in the [value_area] , would you like another location ?\\n###\\nuser: is there and indian restaurant available in that area ?\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2 [restaurant] [value_area] centre [value_food] indian\\nmatch: 3\\naction: [restaurant] [inform] choice food [select] pricerange\\nresponse: there are [value_choice] [value_food] restaurants . do you prefer [value_pricerange] or [value_pricerange] ?\\n###\\nuser: i would like expensive please . l\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2 [restaurant] [value_area] centre [value_food] indian [value_pricerange] expensive\\nmatch: 3\\naction: [restaurant] [inform] choice name\\nresponse: there are [value_choice] options in that area . how about [value_name] ? i hear it 's great .\\n###\\nuser: can you book a reservation for me ?\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2 [restaurant] [value_area] centre [value_food] indian [value_pricerange] expensive [value_name] curry garden\\nmatch: 1\\naction: [restaurant] [request] day\\nresponse: absolutely , what day would you like reservations ?\\n###\\nuser: i would like to book it fro sunday at 14:45 .\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2 [restaurant] [value_area] centre [value_food] indian [value_pricerange] expensive [value_name] curry garden [value_day] sunday [value_time] 14:45\\nmatch: 1\\naction: [restaurant] [request] people\\nresponse: how many people will be dining ?\\n###\\nuser: 4 people will be dining .\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2 [restaurant] [value_area] centre [value_food] indian [value_pricerange] expensive [value_name] curry garden [value_day] sunday [value_time] 14:45 [value_people] 4\\nmatch: 1\\naction: [restaurant] [offerbooked] reference [general] [reqmore]\\nresponse: your table is booked , ref#9bzul5gv . is there anything else i can help you with today ?\\n###\\nuser: i think that is all i need today . thank you for all your help . good bye .\\nstate: [hotel] [value_pricerange] expensive [value_area] centre [value_type] hotel [value_name] university arms hotel [value_day] sunday [value_people] 4 [value_stay] 2 [restaurant] [value_area] centre [value_food] indian [value_pricerange] expensive [value_name] curry garden [value_day] sunday [value_time] 14:45 [value_people] 4\\nmatch: null\\naction: [general] [bye]\\nresponse: have a pleasant trip and , as always , we are here if you need us .\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: i need to get in touch with the police , i have just been the victim of a robbery .\\nstate: \\nmatch: null\\naction: [police] [inform] phone\\nresponse: i am so sorry to hear that . i hope you are all right . you can call them at the number [value_phone] .\\n###\\nuser: can i get the address and post code ?\\nstate: \\nmatch: null\\naction: [police] [inform] address postcode\\nresponse: the address is [value_address] , postcode [value_postcode] . call back if you need further help .\\n###\\nuser: thanks , that 's all . good bye .\\nstate: \\nmatch: null\\naction: [general] [welcome] [bye]\\nresponse: welcome . at your service next time\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: do you have any trains leaving after 15:00 on thursday ?\\nstate: [train] [value_day] thursday [value_leave] 15:00\\nmatch: 3\\naction: [train] [request] destination\\nresponse: yes , there are many . what is your destination ?\\n###\\nuser: my destination is norwich departing from cambridge .\\nstate: [train] [value_day] thursday [value_leave] 15:00 [value_departure] cambridge [value_destination] norwich\\nmatch: 2\\naction: [train] [inform] leave id [offerbook]\\nresponse: [value_id] leaves at [value_leave] , would this one work for you ?\\n###\\nuser: yes , that would be find . i 'll need to book tickets for 7 people . can i have the reference number please ?\\nstate: [train] [value_day] thursday [value_leave] 15:00 [value_departure] cambridge [value_destination] norwich [value_people] 7\\nmatch: 2\\naction: [train] [offerbooked] price reference\\nresponse: booking was successful , the total fee is [value_price] payable at the station . reference number is : [value_reference] .\\n###\\nuser: thank you ! can you help me find somewhere to eat now ?\\nstate: [train] [value_day] thursday [value_leave] 15:00 [value_departure] cambridge [value_destination] norwich [value_people] 7\\nmatch: null\\naction: [restaurant] [request] food pricerange\\nresponse: sure , i can help you with that . do you have a preference in food type or price range ?\\n###\\nuser: yes , do you have a place that serves corsican food ? i also need it to be on the cheap side .\\nstate: [restaurant] [value_food] corsica [value_pricerange] cheap [train] [value_day] thursday [value_leave] 15:00 [value_departure] cambridge [value_destination] norwich [value_people] 7\\nmatch: 0\\naction: [restaurant] [nooffer] food pricerange [request] name\\nresponse: no , unfortunately , i do not have any [value_pricerange] [value_food] restaurants . would you like to look for another restaurant ?\\n###\\nuser: bummer ! i guess i 'll try some asian oriental food then\\nstate: [restaurant] [value_food] asian oriental [value_pricerange] cheap [train] [value_day] thursday [value_leave] 15:00 [value_departure] cambridge [value_destination] norwich [value_people] 7\\nmatch: 2\\naction: [restaurant] [select] area name\\nresponse: i have two that fit your criteria ! [value_name] , [value_name] are both in the [value_area] area . do you have a preference ?\\n###\\nuser: can you suggest one and i need the address , postcode and their phone number too please\\nstate: [restaurant] [value_food] asian oriental [value_pricerange] cheap [train] [value_day] thursday [value_leave] 15:00 [value_departure] cambridge [value_destination] norwich [value_people] 7\\nmatch: 2\\naction: [restaurant] [recommend] address name phone postcode\\nresponse: i would recommend the [value_name] . the address is [value_address] . the postcode is [value_postcode] . the phone number is [value_phone] .\\n###\\nuser: wonderful ! i think that 's all i need for now , thank you so much for your help !\\nstate: [restaurant] [value_food] asian oriental [value_pricerange] cheap [value_name] dojo noodle bar [train] [value_day] thursday [value_leave] 15:00 [value_departure] cambridge [value_destination] norwich [value_people] 7\\nmatch: 1\\naction: [general] [welcome]\\nresponse: you 're welcome , have a great day !\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: hello . i need to book a train ride from ely going to cambridge , can you help me with that ?\\nstate: [train] [value_departure] ely [value_destination] cambridge\\nmatch: 3\\naction: [train] [inform] choice [request] day\\nresponse: absolutely , you have [value_choice] of options . on what day will you be travelling ?\\n###\\nuser: i 'll be leaving on saturday after 12:00 .\\nstate: [train] [value_departure] ely [value_destination] cambridge [value_day] saturday [value_leave] 12:00\\nmatch: 2\\naction: [train] [inform] choice day leave [request] arrive\\nresponse: i have [value_choice] trains leaving after [value_leave] on [value_day] . do you have a preferred time of arrival to cambridge ?\\n###\\nuser: no , i do not . i would , however , like to know the travel time .\\nstate: [train] [value_departure] ely [value_destination] cambridge [value_day] saturday [value_leave] 12:00\\nmatch: 2\\naction: [train] [inform] time\\nresponse: the travel time is [value_duration] . would you like the departure times ?\\n###\\nuser: could you please confirm that train id number with the travel time for me ?\\nstate: [train] [value_departure] ely [value_destination] cambridge [value_day] saturday [value_leave] 12:00\\nmatch: 2\\naction: [train] [inform] arrive time leave id\\nresponse: [value_id] leaves at [value_leave] and it will take [value_duration] to travel before arriving at [value_arrive]\\n###\\nuser: thank you . i would also like to find a hotel .\\nstate: [hotel] [value_type] hotel [train] [value_departure] ely [value_destination] cambridge [value_day] saturday [value_leave] 12:00\\nmatch: 3\\naction: [hotel] [inform] choice type [request] pricerange\\nresponse: i have about [value_choice] different [value_type]s to choose from . is there a price range that would work better for you so we can narrow it down ?\\n###\\nuser: i would like a hotel in the north area in the moderate price range .\\nstate: [hotel] [value_type] hotel [value_area] north [value_pricerange] moderate [train] [value_departure] ely [value_destination] cambridge [value_day] saturday [value_leave] 12:00\\nmatch: 2\\naction: [hotel] [inform] name\\nresponse: [value_name] or [value_name] should suit your needs .\\n###\\nuser: what is the address for the ashley hotel ?\\nstate: [hotel] [value_type] hotel [value_area] north [value_pricerange] moderate [value_name] ashley hotel [train] [value_departure] ely [value_destination] cambridge [value_day] saturday [value_leave] 12:00\\nmatch: 1\\naction: [hotel] [inform] address name\\nresponse: the address for [value_name] is [value_address] .\\n###\\nuser: well that 's all i needed . you have been very helpful today .\\nstate: [hotel] [value_type] hotel [value_area] north [value_pricerange] moderate [value_name] ashley hotel [train] [value_departure] ely [value_destination] cambridge [value_day] saturday [value_leave] 12:00\\nmatch: null\\naction: [general] [welcome] [bye]\\nresponse: thank you . goodbye .\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: can you help me find a place to stay ? i am looking for somewhere that include free wifi and free parking .\\nstate: [hotel] [value_internet] yes [value_parking] yes\\nmatch: 3\\naction: [hotel] [inform] choice internet parking [request] area\\nresponse: i found [value_choice] hotels with free internet and parking . what area are you looking ?\\n###\\nuser: area does n't matter , but i would like a cheap guesthouse , if possible .\\nstate: [hotel] [value_internet] yes [value_parking] yes [value_pricerange] cheap [value_type] guesthouse\\nmatch: 3\\naction: [hotel] [inform] area internet name parking pricerange type [offerbook]\\nresponse: [value_name] is a [value_type] in the north area in the [value_pricerange] range , with parking and wifi . would you care to book a room ?\\n###\\nuser: could i get the hotel 's phone number and postcode please ?\\nstate: [hotel] [value_internet] yes [value_parking] yes [value_pricerange] cheap [value_type] guesthouse\\nmatch: 3\\naction: [hotel] [inform] name phone postcode [general] [reqmore]\\nresponse: yes , [value_name] has a postal code of [value_postcode] . they can be reached at [value_phone] . may i assist with anything else ?\\n###\\nuser: no , you helped me out . thanks . goodbye .\\nstate: [hotel] [value_internet] yes [value_parking] yes [value_pricerange] cheap [value_type] guesthouse\\nmatch: null\\naction: [general] [bye]\\nresponse: enjoy your stay ! goodbye !\\n\"}\n","{\"prompt\": \"\", \"completion\": \"user: i am looking for a hotel , can you help ?\\nstate: [hotel] [value_type] hotel\\nmatch: 3\\naction: [hotel] [inform] choice type [general] [reqmore]\\nresponse: i can , we have [value_choice] [value_type] , do you have anything else to narrow it down ?\\n###\\nuser: it should be in the centre and be on the expensive side .\\nstate: [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive\\nmatch: 2\\naction: [hotel] [inform] choice address name type [offerbook]\\nresponse: i have [value_choice] available [value_type] . one is the [value_name] located on [value_address] . the other is [value_name] on [value_address] . which one would you like to book ?\\n###\\nuser: can you book the one without wifi ?\\nstate: [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] monday [value_people] 1 [value_stay] 1\\nmatch: 2\\naction: [hotel] [inform] internet [request] day people\\nresponse: they both have internet , for what days and how many people should i book ?\\n###\\nuser: do either of them offer free wifi ?\\nstate: [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] monday [value_people] 1 [value_stay] 1\\nmatch: 2\\naction: [hotel] [inform] internet [offerbook]\\nresponse: as previously mentioned , they both have internet available . i do n't have specifications about the internet service however . would you like to book one of these ?\\n###\\nuser: yes , please book for two nights beginning on sunday for one person .\\nstate: [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] sunday [value_people] 1 [value_stay] 1 [value_name] gonville hotel\\nmatch: 1\\naction: [hotel] [offerbooked] reference\\nresponse: your booking was successful and the reference number is [value_reference] .\\n###\\nuser: thank you . i am also looking for a museum to go to in the same area .\\nstate: [attraction] [value_area] centre [value_type] museum [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] sunday [value_people] 1 [value_stay] 1 [value_name] gonville hotel\\nmatch: 3\\naction: [attraction] [inform] choice price [general] [reqmore]\\nresponse: there are [value_choice] all with [value_price] entrance . would you like me to recommend one ?\\n###\\nuser: yes , please . can i get the address and phone number for the one you reccommend ?\\nstate: [attraction] [value_area] centre [value_type] museum [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] sunday [value_people] 1 [value_stay] 1 [value_name] gonville hotel\\nmatch: 3\\naction: [attraction] [recommend] address name postcode [general] [reqmore]\\nresponse: [value_name] is a nice place , their address is [value_address] and their postcode is [value_postcode] , anything else today ?\\n###\\nuser: do you have there phone number ?\\nstate: [attraction] [value_area] centre [value_type] museum [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] sunday [value_people] 1 [value_stay] 1 [value_name] gonville hotel\\nmatch: 3\\naction: [attraction] [inform] phone [general] [reqmore]\\nresponse: their phone is [value_phone] . is there anything else i can help you with today ?\\n###\\nuser: yes , i also need a taxi going between the two places .\\nstate: [attraction] [value_area] centre [value_type] museum [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] sunday [value_people] 1 [value_stay] 1 [value_name] gonville hotel\\nmatch: null\\naction: [taxi] [request] destination leave\\nresponse: what time do you need the taxi , and which is the destination site ?\\n###\\nuser: i want to arrive at the castle galleries by 11:15 .\\nstate: [attraction] [value_area] centre [value_type] museum [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] sunday [value_people] 1 [value_stay] 1 [value_name] gonville hotel [taxi] [value_arrive] 11:15 [value_destination] castle galleries\\nmatch: null\\naction: [taxi] [inform] phone type\\nresponse: your [value_type] taxi booking was successful . its contact number is [value_phone] .\\n###\\nuser: ok , thank you for the help .\\nstate: [attraction] [value_area] centre [value_type] museum [hotel] [value_type] hotel [value_area] centre [value_pricerange] expensive [value_day] sunday [value_people] 1 [value_stay] 1 [value_name] gonville hotel [taxi] [value_arrive] 11:15 [value_destination] castle galleries\\nmatch: null\\naction: [general] [reqmore]\\nresponse: was there anything else you needed today ?\\n\"}\n","    8433  2807597 fine_tune_multiwoz.jsonl\n"]}],"source":["|jsonl_filename='fine_tune_multiwoz.jsonl'\n","create_finetuning_data(jsonl_filename, max_dialogs=8433)\n","\n","!head '{jsonl_filename}' -n 10\n","!wc -lw '{jsonl_filename}'"]},{"cell_type":"markdown","metadata":{"id":"7n2Alv9S6L2l"},"source":["Next, we'll make the fine tuning API call via the command line.  Here the -m argument gives the model.  There are 4 sizes of GPT3 models.  They go in alphabetical order from smallest to largest.\n","* Ada \n","* Baddage\n","* Currie\n","* Davinci\n","\n","The models as the model sizes increase, so does their quality and their cost.  Davinci is the highest quality and highest cost model.  I recommend starting by fine-tuning smaller models to debug your code first so that you don't rack up costs.\n","\n","Fine-tuning babbage on 8433 dialogues costs about $16.\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":629535,"status":"ok","timestamp":1650779033516,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"id":"3n7F3ZBT9tZc","outputId":"e00db513-95fc-4368-f844-c9198e9a4693"},"outputs":[{"output_type":"stream","name":"stdout","text":["Logging requires wandb to be installed. Run `pip install wandb`.\n","Upload progress: 100% 21.8M/21.8M [00:00<00:00, 40.4Git/s]\n","Uploaded file from fine_tune_multiwoz.jsonl: file-h6BXLKo2PRdW0M5QbnLbFe1a\n","Created fine-tune: ft-rLXdG9F7riQWgbMmaqYO7tVi\n","Streaming events until fine-tuning is complete...\n","\n","(Ctrl-C will interrupt the stream, but not cancel the fine-tune)\n","[2022-04-24 05:33:29] Created fine-tune: ft-rLXdG9F7riQWgbMmaqYO7tVi\n","[2022-04-24 05:33:48] Fine-tune costs $15.49\n","[2022-04-24 05:33:48] Fine-tune enqueued. Queue number: 0\n","[2022-04-24 05:33:52] Fine-tune started\n","\n","Stream interrupted (client disconnected).\n","To resume the stream, run:\n","\n","  openai api fine_tunes.follow -i ft-rLXdG9F7riQWgbMmaqYO7tVi\n","\n"]}],"source":["!openai api fine_tunes.create -t '{jsonl_filename}' -m babbage --n_epochs 4 --suffix \"ubar\""]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2389,"status":"ok","timestamp":1650739490784,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"id":"4KfOR5DGfL-e","outputId":"1e07b4c7-1e2e-48f6-8696-6cc58a6ee895"},"outputs":[{"output_type":"stream","name":"stdout","text":["Logging requires wandb to be installed. Run `pip install wandb`.\n","{\n","  \"created_at\": 1650737164,\n","  \"events\": [\n","    {\n","      \"created_at\": 1650737165,\n","      \"level\": \"info\",\n","      \"message\": \"Created fine-tune: ft-BltVwDo5jw3kagGETnz64IvK\",\n","      \"object\": \"fine-tune-event\"\n","    },\n","    {\n","      \"created_at\": 1650737180,\n","      \"level\": \"info\",\n","      \"message\": \"Fine-tune costs $15.56\",\n","      \"object\": \"fine-tune-event\"\n","    },\n","    {\n","      \"created_at\": 1650737180,\n","      \"level\": \"info\",\n","      \"message\": \"Fine-tune enqueued. Queue number: 0\",\n","      \"object\": \"fine-tune-event\"\n","    },\n","    {\n","      \"created_at\": 1650737185,\n","      \"level\": \"info\",\n","      \"message\": \"Fine-tune started\",\n","      \"object\": \"fine-tune-event\"\n","    },\n","    {\n","      \"created_at\": 1650738880,\n","      \"level\": \"info\",\n","      \"message\": \"Completed epoch 1/4\",\n","      \"object\": \"fine-tune-event\"\n","    },\n","    {\n","      \"created_at\": 1650739490,\n","      \"level\": \"info\",\n","      \"message\": \"Fine-tune cancelled\",\n","      \"object\": \"fine-tune-event\"\n","    }\n","  ],\n","  \"fine_tuned_model\": null,\n","  \"hyperparams\": {\n","    \"batch_size\": 16,\n","    \"learning_rate_multiplier\": 0.2,\n","    \"n_epochs\": 4,\n","    \"prompt_loss_weight\": 0.1\n","  },\n","  \"id\": \"ft-BltVwDo5jw3kagGETnz64IvK\",\n","  \"model\": \"babbage\",\n","  \"object\": \"fine-tune\",\n","  \"organization_id\": \"org-bgAXfs8WdU5942SLngg0OGpd\",\n","  \"result_files\": [],\n","  \"status\": \"cancelled\",\n","  \"training_files\": [\n","    {\n","      \"bytes\": 21920298,\n","      \"created_at\": 1650737164,\n","      \"filename\": \"fine_tune_multiwoz.jsonl\",\n","      \"id\": \"file-QqSc5PEg14bmukQqx5YBla29\",\n","      \"object\": \"file\",\n","      \"purpose\": \"fine-tune\",\n","      \"status\": \"processed\",\n","      \"status_details\": null\n","    }\n","  ],\n","  \"updated_at\": 1650739490,\n","  \"validation_files\": []\n","}\n"]}],"source":["!openai api fine_tunes.cancel -i ft-BltVwDo5jw3kagGETnz64IvK"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":224497,"status":"ok","timestamp":1650785047256,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"id":"RcJcZV6IoBxM","outputId":"b86885b3-e9ea-4c02-d07d-270597b0d7cd"},"outputs":[{"output_type":"stream","name":"stdout","text":["Logging requires wandb to be installed. Run `pip install wandb`.\n","[2022-04-24 05:33:29] Created fine-tune: ft-rLXdG9F7riQWgbMmaqYO7tVi\n","[2022-04-24 05:33:48] Fine-tune costs $15.49\n","[2022-04-24 05:33:48] Fine-tune enqueued. Queue number: 0\n","[2022-04-24 05:33:52] Fine-tune started\n","[2022-04-24 06:01:49] Completed epoch 1/4\n","[2022-04-24 06:29:06] Completed epoch 2/4\n","[2022-04-24 06:56:24] Completed epoch 3/4\n","[2022-04-24 07:23:36] Completed epoch 4/4\n","[2022-04-24 07:24:01] Uploaded model: babbage:ft-ccb-lab-members:ubar-2022-04-24-07-23-59\n","[2022-04-24 07:24:04] Uploaded result file: file-OZjsx8M3H2NUMw4pjqpeky4a\n","[2022-04-24 07:24:05] Fine-tune succeeded\n","\n","Job complete! Status: succeeded 🎉\n","Try out your fine-tuned model:\n","\n","openai api completions.create -m babbage:ft-ccb-lab-members:ubar-2022-04-24-07-23-59 -p <YOUR_PROMPT>\n"]}],"source":["!openai api fine_tunes.follow -i ft-rLXdG9F7riQWgbMmaqYO7tVi"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"NF63efdVgHEI"},"outputs":[],"source":["from pathlib import Path\n","from pprint import pprint\n","import string\n","import random\n","import spacy\n","from sacremoses import MosesTokenizer, MosesDetokenizer\n","from collections import deque, defaultdict\n","\n","\n","# generate_response_model = \"babbage:ft-ccb-lab-members:ubar-2022-04-24-07-23-59\"\n","generate_response_model = \"babbage:ft-ccb-lab-members:ubar-2022-04-24-00-56-30\" # better\n","\n","db = MultiWozDB(Path(root_dir) / 'MultiWOZ_2.1' / 'db') # for simulating database query\n","\n","\n","def fill_entities(resp, constraint, turn_domain):\n","    \"\"\"Fill in placeholders based on DB queries. This does not always make sense since\n","    it does not consider sentence structure and dialog context.\n","    `[value_reference]` is not stored in DB so it's randomly generated.\n","    \"\"\"\n","    constraint_dict = constraint_to_dict(constraint)\n","    entries = db.get_match_num(constraint_dict, return_entry=True)\n","    if len(constraint_dict) == 0:\n","        return resp.replace('[value_choice]', 'many')\n","    \n","    for domain in turn_domain:\n","        domain = domain[1:-1] if domain.startswith('[') else domain\n","        if domain == 'general':\n","            continue\n","\n","        entity_slots = set(ENTITY_SLOTS[domain])\n","        entities = entries.get(domain, list())\n","        random.shuffle(entities) # shuffle so system doesn't always recommend the same entity\n","        # pprint(entities)\n","\n","        vals_by_slot = defaultdict(list)\n","        for slot in entity_slots:\n","            seen = set()\n","            for entity in entities:\n","                if slot in entity and slot not in seen:\n","                    vals_by_slot[slot].append(entity[slot])\n","                    seen.add(entity[slot])\n","        \n","        vals_by_slot_iter = {slot: iter(vals) for slot, vals in vals_by_slot.items()}\n","        sents = []\n","        for sent in resp.split(' . '): # each [value_choice] should only cover inside sentence boundary\n","            fill_map = {}\n","            last_choice_match = None\n","            choice_constraints = None\n","            for slot_match in re.finditer(\"\\[value_([a-z]+)\\]\", sent):\n","                slot = slot_match.group(1)\n","                \n","                if slot == 'choice':\n","                    if last_choice_match:\n","                        choice = 0\n","                        for entity in entities:\n","                            if all(entity[slot] == val for slot, val in choice_constraints.items()):\n","                                choice += 1\n","                        fill_map[last_choice_match.span()] = str(choice) if choice < 10 else 'many'\n","                    choice_constraints = {}\n","                    last_choice_match = slot_match\n","                elif slot == 'reference':\n","                    rand_ref = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))\n","                    fill_map[slot_match.span()] = rand_ref\n","                elif slot in vals_by_slot:\n","                    try:\n","                        val = next(vals_by_slot_iter[slot])\n","                    except StopIteration:\n","                        # wrap around and reset generator\n","                        vals_by_slot_iter[slot] = iter(vals_by_slot[slot])\n","                        val = next(vals_by_slot_iter[slot])\n","                    fill_map[slot_match.span()] = val\n","                    if choice_constraints is not None and slot in ENTITY_SLOTS[domain]:\n","                    # if choice_constraints is not None and slot in INFORMABLE_SLOTS[domain]:\n","                        choice_constraints[slot] = val\n","                elif domain in constraint_dict and slot in constraint_dict[domain]:\n","                    fill_map[slot_match.span()] = constraint_dict[domain][slot]\n","            \n","            if last_choice_match:\n","                choice = 0\n","                for entity in entities:\n","                    if all(entity[slot] == val for slot, val in choice_constraints.items()):\n","                        choice += 1\n","                fill_map[last_choice_match.span()] = str(choice) if choice < 10 else 'many'\n","\n","            sent = list(sent)\n","            for span, val in fill_map.items():\n","                for i in range(*span):\n","                    sent[i] = ''\n","                sent[span[0]] = val\n","            sents.append(''.join(sent))\n","        resp = ' . '.join(sents)\n","    return resp\n","\n","\n","def predict_slots_batch(contexts, utterances):\n","    context_strs = ['###\\n'.join(context) for context in contexts]\n","\n","    curr_turns = [(\n","        f\"user: {utterance}\\n\"\n","        f\"state:\"\n","    ) for utterance in utterances]\n","\n","    prompts = [f\"{context}###\\n{curr_turn}\" if context != '' else curr_turn\n","               for context, curr_turn in zip(context_strs, curr_turns)]\n","\n","    response = openai.Completion.create(\n","        model=generate_response_model,\n","        prompt=prompts,\n","        temperature=0.7,\n","        max_tokens=128,\n","        top_p=1,\n","        frequency_penalty=0,\n","        presence_penalty=0,\n","        stop=[\"\\n\"]\n","    )\n","\n","    slots = [ret['text'].strip() for ret in response['choices']]\n","    return slots\n","\n","\n","def generate_response_batch(contexts, utterances, domains, slots=None, budget=1, fill_entity=False):\n","    \"\"\"Generate a list of responses given a list of contexts, user utterances, domains, and optionally slots.\n","    Batch improves inference speed. Note domains are 2D list b/c 1 turn may have more than 1 domains.\n","    \"\"\"\n","    if slots is None:\n","        slots = predict_slots_batch(contexts, utterances)\n","\n","    db_results = [constraint_to_db_result(s, d) for s, d in zip(slots, domains)] # query Database\n","    context_strs = ['###\\n'.join(context) for context in contexts]\n","\n","    curr_turns = [(\n","        f\"user: {utterance}\\n\"\n","        f\"state: {slots}\\n\"\n","        f\"match: {db_res}\\n\"\n","        f\"action:\"\n","    ) for utterance, slots, db_res in zip(utterances, slots, db_results)]\n","\n","    prompts = [f\"{context}###\\n{curr_turn}\" if context != '' else curr_turn\n","               for context, curr_turn in zip(context_strs, curr_turns)]\n","\n","    # Generate actions and responses\n","    response = openai.Completion.create(\n","        model=generate_response_model,\n","        prompt=prompts,\n","        temperature=0.7,\n","        max_tokens=200,\n","        top_p=1,\n","        frequency_penalty=0,\n","        presence_penalty=0,\n","        n=budget,\n","        stop=[\"\\n###\\n\"]\n","    )\n","\n","    act_resps = [ret['text'].strip() for ret in response['choices']]\n","    resps = []\n","    for i, act_resp in enumerate(act_resps):            \n","        try:\n","            resps.append(act_resp.split('response:')[1].strip())\n","        except:\n","            print('Invalid action & response:', act_resp)\n","            print(curr_turns[i // budget])\n","            print(contexts[i // budget])\n","            resps.append('')\n","\n","    new_curr_turns = []\n","    for i, curr_turn in enumerate(curr_turns):\n","        for j in range(budget):\n","            new_curr_turns.append(f\"{curr_turn} {act_resps[i * budget + j]}\\n\")\n","\n","    if fill_entity:\n","        slots = itertools.chain.from_iterable(itertools.repeat(s, budget) for s in slots)\n","        domains = itertools.chain.from_iterable(itertools.repeat(d, budget) for d in domains)\n","        resps = [fill_entities(resp, slots, domain)\n","                 for resp, slots, domain in zip(resps, slots, domains)]\n","\n","    return resps, new_curr_turns\n","\n","\n","def interactive_dialog(domain, context_window=10, fill_entity=True):\n","    \"\"\"Launch interactive dialog session, enter exit to exit\n","    Note `fill_entity` does not always make sense b/c filled responses are\n","    not fed into the model in the dialog context.\n","    \"\"\"\n","    nlp = spacy.load(\"en_core_web_sm\")\n","    mt, md = MosesTokenizer(lang='en'), MosesDetokenizer(lang='en')\n","    \n","    context = deque(maxlen=context_window)\n","    while True:\n","        utterance = input(f'user: ').lower()\n","        # make sure we tokenize utterance like training\n","        utterance = ' '.join(t.text for t in nlp(utterance))\n","        utterance = ' '.join(utterance.replace(\".\", \" . \").split())\n","        if utterance == 'exit':\n","            break\n","\n","        resps, curr_turns = generate_response_batch([context], [utterance], [[domain]],\n","                                                    fill_entity=fill_entity)\n","        context.append(curr_turns[0]) # update context\n","        # detokenize response\n","        detokenized_resp = md.detokenize(mt.tokenize(resps[0],\n","                                                     protected_patterns=['value_', ']-', ']s', ']ly']))\n","        print(f\"system: {detokenized_resp}\")\n","\n","    print(\"\\n====== context ======\\n\\n\" + '\\n'.join(context))\n"]},{"cell_type":"code","source":["# resp = \"we have [value_name]. [value_name] is good . Ref number is [value_reference]\"\n","# slots = \"[hotel] [value_name] acorn guest house \"\n","# turn_domain = ['[hotel]', '[restaurant]']\n","\n","# from pprint import pprint\n","# fill_entities(resp, slots, turn_domain)"],"metadata":{"id":"GfUBMO2VhBMb","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1651040510373,"user_tz":240,"elapsed":129,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"321999e5-6e98-495f-b534-9cf63a30bd1a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'we have acorn guest house. acorn guest house is good . Ref number is gx3vfw5o'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":254}]},{"cell_type":"markdown","source":["## Interactive Interface\n","Good for error analysis"],"metadata":{"id":"45YVXHvMkCXG"}},{"cell_type":"code","source":["interactive_dialog('restaurant')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1AhfbUw85DPQ","executionInfo":{"status":"ok","timestamp":1650843214966,"user_tz":240,"elapsed":72856,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"4ae748b0-0f40-45c3-bded-599a1f5926ca"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["user: can you recommend some place to eat\n","system: sure, what type of food are you looking for?\n","user: chinese\n","system: there are many chinese restaurants, what price range are you looking for?\n","user: moderate\n","system: there are 1 chinese restaurants in north. do you have a particular area in mind?\n","user: No, what is the name\n","system: shanghai family restaurant is a nice place.\n","user: Can you recommend another one\n","system: lan hong house is another option. can i give you their address?\n","user: sure book it please for 2 people\n","system: what day and time are you looking to book?\n","user: tuesday 2 pm\n","system: what day would you like it for?\n","user: I said tuesday\n","system: i was able to book that for you for 2 people on tuesday at 2pm. your reference number is: [value_reference]. is there anything else i can help you with?\n","user: exit\n","\n","====== context ======\n","\n","user: can you recommend some place to eat\n","state: \n","match: null\n","action: [restaurant] [request] food\n","response: sure , what type of food are you looking for ?\n","\n","user: chinese\n","state: [restaurant] [value_food] chinese\n","match: 3\n","action: [restaurant] [inform] choice food [request] pricerange\n","response: there are [value_choice] [value_food] restaurants , what price range are you looking for ?\n","\n","user: moderate\n","state: [restaurant] [value_food] chinese [value_pricerange] moderate\n","match: 3\n","action: [restaurant] [inform] choice food [request]\n","response: there are [value_choice] [value_food] restaurants in [value_area] . do you have a particular area in mind ?\n","\n","user: no , what is the name\n","state: [restaurant] [value_food] chinese [value_pricerange] moderate\n","match: 3\n","action: [restaurant] [recommend] name\n","response: [value_name] is a nice place .\n","\n","user: can you recommend another one\n","state: [restaurant] [value_food] chinese [value_pricerange] moderate\n","match: 3\n","action: [restaurant] [recommend] name [general] [reqmore]\n","response: [value_name] is another option . can i give you their address ?\n","\n","user: sure book it please for 2 people\n","state: [restaurant] [value_food] chinese [value_pricerange] moderate [value_name] ugly duckling [value_people] 2\n","match: 0\n","action: [restaurant] [request] day time\n","response: what day and time are you looking to book ?\n","\n","user: tuesday 2 pm\n","state: [restaurant] [value_food] chinese [value_pricerange] moderate [value_name] ugly duckling [value_people] 2 [value_day] tuesday [value_time] 2pm\n","match: 0\n","action: [restaurant] [request] day\n","response: what day would you like it for ?\n","\n","user: i said tuesday\n","state: [restaurant] [value_food] chinese [value_pricerange] moderate [value_name] ugly duckling [value_people] 2 [value_day] tuesday [value_time] 2pm\n","match: 0\n","action: [restaurant] [offerbooked] day people time reference [general] [reqmore]\n","response: i was able to book that for you for [value_people] people on [value_day] at [value_time] . your reference number is : [value_reference] . is there anything else i can help you with ?\n","\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":87664,"status":"ok","timestamp":1650767381793,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"id":"aiOkVyK-qqu5","outputId":"2fc27d38-55a9-4951-8307-2fb8da18b472"},"outputs":[{"output_type":"stream","name":"stdout","text":["user: I want a place to live for 2 days\n","system: sure. where are you looking for a guesthouse? do you have a price range in mind?\n","user: I'm looking for a hotel actually, not guesthouse. And i want to save money if possible\n","system: what part of town do you prefer?\n","user: anywhere is fine as long as it's cheap\n","system: the cambridge belfry is an cheap hotel in the west. would you like me to book you a room?\n","user: yes please for 1 person\n","system: what day will you be checking in?\n","user: monday\n","system: i will get the booking and then give you a call back\n","user: sounds good\n","system: okay i have booked that for you. your reservation number is [value_reference]. anything else i can help you with?\n","user: exit\n","\n","====== context ======\n","\n","user: i want a place to live for 2 days\n","state: [hotel] [value_stay] 2\n","match: 3\n","action: [hotel] [request] pricerange\n","response: sure . where are you looking for a [value_type] ? do you have a price range in mind ?\n","\n","user: i 'm looking for a hotel actually , not guesthouse . and i want to save money if possible\n","state: [hotel] [value_stay] 2 [value_pricerange] expensive [value_type] hotel\n","match: 3\n","action: [hotel] [request] area\n","response: what part of town do you prefer ?\n","\n","user: anywhere is fine as long as it 's cheap\n","state: [hotel] [value_stay] 2 [value_pricerange] cheap [value_type] hotel\n","match: 1\n","action: [hotel] [inform] area name pricerange type [offerbook]\n","response: [value_name] is an [value_pricerange] [value_type] in the [value_area] . would you like me to book you a room ?\n","\n","user: yes please for 1 person\n","state: [hotel] [value_stay] 2 [value_pricerange] cheap [value_type] hotel [value_name] cambridge belfry [value_people] 1\n","match: 1\n","action: [hotel] [request] day\n","response: what day will you be checking in ?\n","\n","user: monday\n","state: [hotel] [value_stay] 2 [value_pricerange] cheap [value_type] hotel [value_name] cambridge belfry [value_people] 1 [value_day] monday\n","match: 1\n","action: [hotel] [offerbook]\n","response: i will get the booking and then give you a call back\n","\n","user: sounds good\n","state: [hotel] [value_stay] 2 [value_pricerange] cheap [value_type] hotel [value_name] cambridge belfry [value_people] 1 [value_day] monday\n","match: 1\n","action: [hotel] [offerbooked] reference [general] [reqmore]\n","response: okay i have booked that for you . your reservation number is [value_reference] . anything else i can help you with ?\n","\n"]}],"source":["interactive_dialog('hotel')"]},{"cell_type":"code","source":["interactive_dialog('general')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"W3IObP3svKzJ","executionInfo":{"status":"ok","timestamp":1650437485496,"user_tz":240,"elapsed":36690,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"50ac7329-11fe-433e-c706-0f1ab9ade55e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["user: hello\n","system: happy to help! what can i help you find?\n","user: I want a place to eat\n","system: what kind of food are you looking for?\n","user: expensive food\n","system: i am sorry, i have no listings for that. would you like to try a different type of food?\n","user: no \n","system: is there anything else you would like to know?\n","user: no bye\n","system: goodbye.\n","user: exit\n","\n","====== context ======\n","\n","user: hello\n","constraint: \n","database: [db_null]\n","action: [general] [reqmore]\n","response: happy to help ! what can i help you find ?\n","\n","user: I want a place to eat\n","constraint: \n","database: [db_null]\n","action: [restaurant] [request] food\n","response: what kind of food are you looking for ?\n","\n","user: expensive food\n","constraint: [restaurant] [value_food] dontcare [value_pricerange] expensive\n","database: [db_null]\n","action: [restaurant] [nooffer] [request] food\n","response: i am sorry , i have no listings for that . would you like to try a different type of food ?\n","\n","user: no\n","constraint: [restaurant] [value_food] dontcare [value_pricerange] expensive\n","database: [db_null]\n","action: [general] [reqmore]\n","response: is there anything else you would like to know ?\n","\n","user: no bye\n","constraint: [restaurant] [value_food] dontcare [value_pricerange] expensive\n","database: [db_null]\n","action: [general] [bye]\n","response: goodbye .\n","\n"]}]},{"cell_type":"markdown","source":["# Neuro-Symbolic System 2"],"metadata":{"id":"Rptda9OhVkyM"}},{"cell_type":"markdown","source":["## User Action Parser"],"metadata":{"id":"KYxoKQJ8rf97"}},{"cell_type":"markdown","source":["We want to parse user actions given the current utterance and the last system response. Feeding in last constraint might also help because a user might be setting a slot that refers to a value in the last state, e.g. taxi `[value_departure]` and `[value_destination]` when user says \"I want a taxi from the restaurant to the hotel.\""],"metadata":{"id":"9Z_G4SdZtto3"}},{"cell_type":"code","source":["from pathlib import Path\n","from tqdm.auto import tqdm\n","import random\n","import json\n","\n","\n","data_dir = Path(root_dir) / 'processed_multiwoz'\n","\n","\n","def create_utterance_parser_ft_data(filename, max_samples=56745):\n","    fine_tuning_data = []\n","\n","    with open(Path(data_dir) / 'train_data.json') as inp:\n","        data = json.load(inp)\n","\n","    for dialog in tqdm(data.values()):\n","        last_resp = None\n","        for turn in dialog['log']:\n","            data = {}\n","            data['prompt'] = (\n","                f\"agent: {last_resp}\\n\" \n","                f\"user: {turn['user']}\\n\\n###\\n\\n\"\n","            )\n","            data['completion'] = f\" {turn['user_act']} END\"\n","\n","            fine_tuning_data.append(data)\n","            last_resp = turn['nodelx_resp']\n","\n","    print(\"# samples in dataset:\", len(fine_tuning_data))\n","    \n","    fine_tuning_data = random.sample(fine_tuning_data, k=max_samples)\n","    with open(filename, 'w') as out:\n","        for data in fine_tuning_data:\n","            out.write(json.dumps(data))\n","            out.write('\\n')"],"metadata":{"id":"jbUhHKJcrmiD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["jsonl_filename='fine_tune_user_parser.jsonl'\n","create_utterance_parser_ft_data(jsonl_filename, max_samples=56745)\n","\n","!head '{jsonl_filename}'\n","!wc -lw '{jsonl_filename}'"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":277,"referenced_widgets":["d4ad7ff0f55f40a3aae6480b2ea56ffe","f410e94af377487eb15dccc05e9dbd1f","f69d770baa0f4c798b13a980a5c9a7b3","16387c180e1846049308803627c3a9f1","ee1cc0adcf6b4fd7af79525004fffe9c","b509ba5220f74a0ebb5154c6c9efd7d0","ca535199c87e46b1a5afc8719a107288","9a4dece219b143e890775caa21f4d9d7","dd868393dc1448199decf276d16433b7","6469a8019eca4eeabc4fd4e069e8c6af","c7818afcb3f84d1b815eb2c1fd684b72"]},"id":"Hopl_kfetYId","executionInfo":{"status":"ok","timestamp":1651097629992,"user_tz":240,"elapsed":2214,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"73fe38d1-1636-41c3-fbe9-823454b1bb06"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/8433 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d4ad7ff0f55f40a3aae6480b2ea56ffe"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["# samples in dataset: 56745\n","{\"prompt\": \"agent: None\\nuser: i am looking for a place to stay , i would prefer a hotel .\\n\\n###\\n\\n\", \"completion\": \" [hotel] [inform] [value_type] hotel END\"}\n","{\"prompt\": \"agent: where will you be traveling to ?\\nuser: i will be going to broxbourne .\\n\\n###\\n\\n\", \"completion\": \" [train] [inform] [value_destination] broxbourne END\"}\n","{\"prompt\": \"agent: the place is a nightclub in the south .\\nuser: which nightclub is it ? and can you get me the information on it ?\\n\\n###\\n\\n\", \"completion\": \" [attraction] [inform] [value_name] the place END\"}\n","{\"prompt\": \"agent: there are two options for you . royal spice and da vinci pizzeria . do you have a preference ?\\nuser: let 's try the da vinci pizzeria . i 'll need the address and phone number too , please .\\n\\n###\\n\\n\", \"completion\": \" [restaurant] [inform] [value_name] da vinci pizzeria [request] [value_address] [value_phone] END\"}\n","{\"prompt\": \"agent: i have 2 choices . the lucky star serves chinese . and , nando 's is a portuguese restaurant . which do you prefer ?\\nuser: i was goona ask to search for international , but it sounds like their is nt an option with a cheap price . lets go with nando 's\\n\\n###\\n\\n\", \"completion\": \" [restaurant] [inform] [value_food] international [value_name] nando's [value_pricerange] cheap END\"}\n","{\"prompt\": \"agent: None\\nuser: i need a taxi from kings lynn station at 05:15 , going to hk fusion , please .\\n\\n###\\n\\n\", \"completion\": \" [taxi] [inform] [value_departure] kings lynn station [value_destination] hk fusion [value_leave] 05:15 END\"}\n","{\"prompt\": \"agent: the number is 01223315702 . is there anything else that i can help you with ?\\nuser: no , you have been very helpful , thanks . i do n't need anything else at the moment . bye .\\n\\n###\\n\\n\", \"completion\": \" [general] [bye] END\"}\n","{\"prompt\": \"agent: 08712240240 . do you need anything else ?\\nuser: can i have a taxi , please ? i want to leave vue cinema at 14:45 .\\n\\n###\\n\\n\", \"completion\": \" [taxi] [inform] [value_departure] vue cinema [value_leave] 14:45 END\"}\n","{\"prompt\": \"agent: that was quick , are you sure you do not need any other assistance ?\\nuser: i need a place to stay . i would like a hotel with free wifi .\\n\\n###\\n\\n\", \"completion\": \" [hotel] [inform] [value_internet] yes [value_type] hotel [value_area] centre END\"}\n","{\"prompt\": \"agent: ok . i will book the tr6009 for you . how many tickets should i book ?\\nuser: just one please .\\n\\n###\\n\\n\", \"completion\": \" [train] [inform] [value_people] 1 END\"}\n","   56745  2252377 fine_tune_user_parser.jsonl\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":790720,"status":"ok","timestamp":1651098433642,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"outputId":"59b735cb-49ea-47e7-d63c-c723bca15713","id":"hUuJc78g23LE"},"outputs":[{"output_type":"stream","name":"stdout","text":["Logging requires wandb to be installed. Run `pip install wandb`.\n","Upload progress: 100% 14.4M/14.4M [00:00<00:00, 24.0Git/s]\n","Uploaded file from fine_tune_user_parser.jsonl: file-ffwh3Z42AzTJ7tT4JOhKnRuH\n","Created fine-tune: ft-ZgqUkOHaZmXzhfTcemDnT7ma\n","Streaming events until fine-tuning is complete...\n","\n","(Ctrl-C will interrupt the stream, but not cancel the fine-tune)\n","[2022-04-27 22:14:11] Created fine-tune: ft-ZgqUkOHaZmXzhfTcemDnT7ma\n","[2022-04-27 22:14:38] Fine-tune costs $5.59\n","[2022-04-27 22:14:38] Fine-tune enqueued. Queue number: 0\n","[2022-04-27 22:17:12] Fine-tune started\n","\n","Stream interrupted (client disconnected).\n","To resume the stream, run:\n","\n","  openai api fine_tunes.follow -i ft-ZgqUkOHaZmXzhfTcemDnT7ma\n","\n"]}],"source":["!openai api fine_tunes.create -t '{jsonl_filename}' -m ada --n_epochs 4 --suffix \"user parser\""]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2225,"status":"ok","timestamp":1651099873295,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"outputId":"036ce936-92ab-44ff-e540-a90c1c18a3a5","id":"bnvOxlSO23LF"},"outputs":[{"output_type":"stream","name":"stdout","text":["Logging requires wandb to be installed. Run `pip install wandb`.\n","[2022-04-27 22:14:11] Created fine-tune: ft-ZgqUkOHaZmXzhfTcemDnT7ma\n","[2022-04-27 22:14:38] Fine-tune costs $5.59\n","[2022-04-27 22:14:38] Fine-tune enqueued. Queue number: 0\n","[2022-04-27 22:17:12] Fine-tune started\n","[2022-04-27 22:24:37] Completed epoch 1/4\n","[2022-04-27 22:31:31] Completed epoch 2/4\n","[2022-04-27 22:38:21] Completed epoch 3/4\n","[2022-04-27 22:45:11] Completed epoch 4/4\n","[2022-04-27 22:45:29] Uploaded model: ada:ft-ccb-lab-members:user-parser-2022-04-27-22-45-28\n","[2022-04-27 22:45:32] Uploaded result file: file-rm9KvRGTZtTVhaTsl1bo3Bz1\n","[2022-04-27 22:45:33] Fine-tune succeeded\n","\n","Job complete! Status: succeeded 🎉\n","Try out your fine-tuned model:\n","\n","openai api completions.create -m ada:ft-ccb-lab-members:user-parser-2022-04-27-22-45-28 -p <YOUR_PROMPT>\n"]}],"source":["!openai api fine_tunes.follow -i ft-ZgqUkOHaZmXzhfTcemDnT7ma"]},{"cell_type":"code","source":["from itertools import chain\n","import re\n","\n","\n","parse_user_acts_model = \"ada:ft-ccb-lab-members:user-parser-2022-04-27-22-45-28\" # better\n","# parse_user_acts_model = \"ada:ft-ccb-lab-members:user-parser-2022-04-25-07-24-01\"\n","\n","\n","def nodelx_act_to_dict(nodelx_act, user=True):\n","    \"\"\"Convert nodelexed action with action, slot, value triplets to dict.\n","    This function can also be used for system act if it is not delexed.\n","    If a slot have no value, `?` is used as the value.\n","    \"\"\"\n","    allowed_acts = set(USER_ACTIONS if user else chain.from_iterable(SYSTEM_ACTIONS.values()))\n","\n","    act_dict = {}\n","    domain, action, slot = None, None, None\n","    last_match = None\n","    # Iterate over special tokens\n","    for match in re.finditer(r\"\\[(\\w+)\\]\", nodelx_act):\n","        if domain and action and slot and last_match and last_match.group(1) == slot:\n","            slot = last_match.group(1)[6:] # [value_name] -> name\n","            value = nodelx_act[last_match.end():match.start()].strip()\n","            value = '?' if value == '' else value\n","            act_dict[domain][action][slot] = value\n","\n","        if re.match(r\"\\[value_[a-z]+\\]\", match.group(0)):\n","            if action:\n","                slot = match.group(1)\n","        elif match.group(1) in allowed_acts:\n","            if domain:\n","                action = match.group(1)\n","                act_dict[domain][action] = {}\n","        elif match.group(1) in ALL_DOMAINS + ['general']:\n","            domain = match.group(1)\n","            act_dict[domain] = {}\n","        else:\n","            role = 'user' if user else 'system'\n","            print(f'Invalid {role} act token:', match.group(1))\n","            print('Action string:', nodelx_act)\n","\n","        last_match = match\n","\n","    if domain and action and slot and last_match and last_match.group(1) == slot:\n","        slot = last_match.group(1)[6:] # [value_name] -> name\n","        value = nodelx_act[last_match.end():].strip()\n","        value = '?' if value == '' else value\n","        act_dict[domain][action][slot] = value\n","\n","    return act_dict\n","\n","\n","def parse_user_acts_batch(last_resps, utterances):\n","    prompts = [(\n","        f\"agent: {last_resp}\\n\"\n","        f\"user: {utterance}\\n\\n###\\n\\n\"\n","    ) for last_resp, utterance in zip(last_resps, utterances)]\n","\n","    response = openai.Completion.create(\n","        model=parse_user_acts_model,\n","        prompt=prompts,\n","        temperature=1.0,\n","        max_tokens=64,\n","        top_p=0.2,\n","        frequency_penalty=0,\n","        presence_penalty=0,\n","        stop=[\" END\"]\n","    )\n","    # print(response['choices'][0]['text'].strip())\n","\n","    actions = [nodelx_act_to_dict(ret['text'].strip(), user=True)\n","               for ret in response['choices']]\n","    return actions"],"metadata":{"id":"iby1EKdo5Aat"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["nodelx_act_to_dict('[hotel] [inform] [value_address] [request] [value_name]')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VDquGxHQYH4D","executionInfo":{"status":"ok","timestamp":1651342029404,"user_tz":240,"elapsed":214,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"fd4c50c3-02a5-45e4-f862-c572adc1c8cc"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'hotel': {'inform': {'address': '?'}, 'request': {'name': '?'}}}"]},"metadata":{},"execution_count":11}]},{"cell_type":"code","source":["# last_slots = \"[hotel] [value_area] south [value_stars] 4 [value_parking] yes [value_pricerange] cheap [value_name] rosa's [value_type] guesthouse [value_day] wednesday [value_people] 2 [value_stay] 2 [restaurant] [value_name] maharajah tandoori [value_day] wednesday [value_people] 2 [value_time] 11:45\"\n","# utterance = \"i need a taxi to the restaurant .\"\n","\n","# last_slots = \"[hotel] [value_area] south [value_stars] 4 [value_parking] yes [value_pricerange] cheap [value_name] rosa's [value_type] guesthouse [value_day] wednesday [value_people] 2 [value_stay] 2\"\n","# utterance = \"I want a chinese place to dine.\"\n","\n","# last_slots = None\n","# utterance = \"sure. just a moment.\"\n","\n","# last_resp = \"there are 9 that fit your request . may i suggest the acorn guest house ? it 's a nice place to stay .\"\n","# utterance = \"that 's a guesthouse right ? could you give me it 's address and star rating ?\"\n","\n","# last_resp = None\n","# utterance = \"i am looking for a guesthouse to stay in with free wifi .\"\n","\n","# last_resp = \"we have 23 guesthouses in cambridge that include internet . would you like to narrow it down by area , rating , or price range at all ?\"\n","# utterance = \"yes , i would like a moderately priced place in the north .\"\n","\n","# last_resp = \"7 places still have that , can i suggest the acorn guest house for you ?\"\n","# utterance = \"can i get the postcode of that place ?\"\n","\n","# last_resp = \"i have two pools in the north area of town : jesus green outdoor pool and kings hedges learner pool . which do you prefer ?\"\n","# utterance = \"kings hedges sounds nice . can i get the address please ?\"\n","\n","# last_resp = \"i have 5 expensive places to stay . how about one in the town centre ?\"\n","# utterance = \"sure , it should have free parking and have 4 stars .\"\n","\n","# User rejection not considered\n","# last_resp = \"how about acorn guest house ?\"\n","# utterance = \"no I hate that place . \"\n","\n","# Not enough domain info\n","# last_resp = \"can i go ahead and book this for you ?\"\n","# utterance = \"no , i just need their address .\"\n","\n","# Complex multi-domain parsing not working well, potential future work: parse domains first then generate actions for each domain\n","last_resp = \"i have the archway house , it is a guesthouse in the moderate price range and has free parking .\"\n","utterance = \"what is the full phone number and area code of the archway house ? also , is there a persian restaurant near the centre of the location ?\"\n","\n","# last_resp = \"the phone number is 01223575314 . what price range would you like the persian restaurant to be ?\"\n","# utterance = \"also expensive . may i have the post code and area of the archway house as well ?\"\n","\n","parse_user_acts_batch([last_resp], [utterance])[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ksO2QZyDffgF","executionInfo":{"status":"ok","timestamp":1651108167040,"user_tz":240,"elapsed":513,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"3c626abe-d43e-4e19-e053-b675c475c085"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'hotel': {'inform': {'name': 'archway house'},\n","  'request': {'area': '?', 'persian': '?', 'phone': '?'}}}"]},"metadata":{},"execution_count":178}]},{"cell_type":"markdown","source":["## System Action Parser"],"metadata":{"id":"XqZd4pZhdBHJ"}},{"cell_type":"code","source":["from pathlib import Path\n","from tqdm.auto import tqdm\n","import random\n","import json\n","\n","\n","data_dir = Path(root_dir) / 'processed_multiwoz'\n","\n","\n","def create_response_parser_ft_data(filename, max_samples=56745):\n","    fine_tuning_data = []\n","\n","    with open(Path(data_dir) / 'train_data.json') as inp:\n","        data = json.load(inp)\n","\n","    for dialog in tqdm(data.values()):\n","        for turn in dialog['log']:\n","            data = {}\n","            data['prompt'] = (\n","                f\"user: {turn['user']}\\n\" \n","                f\"agent: {turn['resp']}\\n\\n###\\n\\n\"\n","            )\n","            data['completion'] = f\" {turn['sys_act']} END\"\n","\n","            fine_tuning_data.append(data)\n","\n","    print(\"# samples in dataset:\", len(fine_tuning_data))\n","    \n","    fine_tuning_data = random.sample(fine_tuning_data, k=max_samples)\n","    with open(filename, 'w') as out:\n","        for data in fine_tuning_data:\n","            out.write(json.dumps(data))\n","            out.write('\\n')"],"metadata":{"id":"57UOyE_2Fq8P"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["jsonl_filename='fine_tune_agent_parser.jsonl'\n","create_response_parser_ft_data(jsonl_filename, max_samples=56745)\n","\n","!head '{jsonl_filename}'\n","!wc -lw '{jsonl_filename}'"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":277,"referenced_widgets":["507977436fff4bc9910e498ce27a206e","7e2bc8b499c64f67befcf2627f174c72","caf50d5f77234e8d8bde0bd64bcd61e1","1d21915c594e4902a507070a4dc51a64","53ea1b21670142e29a3ca4711ca54374","b2f399360ee7470591588bdbc1104234","0e557c0698164754929b96d41796e71f","da77fd8192ea4646af72bc5eb3ae3b31","d0216e0292c4497cb4118c2ce194df2f","038a1d215b7944409fb4bf8ca7270a87","58d10b029416452b9aee009a1428e5a6"]},"id":"55XThCbeK9cP","executionInfo":{"status":"ok","timestamp":1651100025059,"user_tz":240,"elapsed":2293,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"ff0bbe48-0b23-4cf6-8080-2b43a5cff768"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/8433 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"507977436fff4bc9910e498ce27a206e"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["# samples in dataset: 56745\n","{\"prompt\": \"user: i like you no problem .\\nagent: thanks for using our service today !\\n\\n###\\n\\n\", \"completion\": \" [general] [bye] END\"}\n","{\"prompt\": \"user: that does not matter but i want it to have 4 stars .\\nagent: i would recommend the [value_name] , would you like me to book you ?\\n\\n###\\n\\n\", \"completion\": \" [hotel] [recommend] name [offerbook] END\"}\n","{\"prompt\": \"user: thank you . i also want a hotel to stay which should a have a star of 4\\nagent: what area would you like to stay in ?\\n\\n###\\n\\n\", \"completion\": \" [hotel] [request] area END\"}\n","{\"prompt\": \"user: i actually do nt need a reservation . just the address , postcode and phone number please .\\nagent: they are located [value_address] , [value_postcode] . phone : [value_phone] .\\n\\n###\\n\\n\", \"completion\": \" [restaurant] [inform] address phone postcode END\"}\n","{\"prompt\": \"user: thank you for your help .\\nagent: is there anything else i can help you with today ?\\n\\n###\\n\\n\", \"completion\": \" [general] [reqmore] END\"}\n","{\"prompt\": \"user: hello , i am looking for some entertainment .\\nagent: we have got [value_choice] attractions [value_area] , is there a particular area you were looking at ?\\n\\n###\\n\\n\", \"completion\": \" [attraction] [inform] choice area [request] area END\"}\n","{\"prompt\": \"user: thank you very much . have a nice day !\\nagent: you 're welcome ! have a great day !\\n\\n###\\n\\n\", \"completion\": \" [general] [welcome] [bye] END\"}\n","{\"prompt\": \"user: grand . can i also get your help with train ? i need to get to stansted airport by 16:45 on wednesday .\\nagent: and where are you leaving from ?\\n\\n###\\n\\n\", \"completion\": \" [train] [request] departure END\"}\n","{\"prompt\": \"user: i would like that to be located in the north .\\nagent: what is your preferred price range ?\\n\\n###\\n\\n\", \"completion\": \" [hotel] [request] pricerange END\"}\n","{\"prompt\": \"user: thank you . can yo also find a restaurant that serves traditional food and in the expensive price range ?\\nagent: unfortunately , we do n't have any restaurants [value_area] that offer [value_food] food . is there another type of cuisine i can search for ?\\n\\n###\\n\\n\", \"completion\": \" [restaurant] [nooffer] food [request] food END\"}\n","   56745  2226816 fine_tune_agent_parser.jsonl\n"]}]},{"cell_type":"code","source":["!openai api fine_tunes.create -t '{jsonl_filename}' -m ada --n_epochs 4 --suffix \"agent parser\""],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JRuEQX3qLgMq","executionInfo":{"status":"ok","timestamp":1651100776160,"user_tz":240,"elapsed":627953,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"33966209-4e4b-4c54-b6ec-fb8bb3f3ec66"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Logging requires wandb to be installed. Run `pip install wandb`.\n","Upload progress: 100% 14.3M/14.3M [00:00<00:00, 23.7Git/s]\n","Uploaded file from fine_tune_agent_parser.jsonl: file-NLPYLR6O50PaalmxrPghhy8l\n","Created fine-tune: ft-sUTWaRhNBngIhADEqVZgxjzi\n","Streaming events until fine-tuning is complete...\n","\n","(Ctrl-C will interrupt the stream, but not cancel the fine-tune)\n","[2022-04-27 22:55:53] Created fine-tune: ft-sUTWaRhNBngIhADEqVZgxjzi\n","[2022-04-27 22:56:14] Fine-tune costs $5.46\n","[2022-04-27 22:56:14] Fine-tune enqueued. Queue number: 0\n","\n","Stream interrupted (client disconnected).\n","To resume the stream, run:\n","\n","  openai api fine_tunes.follow -i ft-sUTWaRhNBngIhADEqVZgxjzi\n","\n"]}]},{"cell_type":"code","source":["!openai api fine_tunes.follow -i ft-sUTWaRhNBngIhADEqVZgxjzi"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"j4XG4gHXSzYG","executionInfo":{"status":"ok","timestamp":1651102283578,"user_tz":240,"elapsed":16646,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"44fb09ad-3a5d-4054-9bfc-64fa25bdeb68"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Logging requires wandb to be installed. Run `pip install wandb`.\n","[2022-04-27 22:55:53] Created fine-tune: ft-sUTWaRhNBngIhADEqVZgxjzi\n","[2022-04-27 22:56:14] Fine-tune costs $5.46\n","[2022-04-27 22:56:14] Fine-tune enqueued. Queue number: 0\n","[2022-04-27 23:04:02] Fine-tune started\n","[2022-04-27 23:11:47] Completed epoch 1/4\n","[2022-04-27 23:18:12] Completed epoch 2/4\n","[2022-04-27 23:24:34] Completed epoch 3/4\n","[2022-04-27 23:30:58] Completed epoch 4/4\n","[2022-04-27 23:31:17] Uploaded model: ada:ft-ccb-lab-members:agent-parser-2022-04-27-23-31-15\n","[2022-04-27 23:31:20] Uploaded result file: file-zgeMWw95n5P6ejaEDBhQ1Q6Y\n","[2022-04-27 23:31:21] Fine-tune succeeded\n","\n","Job complete! Status: succeeded 🎉\n","Try out your fine-tuned model:\n","\n","openai api completions.create -m ada:ft-ccb-lab-members:agent-parser-2022-04-27-23-31-15 -p <YOUR_PROMPT>\n"]}]},{"cell_type":"code","source":["import re\n","\n","\n","parse_system_acts_model = \"ada:ft-ccb-lab-members:agent-parser-2022-04-27-23-31-15\" # better\n","# parse_system_acts_model = \"ada:ft-ccb-lab-members:agent-parser-2022-04-26-05-08-50\"\n","\n","\n","def delexed_act_to_dict(delexed_act, user=False):\n","    \"\"\"Convert delexed action with action, slot pairs to dict.\n","    \"\"\"\n","    allowed_acts = set(USER_ACTIONS if user else chain.from_iterable(SYSTEM_ACTIONS.values()))\n","    \n","    act_dict = {}\n","    domain = None\n","    action = None\n","    last_match = None\n","    # Iterate over special tokens\n","    for match in re.finditer(r\"\\[(\\w+)\\]\", delexed_act):\n","        if domain and action and last_match and last_match.group(1) == action:\n","            slots = delexed_act[last_match.end():match.start()].split()\n","            act_dict[domain][action].update(slots)\n","\n","        if match.group(1) in allowed_acts:\n","            if domain:\n","                action = match.group(1)\n","                act_dict[domain][action] = set()\n","        elif match.group(1) in ALL_DOMAINS + ['general']:\n","            domain = match.group(1)\n","            act_dict[domain] = {}\n","        else:\n","            role = 'user' if user else 'system'\n","            print(f'Invalid {role} act token:', match.group(1))\n","            print('Action string:', delexed_act)\n","\n","        last_match = match\n","\n","    if domain and action and last_match and last_match.group(1) == action:\n","        slots = delexed_act[last_match.end():].split()\n","        act_dict[domain][action].update(slots)\n","\n","    return act_dict\n","\n","\n","def parse_system_acts_batch(user_queries, responses):\n","    prompts = [(\n","        f\"user: {user_query}\\n\"\n","        f\"agent: {response}\\n\\n###\\n\\n\"\n","    ) for user_query, response in zip(user_queries, responses)]\n","\n","    response = openai.Completion.create(\n","        model=parse_system_acts_model,\n","        prompt=prompts,\n","        temperature=1.0,\n","        max_tokens=64,\n","        top_p=0.2,\n","        frequency_penalty=0,\n","        presence_penalty=0,\n","        stop=[\" END\"]\n","    )\n","    # print(response['choices'][0]['text'].strip())\n","\n","    actions = [delexed_act_to_dict(ret['text'].strip(), user=False)\n","               for ret in response['choices']]\n","    \n","    return actions"],"metadata":{"id":"niNyLvkadCoJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["utterance = \"i want to book a table for 4 people at 15:00 on wednesday . please give me a reference number .\"\n","response = \"sure thing ! your reference number is [value_reference] . can i help you with anything else today ?\"\n","\n","# Is domain prediction reliable when there is no info?\n","# utterance = \"that 's perfect , thanks !\"\n","# response = \"can i go ahead and book this for you ?\"\n","\n","# utterance = \"hello , i want a nice place to dine .\"\n","# response = \"in what area ?\"\n","\n","parse_system_acts_batch([utterance], [response])[0] "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NKRMvu_OejNf","executionInfo":{"status":"ok","timestamp":1651342164914,"user_tz":240,"elapsed":1601,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"96861c95-8518-4e89-f688-364768aae9d9"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'general': {'reqmore': set()}, 'restaurant': {'offerbooked': {'reference'}}}"]},"metadata":{},"execution_count":14}]},{"cell_type":"markdown","source":["## Schema for Belief State\n","IMPORTANT: When adding a rule, you must make sure it improves scores on the dev set in the A/B test eval section below"],"metadata":{"id":"mTB26vVB8EQg"}},{"cell_type":"code","source":["from pathlib import Path\n","from collections import defaultdict\n","from itertools import chain\n","import json\n","\n","\n","db = MultiWozDB(Path(root_dir) / 'MultiWOZ_2.1' / 'db')\n","\n","\n","class ValidationError(Exception):\n","    pass\n","\n","class InputError(Exception):\n","    '''When user's input is out of range. Should let user retype another utterance'''\n","    pass\n","\n","\n","class BeliefState:\n","    SCHEMA_DOMAINS = ['restaurant', 'hotel']\n","    SYS_INFO_ACTS = {'inform', 'recommend', 'offerbook', 'offerbooked'}\n","    SYS_REQ_ACTS = {'request', 'select'}\n","\n","    def __init__(self):\n","        # user informed slot, value pair per domain\n","        self.info_state = defaultdict(dict)\n","        # user requested slots per domain\n","        self.req_state = defaultdict(set)\n","        self.active_domains = ['general']\n","        self.failed_bookings = set()\n","        self.user_act = None\n","\n","    def update_user(self, user_act):\n","        user_act = defaultdict(dict, user_act)\n","        self.user_act = user_act\n","        # Update active domains\n","        turn_domains = [domain for domain in user_act\n","                        if domain in self.SCHEMA_DOMAINS + ['general']]\n","        # print(\"turn_domains\", turn_domains)\n","        # print(\"active_domains\", self.active_domains)\n","        # print(\"user_act\", user_act)\n","        if len(turn_domains) == 2 and 'general' in turn_domains:\n","            turn_domains.remove('general')\n","        if (len(turn_domains) == 2 and len(self.active_domains) == 1\n","            # keep current domain the last one if there are multiple active\n","            and self.active_domains[0] == turn_domains[1]):\n","            turn_domains = turn_domains[::-1]\n","        if len(turn_domains) > 0:\n","            self.active_domains = turn_domains\n","\n","        # Update info_state and req_state\n","        for domain in turn_domains:\n","            # print(\"domain\", domain)\n","            if 'inform' in user_act[domain]:\n","                for slot, val in user_act[domain]['inform'].items():\n","                    if slot in INFORMABLE_SLOTS[domain]:\n","                        if val == '?':\n","                            print(f'[{domain}] user informed slot has no value:', slot)\n","                            print(user_act)\n","                            continue\n","                        if (slot in CATEGORICAL_SLOTS[domain]\n","                            and val not in CATEGORICAL_SLOTS[domain][slot] + ['unknown', 'dontcare']): \n","                            print(f'[{domain}] user informed categorical slot value out of range: {slot}={val}')\n","                            print(user_act)\n","                            continue\n","                        else:\n","                            self.info_state[domain][slot] = val\n","                    else:\n","                        print(f'[{domain}] user informed slot out of range:', slot)\n","                        print(user_act)\n","            if 'request' in user_act[domain]:\n","                for slot in user_act[domain]['request']:\n","                    if slot in REQUESTABLE_SLOTS[domain]:\n","                        self.req_state[domain].add(slot)\n","                    else:\n","                        print(f'[{domain}] user requested slot out of range:', slot)\n","                        print(user_act)\n","        \n","        # print(\"self.info_state\", self.info_state)\n","        # print(\"self.req_state\", self.req_state)\n","\n","    def update_system(self, system_act, force=False):\n","        system_act = defaultdict(dict, system_act)\n","        valid, reason = self.check_validity(system_act)\n","\n","        # Remove system informed slots from req_state\n","        # Uninformed ones can be informed in the next turn\n","        for domain in self.active_domains:\n","            for act, slots in system_act[domain].items():\n","                if act in self.SYS_INFO_ACTS:\n","                    self.req_state[domain].difference_update(slots)\n","        \n","        if not valid and not force:\n","            raise ValidationError(reason)\n","        \n","    def check_validity(self, system_act):\n","        # Query # DB match based on informed slots\n","        curr_domain = self.active_domains[-1]\n","        # curr_domain = curr_domain[1:-1] if curr_domain.startswith(\"[\") else curr_domain\n","        db_match = db.get_match_num(self.info_state)[curr_domain]\n","        \n","        # print(\"system_act\", system_act)\n","        # print(\"self.info_state\", self.info_state)\n","        # print(\"self.req_state\", self.req_state)\n","        # print(\"db_match\", db_match)\n","\n","        # System switched domain from user domain, but it's okay if one is general domain\n","        if (curr_domain not in system_act\n","            and 'general' not in system_act\n","            and curr_domain != 'general'):\n","            return False, 'System switched domain'  \n","\n","        for domain in self.active_domains:\n","            if domain == 'general':\n","                if 'bye' in system_act[domain]:\n","                    if not 'bye' in self.user_act[domain]:\n","                      return False, 'The user need to end the conversation first'\n","                if 'greet' in system_act[domain]:\n","                    pass  # TODO\n","                if 'reqmore' in system_act[domain]:\n","                    pass  # TODO\n","                if 'thank' in system_act[domain]:\n","                    pass  # TODO\n","                if 'welcome' in system_act[domain]:\n","                    if not 'thank' in self.user_act[domain]:\n","                      return False, 'The user did not express gratitude'\n","            else:\n","                # Retrieve all slots informed by system\n","                sys_info_slots = set(chain.from_iterable(slots for act, slots in system_act[domain].items()\n","                                     if act in self.SYS_INFO_ACTS))\n","\n","                # slot is requested by user but not informed by system\n","                if not self.req_state[domain].issubset(sys_info_slots):\n","                    return False, 'Did not inform user with a slot user requested'\n","                elif self.req_state[domain] < sys_info_slots:\n","                    # system informed something that user doesn't ask. do we need to do anything?\n","                    pass\n","\n","                # Retrieve all slots requested by system\n","                sys_req_slots = set(chain.from_iterable(slots for act, slots in system_act[domain].items()\n","                                    if act in self.SYS_REQ_ACTS))\n","\n","                for slot, val in self.info_state[domain].items():\n","                    if val == 'unknown':\n","                        if slot not in sys_req_slots:\n","                            return False, 'Did not request an unknown slot from user'\n","                    else:\n","                        if slot in sys_req_slots and db_match > 0:\n","                            return False, 'Requesting a slot that is already filled'\n","                        # if (slot == 'name' and val != 'dontcare'\n","                        #     and any(s in sys_req_slots for s in ENTITY_SLOTS[domain])):\n","                        #     return False, 'Requesting an entity slot that is entailed'\n","                \n","                # TODO: implement mechanism to relax constraint when there is no match or availability\n","                # TODO: also see if narrowing down entries by requesting important slots helps.\n","                # Some actions to consider: select, request, recommend\n","\n","                # TODO: offerbooked when there is no booking availability\n","                # TODO: nobook when there is booking availability\n","\n","                if 'recommend' in system_act[domain]:\n","                    pass # TODO\n","                if 'select' in system_act[domain]:\n","                    pass\n","                # Offerbook if there is no database match\n","                if 'offerbook' in system_act[domain]:\n","                    # db_match is empty string when domain not in info_state\n","                    if db_match == '' or db_match == 0:  \n","                        return False, 'Offerbook but there is no database match'\n","                    if 'nooffer' in system_act[domain]:  \n","                        return False, 'Offerbook and nooffer at the same time'\n","                # Nooffer when there are database matches\n","                if 'nooffer' in system_act[domain]:\n","                    # db_match is empty string when domain not in info_state or domain is general\n","                    if db_match == '' or db_match > 0:  \n","                        return False, 'Nooffer when there are database matches'\n","                    if 'offerbook' in system_act[domain]:  \n","                        return False, 'Offerbook and nooffer at the same time'\n","                # Offerbook if there is no database match\n","                if 'offerbooked' in system_act[domain]:\n","                    # db_match is empty string when domain not in info_state\n","                    if db_match == '' or db_match == 0:  \n","                        return False, 'Offerbook but there is no database match'\n","                    if 'nobook' in system_act[domain]:  \n","                        return False, 'Offerbooked and nobook at the same time'\n","                if 'nobook' in system_act[domain]:\n","                    if 'offerbooked' in system_act[domain]:  \n","                        return False, 'Offerbooked and nobook at the same time'\n","                \n","                # Consistent so far\n","                # Reject if this booking failed before but is available now\n","                #print(self.serialize(domain = domain))\n","                if 'offerbook' in system_act[domain] or 'offerbooked' in system_act[domain]:\n","                  if self.serialize(domain = domain) in self.failed_bookings:\n","                    return False, 'This booking has failed previously.'\n","                # Record this booking if no offer is made\n","                if 'nooffer' in system_act[domain]:\n","                  self.failed_bookings.add(self.serialize(domain = domain))\n","        return True, 'success'\n","\n","    def serialize(self, domain = None):\n","        \"\"\"Convert informed slots to string\n","        \"\"\"\n","        serialized = []\n","        info_state = self.info_state[domain] if not domain==None else self.info_state\n","        for domain, state in info_state.items():\n","            if len(state) > 0:\n","                serialized.append(f'[{domain}]')\n","                if (type(state) is str):\n","                  serialized.append(state)\n","                else:\n","                  for slot, val in state.items():\n","                      serialized.append(f'[value_{slot}] {val}')\n","        return ' '.join(serialized)\n"],"metadata":{"id":"hWY2YQDThCsu"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Unit Tests"],"metadata":{"id":"gdmfPlktAiWA"}},{"cell_type":"code","source":["def unit_test_schema(user_acts, system_act):\n","    \"\"\"Unit test belief state schema\n","    \"\"\"\n","    belief_state = BeliefState()\n","    print(\"--------------------------------------\")\n","    print(\"user_acts\", user_acts)\n","    belief_state.update_user(user_acts)\n","    print(\"belief_state\", belief_state.serialize())\n","\n","    print(\"system_acts\", system_act)\n","    try:\n","        belief_state.update_system(system_act)\n","        return \"no error\"\n","    except Exception as e:\n","        return str(e)\n","\n","# valid\n","user_acts1 = {'restaurant': {'inform': {'food': 'chinese'}}}\n","system_act1 = {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'pricerange': '?'}}}\n","expected = 'no error'\n","result = unit_test_schema(user_acts1, system_act1)\n","print('test1', 'passed' if result == expected else 'failed: ' + result) \n","\n","# user requested name but system doesn't inform name\n","user_acts2 = {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'name': '?'}}}\n","system_act2 = {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'pricerange': '?'}}}\n","expected = 'Did not inform user with a slot user requested'\n","result = unit_test_schema(user_acts2, system_act2)\n","print('test2', 'passed' if result == expected else 'failed: ' + result) \n","\n","# user requested name but system doesn't inform name\n","user_acts3 = {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'name': '?'}}}\n","system_act3 = {'restaurant': {'inform': {'food': 'chinese', 'name': 'hakka'}, 'request': {'pricerange': '?'}}}\n","expected = 'no error'\n","result = unit_test_schema(user_acts3, system_act3)\n","print('test3', 'passed' if result == expected else 'failed: ' + result) \n","\n","# offerbook when there is no db match for user act\n","# Expected: err - Offerbook but there is no database match\n","user_acts4 = {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}}}\n","system_act4 = {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'offerbook': {}}}\n","expected = 'Offerbook but there is no database match'\n","result = unit_test_schema(user_acts4, system_act4)\n","print('test4', 'passed' if result == expected else 'failed: ' + result) \n","\n","# pricerange is unknown so it's okay for system to ask\n","user_acts5 = {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'unknown'}}}\n","system_act5 = {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'request': {'pricerange': '?'}}}\n","expected = 'no error'\n","result = unit_test_schema(user_acts5, system_act5)\n","print('test5', 'passed' if result == expected else 'failed: ' + result)  \n","\n","# user_acts6 = {'hotel': {'inform': {'name': 'acorn guest house'}}}\n","# system_act6 = {'hotel': {'request': {'area', 'people'}}}\n","# expected = 'Requesting an entity slot that is entailed'\n","# result = unit_test_schema(user_acts6, system_act6)\n","# print('test6', 'passed' if result == expected else 'failed: ' + result) \n","\n","# not ok to switch domain from restaurant to hotel\n","user_acts7 = {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'unknown'}}}\n","system_act7 = {'hotel': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'request': {'pricerange': '?'}}}\n","expected = 'System switched domain'\n","result = unit_test_schema(user_acts7, system_act7)\n","print('test7', 'passed' if result == expected else 'failed: ' + result) \n","\n","# ok to switch domain from restaurant to general\n","user_acts8 = {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'cheap'}}}\n","system_act8 = {'general': {'bye': {}}}\n","expected = 'no error'\n","result = unit_test_schema(user_acts8, system_act8)\n","print('test8', 'passed' if result == expected else 'failed: ' + result) \n","\n","# ok to switch domain from general to restaurant\n","user_acts9 = {'general': {'greet': {}}}\n","system_act9 = {'hotel': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'request': {'pricerange': '?'}}}\n","expected = 'no error'\n","result = unit_test_schema(user_acts9, system_act9)\n","print('test9', 'passed' if result == expected else 'failed: ' + result) "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"BdLfeqP4BRY5","executionInfo":{"status":"ok","timestamp":1651345550480,"user_tz":240,"elapsed":256,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"d5f67985-7b4f-49dd-969f-89c2fe26fd88"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["--------------------------------------\n","user_acts {'restaurant': {'inform': {'food': 'chinese'}}}\n","belief_state [restaurant] [value_food] chinese\n","system_acts {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'pricerange': '?'}}}\n","test1 passed\n","--------------------------------------\n","user_acts {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'name': '?'}}}\n","belief_state [restaurant] [value_food] chinese\n","system_acts {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'pricerange': '?'}}}\n","test2 passed\n","--------------------------------------\n","user_acts {'restaurant': {'inform': {'food': 'chinese'}, 'request': {'name': '?'}}}\n","belief_state [restaurant] [value_food] chinese\n","system_acts {'restaurant': {'inform': {'food': 'chinese', 'name': 'hakka'}, 'request': {'pricerange': '?'}}}\n","test3 passed\n","--------------------------------------\n","user_acts {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}}}\n","[restaurant] user informed categorical slot value out of range: pricerange=hight\n","defaultdict(<class 'dict'>, {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}}})\n","belief_state [restaurant] [value_food] chinese [value_area] west\n","system_acts {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'offerbook': {}}}\n","test4 passed\n","--------------------------------------\n","user_acts {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'unknown'}}}\n","belief_state [restaurant] [value_food] chinese [value_area] west [value_pricerange] unknown\n","system_acts {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'request': {'pricerange': '?'}}}\n","test5 passed\n","--------------------------------------\n","user_acts {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'unknown'}}}\n","belief_state [restaurant] [value_food] chinese [value_area] west [value_pricerange] unknown\n","system_acts {'hotel': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'request': {'pricerange': '?'}}}\n","test7 passed\n","--------------------------------------\n","user_acts {'restaurant': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'cheap'}}}\n","belief_state [restaurant] [value_food] chinese [value_area] west [value_pricerange] cheap\n","system_acts {'general': {'bye': {}}}\n","test8 passed\n","--------------------------------------\n","user_acts {'general': {'greet': {}}}\n","belief_state \n","system_acts {'hotel': {'inform': {'food': 'chinese', 'area': 'west', 'pricerange': 'hight'}, 'request': {'pricerange': '?'}}}\n","test9 passed\n"]}]},{"cell_type":"markdown","source":["## Interactive Interface\n","Good for manual testing or error analysis"],"metadata":{"id":"YW5x_3a1JWEw"}},{"cell_type":"code","source":["from collections import deque\n","\n","\n","def interactive_dialog_dual(context_window=10, budget=5):\n","    \"\"\"Launch interactive dialog session with dual system, enter exit to exit\n","    Domain not needed because it's tracked in belief state and thus dialog not restricted\n","    to any single domain.\n","    \"\"\"\n","    nlp = spacy.load(\"en_core_web_sm\")\n","    mt, md = MosesTokenizer(lang='en'), MosesDetokenizer(lang='en')\n","\n","    belief_state = BeliefState()\n","    context = deque(maxlen=context_window)\n","    last_resp = None\n","\n","    while True:\n","        utterance = input(f'user: ').lower()\n","        # Make sure we tokenize utterance like training\n","        utterance = ' '.join(t.text for t in nlp(utterance))\n","        utterance = ' '.join(utterance.replace(\".\", \" . \").split())\n","        if utterance == 'exit':\n","            break\n","\n","        # Parse user action\n","        user_act = parse_user_acts_batch([last_resp], [utterance])[0]\n","        print('user_act:', user_act)\n","        belief_state.update_user(user_act)\n","\n","        # Feed into system 1 slots from tracked belief state and generate 5 candidate responses\n","        slots = belief_state.serialize()\n","        resps, curr_turns = generate_response_batch([context], [utterance],\n","                                                    [belief_state.active_domains],\n","                                                    slots=[slots],\n","                                                    budget=budget)\n","\n","        # Fill placesholders with DB entities\n","        filled_resps = [fill_entities(resp, slots, belief_state.active_domains) for resp in resps]\n","\n","        # Parse system actions batch\n","        system_acts = parse_system_acts_batch([utterance] * budget, resps)\n","        print('candidate system_acts:', system_acts)\n","        # Determine which candidate response out of 5 passes consistency check\n","        for valid_idx, system_act in enumerate(system_acts):\n","            try:\n","                belief_state.update_system(system_act)\n","                break\n","            except Exception as e:\n","                print(e)\n","                pass\n","        else:\n","            # If no candidate is consistent, accepts the last one\n","            print('All candidates are inconsistent')\n","            belief_state.update_system(system_acts[-1], force=True)\n","\n","        print(\"chosen sys act:\", system_acts[valid_idx])\n","        valid_filled_resp, valid_curr_turn = filled_resps[valid_idx], curr_turns[valid_idx]\n","        context.append(valid_curr_turn) # update context\n","        last_resp = valid_filled_resp\n","\n","        # detokenize response\n","        detokenized_resp = md.detokenize(mt.tokenize(valid_filled_resp,\n","                                                     protected_patterns=['value_', ']-', ']s', ']ly']))\n","        print(f\"system: {detokenized_resp}\")\n","\n","    print(\"\\n====== context ======\\n\\n\" + '\\n'.join(context))\n"],"metadata":{"id":"LhRKVd3vZR_C"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["interactive_dialog_dual()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zgT9WT6f7yT6","executionInfo":{"status":"ok","timestamp":1651112057390,"user_tz":240,"elapsed":214282,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"8a00b058-5329-4c21-ff7a-51dd3905b0f9"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["user: hello, i'm looking for a nice place to dine\n","user_act: {'general': {'greet': {}}}\n","candidate system_acts: [{'restaurant': {'request': {'food'}}}, {'restaurant': {'request': {'pricerange'}}}, {'restaurant': {'request': {'area', 'food'}}}, {'restaurant': {'inform': {'choice'}, 'request': {'pricerange'}}}, {'restaurant': {'inform': {'choice'}, 'request': {'food'}}}]\n","chosen sys act: {'restaurant': {'request': {'food'}}}\n","system: sure, do you want to eat a particular type of food?\n","user: chinese cuisine pleae\n","user_act: {'restaurant': {'inform': {'food': 'chinese'}}}\n","candidate system_acts: [{'restaurant': {'inform': {'choice', 'food'}, 'request': {'pricerange'}}}, {'restaurant': {'inform': {'choice', 'food'}, 'request': {'pricerange'}}}, {'restaurant': {'request': {'pricerange'}}}, {'restaurant': {'request': {'area'}}}, {'restaurant': {'inform': {'choice', 'food'}, 'recommend': {'name'}, 'offerbook': set()}}]\n","chosen sys act: {'restaurant': {'inform': {'choice', 'food'}, 'request': {'pricerange'}}}\n","system: there are many chinese restaurants in town. do you have a price range?\n","user: no, i don't care\n","user_act: {'restaurant': {'inform': {'pricerange': 'dontcare'}}}\n","candidate system_acts: [{'restaurant': {'inform': {'choice', 'food'}, 'request': {'area'}}}, {'restaurant': {'recommend': {'area', 'name', 'pricerange'}, 'offerbook': set()}}, {'restaurant': {'recommend': {'name'}, 'offerbook': set()}}, {'restaurant': {'inform': {'choice', 'food'}, 'recommend': {'name'}}}, {'restaurant': {'recommend': {'name'}}}]\n","chosen sys act: {'restaurant': {'inform': {'choice', 'food'}, 'request': {'area'}}}\n","system: there are many chinese restaurants available. do you prefer a specific part of town?\n","user: somewhere in the city hub please\n","user_act: {'restaurant': {'inform': {'area': 'centre'}}}\n","candidate system_acts: [{'restaurant': {'recommend': {'area', 'name', 'pricerange'}}, 'general': {'reqmore': set()}}, {'restaurant': {'inform': {'pricerange', 'choice', 'food'}, 'request': {'pricerange'}}}, {'restaurant': {'inform': {'name', 'pricerange'}, 'offerbook': set()}}, {'restaurant': {'inform': {'area', 'pricerange', 'name', 'food'}, 'offerbook': set()}}, {'restaurant': {'inform': {'area', 'choice', 'food'}, 'select': set()}}]\n","chosen sys act: {'restaurant': {'recommend': {'area', 'name', 'pricerange'}}, 'general': {'reqmore': set()}}\n","system: shanghai family restaurant is a nice place in the centre that is moderate. would you like their address?\n","user: book it for 2 people at 2 pm on monday\n","user_act: {'restaurant': {'inform': {'day': 'monday', 'people': '2', 'time': '2 pm', 'name': 'shanghai family restaurant'}}}\n","candidate system_acts: [{'restaurant': {'nobook': set()}}, {'restaurant': {'offerbook': {'time', 'people', 'day'}}}, {'restaurant': {'offerbooked': {'time', 'reference'}}, 'general': {'reqmore': set()}}, {'restaurant': {'offerbooked': {'time', 'reference', 'people', 'day'}}}, {'restaurant': {'offerbooked': {'people', 'name', 'day', 'reference', 'time'}}, 'general': {'reqmore': set()}}]\n","chosen sys act: {'restaurant': {'nobook': set()}}\n","system: i am sorry, they do n't have any availability at that time.\n","user: that's too bad. can you recommend another restaurant\n","user_act: {'restaurant': {}}\n","candidate system_acts: [{'restaurant': {'inform': {'choice'}, 'select': {'food'}, 'offerbook': set()}}, {'restaurant': {'offerbooked': {'reference'}}, 'general': {'reqmore': set()}}, {'restaurant': {'nooffer': {'pricerange', 'food'}, 'request': {'food'}}}, {'restaurant': {'inform': {'name', 'food'}}, 'general': {'reqmore': set()}}, {'restaurant': {'nobook': set(), 'request': {'time'}}}]\n","Requesting a slot that is already filled\n","chosen sys act: {'restaurant': {'offerbooked': {'reference'}}, 'general': {'reqmore': set()}}\n","system: booking was successful. the table will be reserved for 15 minutes. reference number is: 1n367zx5. can i help you with anything else today?\n","user: exit\n","\n","====== context ======\n","\n","user: hello , i 'm looking for a nice place to dine\n","state: \n","match: null\n","action: [restaurant] [request] food\n","response: sure , do you want to eat a particular type of food ?\n","\n","user: chinese cuisine pleae\n","state: [restaurant] [value_food] chinese\n","match: 3\n","action: [restaurant] [inform] choice food [request] pricerange\n","response: there are [value_choice] [value_food] restaurants in town . do you have a price range ?\n","\n","user: no , i do n't care\n","state: [restaurant] [value_food] chinese [value_pricerange] dontcare\n","match: 3\n","action: [restaurant] [inform] choice food [request]\n","response: there are [value_choice] [value_food] restaurants available . do you prefer a specific part of town ?\n","\n","user: somewhere in the city hub please\n","state: [restaurant] [value_food] chinese [value_pricerange] dontcare [value_area] centre\n","match: 3\n","action: [restaurant] [recommend] area name pricerange [general] [reqmore]\n","response: [value_name] is a nice place in the [value_area] that is [value_pricerange] . would you like their address ?\n","\n","user: book it for 2 people at 2 pm on monday\n","state: [restaurant] [value_food] chinese [value_pricerange] dontcare [value_area] centre [value_day] monday [value_people] 2 [value_time] 2 pm [value_name] shanghai family restaurant\n","match: 1\n","action: [restaurant] [nobook]\n","response: i am sorry , they do n't have any availability at that time .\n","\n","user: that 's too bad . can you recommend another restaurant\n","state: [restaurant] [value_food] chinese [value_pricerange] dontcare [value_area] centre [value_day] monday [value_people] 2 [value_time] 2 pm [value_name] shanghai family restaurant\n","match: 1\n","action: [restaurant] [offerbooked] reference [general] [reqmore]\n","response: booking was successful . the table will be reserved for 15 minutes . reference number is : [value_reference] . can i help you with anything else today ?\n","\n"]}]},{"cell_type":"markdown","metadata":{"id":"mG9KN4TsyCvN"},"source":["# Evaluate Inform, Success, BLEU\n","A dialogue is marked as successful if for each domain in the user’s dialogue goal: (1) the last offered entity (booking) matches (satisfies the goal constraints), and (2) the system mentioned all requestable slots required by the user. The Inform rate then marks the proportion of dialogues complying to (1), Success rate is the proportion of fully successful dialogues.\n","\n","BLEU measures the fluency of generated responses."]},{"cell_type":"markdown","source":["Add slot mapping in eval script"],"metadata":{"id":"ah2RVyUhs1N2"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":770,"status":"ok","timestamp":1651277271866,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"},"user_tz":240},"id":"zaPrzpGWvsn1","outputId":"afdb8dd8-7cde-4d0e-c40d-dd0bf7a6b973"},"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MultiWOZ_Evaluation\n","patching file mwzeval/normalization.py\n"]}],"source":["%cd /content/MultiWOZ_Evaluation\n","\n","patch = \"\"\"\\\n","51,52c51,52\n","<      'INTERNET'  : ['internet', 'hotel_internet'],\n","<      'PARKING'   : ['parking', 'hotel_parking'],\n","---\n",">      'INTERNET'  : ['internet', 'hotel_internet', 'value_internet'],\n",">      'PARKING'   : ['parking', 'hotel_parking', 'value_parking'],\n","55c55\n","<      'OPEN'      : ['openhours']\n","---\n",">      'OPEN'      : ['openhours', 'value_openhours']\n","\"\"\"\n","\n","with open('normalization_patch.txt', 'w') as f:\n","    f.write(patch)\n","\n","!patch mwzeval/normalization.py < normalization_patch.txt"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"sFxZRmoxyJjK"},"outputs":[],"source":["from tqdm.auto import tqdm\n","from collections import defaultdict, deque\n","from itertools import chain\n","\n","\n","TEST_DIAL_DOMAINS = ['hotel', 'restaurant', 'restaurant-hotel']\n","\n","\n","def chunks(lst, n):\n","    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n","    for i in range(0, len(lst), n):\n","        yield lst[i:i + n]\n","\n","\n","def test(output_path, batch_size=20, max_dialogs=1000, context_window=10, budget=5, dual_system=False, split='test'):\n","    outputs = defaultdict(list)\n","\n","    with open(Path(data_dir) / f'{split}_data.json') as inp:\n","        data = json.load(inp)\n","    with open(Path(root_dir) / 'MultiWOZ_2.1' / 'dial_by_domain.json') as inp:\n","        dial_by_domain = json.load(inp)\n","    print(\"# dialogs in dataset:\", len(data))\n","\n","    domain_dial_ids = set(chain.from_iterable(dial_by_domain[domain] for domain in TEST_DIAL_DOMAINS))\n","    dial_ids = [dial_id for dial_id in data if dial_id in domain_dial_ids]\n","\n","    num_dialogs = min(max_dialogs, len(dial_ids))\n","    print(\"# dialogs tested:\", num_dialogs)\n","    dial_ids = dial_ids[:num_dialogs]\n","    total_iter = num_dialogs // batch_size if num_dialogs > batch_size else 1\n","\n","    context_by_id = defaultdict(lambda: deque(maxlen=context_window))\n","    belief_state_by_id = defaultdict(lambda: BeliefState())\n","    last_resp_by_id = defaultdict(lambda: None)\n","    rej_count, fail_count = 0, 0\n","\n","    dial_len_by_id = {}\n","    for dial_id, dialog in data.items():\n","        dial_len_by_id[dial_id] = len(dialog['log'])\n","    \n","    # Perform inference for a batch of dialogs at the same time\n","    for dial_id_batch in tqdm(chunks(dial_ids, batch_size), total=total_iter):\n","        turn_num = 0\n","\n","        while True:\n","            active_dial_ids = [dial_id for dial_id in dial_id_batch if turn_num < dial_len_by_id[dial_id]]\n","            if len(active_dial_ids) == 0:\n","                break\n","\n","            contexts = [context_by_id[dial_id] for dial_id in active_dial_ids]\n","            utterances = [data[dial_id]['log'][turn_num]['user'] for dial_id in active_dial_ids]\n","\n","            # groundtruth domains are used to query Database\n","            gt_domains = [data[dial_id]['log'][turn_num]['turn_domain'].split()\n","                          for dial_id in active_dial_ids]\n","            # gt_slots = [data[dial_id]['log'][turn_num]['constraint'] for dial_id in active_dial_ids]\n","            # gt_acts = [data[dial_id]['log'][turn_num]['sys_act'] for dial_id in active_dial_ids]\n","            gt_resps = [data[dial_id]['log'][turn_num]['resp'] for dial_id in active_dial_ids]\n","            gt_nodelx_resps = [data[dial_id]['log'][turn_num]['nodelx_resp'] for dial_id in active_dial_ids]\n","\n","            # Generate responses for the current turn\n","            if dual_system:\n","                belief_states = [belief_state_by_id[dial_id] for dial_id in active_dial_ids]\n","                last_resps = [last_resp_by_id[dial_id] for dial_id in active_dial_ids]\n","                domains = [bs.active_domains for bs in belief_states]\n","                \n","                # Parse user utterance\n","                # TODO: maybe use gt last resps b/c last_resps may not relate to gt utterance\n","                # serialized_bs = [bs.serialize() for bs in belief_states]\n","                user_acts = parse_user_acts_batch(last_resps, utterances)\n","\n","                # Update belief state based on user actions\n","                for bs, user_act in zip(belief_states, user_acts):\n","                    bs.update_user(user_act)\n","\n","                # Feed in slots from tracked belief state\n","                # Generate 5 candidate responses for each dialog\n","                slots_batch = [bs.serialize() for bs in belief_states]\n","                resps, curr_turns = generate_response_batch(contexts, utterances, domains,\n","                                                            slots=slots_batch, budget=budget)\n","                # resps, curr_turns = generate_response_batch(contexts, utterances, gt_domains,\n","                #                                             slots=slots_batch, budget=budget)\n","\n","                output_resps = []\n","                for i in range(0, len(resps), budget):\n","                    dial_idx = i // budget\n","                    candidate_resps = resps[i:i + budget]\n","\n","                    # Fill placeholders with DB entities\n","                    # filled_resps = [fill_entities(resp, slots_batch[dial_idx], gt_domains[dial_idx])\n","                    #                 for resp in candidate_resps]\n","                    # filled_resps = [fill_entities(resp, slots_batch[dial_idx], domains[dial_idx])\n","                    #                 for resp in candidate_resps]\n","\n","                    # Parse system actions batch\n","                    system_acts = parse_system_acts_batch([utterances[dial_idx]] * budget, candidate_resps)\n","    \n","                    # Determine which candidate response out of 5 passes consistency check\n","                    for valid_idx, system_act in enumerate(system_acts):\n","                        try:\n","                            belief_states[dial_idx].update_system(system_act)\n","                            # belief_states[dial_idx].update_system(system_act, gt_domains[dial_idx])\n","                            break\n","                        except:\n","                            rej_count += 1         \n","                    else:\n","                        # If no candidate is consistent, accepts the last one\n","                        belief_states[dial_idx].update_system(system_acts[-1], force=True)\n","                        # belief_states[dial_idx].update_system(system_acts[-1],\n","                        #                                       gt_domains[dial_idx],\n","                        #                                       force=True)\n","                        fail_count += 1 \n","\n","                    output_resps.append(candidate_resps[valid_idx])\n","\n","                    # Update context and last response                    \n","                    contexts[dial_idx].append(curr_turns[i + valid_idx])\n","                    # last_resp_by_id[active_dial_ids[dial_idx]] = filled_resps[valid_idx]\n","                    last_resp_by_id[active_dial_ids[dial_idx]] = gt_nodelx_resps[dial_idx]\n","            else:\n","                slots_batch = predict_slots_batch(contexts, utterances)\n","                output_resps, curr_turns = generate_response_batch(contexts, utterances, gt_domains,\n","                                                                   slots=slots_batch)\n","                # Update contexts\n","                for context, curr_turn in zip(contexts, curr_turns):\n","                    context.append(curr_turn)\n","\n","            # Update output\n","            for i, dial_id in enumerate(active_dial_ids):\n","                dial_id = dial_id.split('.')[0]\n","                outputs[dial_id].append({\n","                    'response': output_resps[i],\n","                    'state': constraint_to_dict(slots_batch[i])\n","                })\n","\n","            turn_num += 1\n","\n","    with open(output_path, 'w') as out:\n","        json.dump(outputs, out, indent=4)\n","\n","    if dual_system:\n","        total_dial_len = sum(dial_len_by_id[dial_id] for dial_id in dial_ids)\n","        print(\"-------------------------------------\")\n","        print(\"Avg reject rate:\", rej_count / (total_dial_len * budget))\n","        print(\"Avg fail rate:\", fail_count / total_dial_len)\n"]},{"cell_type":"markdown","source":["## Rules A/B test on dev set"],"metadata":{"id":"KpaufXO-EcSC"}},{"cell_type":"markdown","source":["Dual system:"],"metadata":{"id":"-pym4JgmC2zD"}},{"cell_type":"code","source":["dual_system_output = Path(root_dir) / 'dual_system_dev.json'\n","test(dual_system_output, max_dialogs=100, dual_system=True, split='dev')"],"metadata":{"id":"X7wSNSeHEvKH","colab":{"base_uri":"https://localhost:8080/","height":260,"referenced_widgets":["5a32f1c3aa2445578205fd19db115ff0","425aed8ff1e04c3bb75056fe3d94c65f","d780eb2f01a147dcb47fbe3dc64045d3","8ab1abd2052d4776a5e2c2ac2fee3bbb","3d163edd49d54fa8b6bf0607d1b4bed0","dee40cf47f1a43df9ad499dec32f9fdc","c853f4fa74b24f4bb9b8c82dd874191d","55028b0cd3964150856db5b5afc7fc74","c614cbf9b0124374a469f1c5b46a8c92","377742744d0748d1a7a44669aac403c9","9a9b6736054e4166a48b53ebf46bf5ad"]},"executionInfo":{"status":"ok","timestamp":1651347270080,"user_tz":240,"elapsed":263182,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"4ef59d90-fb8f-4ace-cd2f-316d31692705"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["# dialogs in dataset: 999\n","# dialogs tested: 100\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/5 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5a32f1c3aa2445578205fd19db115ff0"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Invalid system act token: restaurora\n","Action string: [restaurora] [inform] choice area food name phone postcode\n","[hotel] user requested slot out of range: persian\n","defaultdict(<class 'dict'>, {'hotel': {'inform': {'name': 'archway house'}, 'request': {'area': '?', 'phone': '?', 'persian': '?'}}})\n","[restaurant] user requested slot out of range: stars\n","defaultdict(<class 'dict'>, {'restaurant': {'inform': {'name': 'cotto'}, 'request': {'address': '?', 'phone': '?', 'stars': '?'}}})\n","-------------------------------------\n","Avg reject rate: 0.15415472779369627\n","Avg fail rate: 0.09169054441260745\n"]}]},{"cell_type":"markdown","source":["After:"],"metadata":{"id":"Ty2Y1x19IwOB"}},{"cell_type":"code","source":["%cd /content/MultiWOZ_Evaluation\n","#!python evaluate.py --bleu --success --input {dual_system_output}\n","!python evaluate.py --bleu --success --input '/content/drive/MyDrive/CIS700 Project/dual_system_dev.json'"],"metadata":{"id":"KlBti--cE0O8","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1651347272025,"user_tz":240,"elapsed":1961,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"510eaf09-a0bd-4dd4-ae8d-e368cd152b63"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MultiWOZ_Evaluation\n","====== BLEU ======\n","mwz22          11.772919339758692\n","\n","====== SUCCESS ======\n","inform         {'hotel': 74.2, 'restaurant': 90.3, 'total': 78.0}\n","success        {'hotel': 68.2, 'restaurant': 69.9, 'total': 72.0}\n","\n"]}]},{"cell_type":"markdown","source":["Before:"],"metadata":{"id":"RWCLvzdpIxkg"}},{"cell_type":"code","source":["%cd /content/MultiWOZ_Evaluation\n","!python evaluate.py --bleu --success --input {dual_system_output}"],"metadata":{"id":"8EGbxbpwnlys"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["/content/MultiWOZ_Evaluation\n","\n","====== BLEU ======\n","\n","mwz22          12.238295357960922\n","\n","====== SUCCESS ======\n","\n","inform         {'hotel': 78.8, 'total': 78.0, 'restaurant': 89.2}\n","\n","success        {'hotel': 65.2, 'total': 72.0, 'restaurant': 72.0}"],"metadata":{"id":"Z0JWhB3A7IYP"}},{"cell_type":"markdown","source":["Single system baseline:"],"metadata":{"id":"WFfZ_BJiCyIq"}},{"cell_type":"code","source":["baseline_output = Path(root_dir) / 'ubar_gpt3_dev.json'\n","test(baseline_output, max_dialogs=100, split='dev')"],"metadata":{"id":"3c36hMFfZhee"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["%cd /content/MultiWOZ_Evaluation\n","!python evaluate.py --bleu --success --input {baseline_output}"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1651102983104,"user_tz":240,"elapsed":1896,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"4ca9ee5c-08bc-4fc8-b407-f34b1f320643","id":"kRqDwKuSZhee"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MultiWOZ_Evaluation\n","====== BLEU ======\n","mwz22          13.47579690127236\n","\n","====== SUCCESS ======\n","inform         {'total': 69.0, 'restaurant': 80.6, 'hotel': 77.3}\n","success        {'total': 52.0, 'restaurant': 57.0, 'hotel': 60.6}\n","\n"]}]},{"cell_type":"markdown","source":["## Evaluate on test set"],"metadata":{"id":"VCk_lxUuvOIH"}},{"cell_type":"markdown","source":["Dual system"],"metadata":{"id":"ZhqeIlfiDFhJ"}},{"cell_type":"code","source":["dual_system_output = Path(root_dir) / 'dual_system.json'\n","test(dual_system_output, max_dialogs=1000, dual_system=True)"],"metadata":{"id":"95DhOHptOo0n","colab":{"base_uri":"https://localhost:8080/","height":171,"referenced_widgets":["a3052ce4e6474a318d4d51c8ddf709b9","1c51ce5b7ab04d2e82254e49fb743047","ef52e47755614d4688209783adb79354","bd1f36f9b22f42a8b47f3635d21755e0","52194bd6bf8445d48352f0308c2b5c59","a658acab045f4bc0b632e438d2a138bb","cd553e27e65343c4886ac802b1c584f3","e294e6d1f8f6481eb93a0053f457216e","5a1677792cd34496b1e4592f10b16962","7361fc0fc42f44e4bb433752ba385294","09b904830086415697a3400c8640b362"]},"executionInfo":{"status":"ok","timestamp":1651348474023,"user_tz":240,"elapsed":404209,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"50afd06f-3d45-41d4-c89b-ea8a37af2ce9"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["# dialogs in dataset: 1000\n","# dialogs tested: 178\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/8 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a3052ce4e6474a318d4d51c8ddf709b9"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Invalid system act token: gastro\n","Action string: [general] [gastro] [hotel] [request] food\n","-------------------------------------\n","Avg reject rate: 0.14953271028037382\n","Avg fail rate: 0.08037383177570094\n"]}]},{"cell_type":"code","source":["%cd /content/MultiWOZ_Evaluation\n","!python evaluate.py --bleu --success --richness --input '/content/drive/MyDrive/CIS700 Project/dual_system.json'"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"uz6sQnGN-X8j","executionInfo":{"status":"ok","timestamp":1651348476040,"user_tz":240,"elapsed":2039,"user":{"displayName":"Liyang Zhou","userId":"12409476534595626967"}},"outputId":"65c0cc19-bfdd-4d15-8f22-7dfcab209e3c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MultiWOZ_Evaluation\n","====== BLEU ======\n","mwz22          11.713521875890796\n","\n","====== SUCCESS ======\n","inform         {'total': 84.8, 'hotel': 88.8, 'restaurant': 87.4}\n","success        {'total': 77.0, 'hotel': 76.7, 'restaurant': 73.0}\n","\n","====== RICHNESS ======\n","entropy        6.5238126634077735\n","cond_entropy   2.5493941947130763\n","avg_lengths    14.13551401869159\n","msttr          0.6823841059602643\n","num_unigrams   413\n","num_bigrams    2143\n","num_trigrams   3891\n","\n"]}]},{"cell_type":"markdown","source":["Single system baseline"],"metadata":{"id":"WJSO19BfvIfP"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"KjKRDMcuofWo","colab":{"base_uri":"https://localhost:8080/","height":84,"referenced_widgets":["2d1b47eacde8411a8f795afc3c3188f5","da0d9ce8602541148f573c66891985ac","518e24d8966a4be4bd5fcff606bba49c","46ec9df482f2406baba9ed4228e835cc","c0fc13542c1f4c4f8b560fbb7f7d8581","335dd67693854b84a97b14b91cabfdb1","42f11698d4ba49c882366ece07292d8f","de93c75b7bdd4a2ca066c0749fd67190","7deabb06869b401f849b5dfa4bd1f58d","8d1e656a199d422c9b11b5da943293cb","7d3dba1b720845d9941275e14066f7ad"]},"executionInfo":{"status":"ok","timestamp":1651042900465,"user_tz":240,"elapsed":77562,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"622ab38b-bafd-4614-dbf3-7135b27a8bd7"},"outputs":[{"output_type":"stream","name":"stdout","text":["# dialogs in dataset: 1000\n","# dialogs tested: 100\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/5 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2d1b47eacde8411a8f795afc3c3188f5"}},"metadata":{}}],"source":["baseline_output = Path(root_dir) / 'ubar_gpt3.json'\n","test(baseline_output, max_dialogs=1000)"]},{"cell_type":"code","source":["%cd /content/MultiWOZ_Evaluation\n","!python evaluate.py --bleu --success --richness --input {baseline_output}"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eWMoPEKuMM9m","executionInfo":{"status":"ok","timestamp":1650768893986,"user_tz":240,"elapsed":2274,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"e2d19430-75b3-43c8-c919-75796b2cba12"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MultiWOZ_Evaluation\n","====== BLEU ======\n","mwz22          12.179223790023183\n","\n","====== SUCCESS ======\n","inform         {'restaurant': 90.1, 'hotel': 76.7, 'total': 79.8}\n","success        {'restaurant': 64.9, 'hotel': 64.7, 'total': 61.2}\n","\n"]}]},{"cell_type":"markdown","source":["## Original UBAR performance\n","\n","They use special tokens for domains/slots and trained GPT-2 for 60 epochs. Also, their model was trained on MultiWOZ 2.1 not 2.2"],"metadata":{"id":"_KhrTPVbvQrJ"}},{"cell_type":"code","source":["%cd /content/MultiWOZ_Evaluation\n","\n","with open(Path(root_dir) / 'MultiWOZ_2.1' / 'dial_by_domain.json') as inp:\n","    dial_by_domain = json.load(inp)\n","\n","with open('predictions/ubar.json') as inp:\n","    ubar_pred = json.load(inp)\n","\n","domain_dial_ids = set(chain.from_iterable(dial_by_domain[domain] for domain in TEST_DIAL_DOMAINS))\n","ubar_pred = {dial_id: pred for dial_id, pred in ubar_pred.items() if f'{dial_id}.json'in domain_dial_ids}\n","\n","with open('predictions/ubar_hotel_restaurant.json', 'w') as outp:\n","    json.dump(ubar_pred, outp, indent=4)\n","\n","!python evaluate.py --bleu --success --richness --input predictions/ubar_hotel_restaurant.json"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0Oj8Ia7qY5V4","executionInfo":{"status":"ok","timestamp":1651119821373,"user_tz":240,"elapsed":2715,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"}},"outputId":"71e3edf2-b1b6-4318-bf23-ff782ca89747"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/MultiWOZ_Evaluation\n","====== BLEU ======\n","mwz22          15.488692099463337\n","\n","====== SUCCESS ======\n","inform         {'restaurant': 91.0, 'total': 85.4, 'hotel': 84.5}\n","success        {'restaurant': 78.4, 'total': 75.3, 'hotel': 72.4}\n","\n","====== RICHNESS ======\n","entropy        6.255988954256211\n","cond_entropy   1.9250400825755103\n","avg_lengths    13.836448598130842\n","msttr          0.6917567567567565\n","num_unigrams   271\n","num_bigrams    1094\n","num_trigrams   1846\n","\n"]}]},{"cell_type":"markdown","source":["All domains"],"metadata":{"id":"TzH04H5TZEi9"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":16832,"status":"ok","timestamp":1651119534289,"user":{"displayName":"Alexis Yang","userId":"04227381221697288986"},"user_tz":240},"id":"8_-nziE_--Re","outputId":"80876eac-9753-4045-ec22-b7f399e9adcf"},"outputs":[{"output_type":"stream","name":"stdout","text":["====== BLEU ======\n","mwz22          17.59580063564326\n","\n","====== SUCCESS ======\n","inform         {'total': 83.4, 'taxi': 100.0, 'attraction': 92.4, 'restaurant': 89.9, 'train': 92.5, 'hotel': 83.8}\n","success        {'total': 70.3, 'taxi': 77.4, 'attraction': 71.5, 'restaurant': 74.1, 'train': 81.0, 'hotel': 68.8}\n","\n"]}],"source":["%cd /content/MultiWOZ_Evaluation\n","!python evaluate.py --bleu --success --input predictions/ubar.json"]}],"metadata":{"colab":{"collapsed_sections":[],"provenance":[{"file_id":"1iGyHc6H5ENU0uB9V4tgTqtaAQBZhs2na","timestamp":1651249743966}],"machine_shape":"hm"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"e65507122c8546a7b775cf2ac303d830":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_44bcad28d38c4d90a0f647ba5eb7cf15","IPY_MODEL_113f485c51b94274baf77c15cb9abbb1","IPY_MODEL_44737be04f924cdc9a0b8e68516070e7"],"layout":"IPY_MODEL_483a3a4b77ce4b16a73e0e03ba120a94"}},"44bcad28d38c4d90a0f647ba5eb7cf15":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d796229fee5b415fba8bdc7ed67f0030","placeholder":"​","style":"IPY_MODEL_c052b264d10a437eb741c8d6b09e4883","value":"100%"}},"113f485c51b94274baf77c15cb9abbb1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_aee2fd94b8d84b818bb455ded2c40bcd","max":8433,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c10d3b18fe8245c5a7e18144210dd18d","value":8433}},"44737be04f924cdc9a0b8e68516070e7":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5549fb8db17d477c9b31114ef001196e","placeholder":"​","style":"IPY_MODEL_9576e77f50b94f5ca953da58a63a75f9","value":" 8433/8433 [03:48&lt;00:00, 236.20it/s]"}},"483a3a4b77ce4b16a73e0e03ba120a94":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d796229fee5b415fba8bdc7ed67f0030":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c052b264d10a437eb741c8d6b09e4883":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"aee2fd94b8d84b818bb455ded2c40bcd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c10d3b18fe8245c5a7e18144210dd18d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"5549fb8db17d477c9b31114ef001196e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9576e77f50b94f5ca953da58a63a75f9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d4ad7ff0f55f40a3aae6480b2ea56ffe":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f410e94af377487eb15dccc05e9dbd1f","IPY_MODEL_f69d770baa0f4c798b13a980a5c9a7b3","IPY_MODEL_16387c180e1846049308803627c3a9f1"],"layout":"IPY_MODEL_ee1cc0adcf6b4fd7af79525004fffe9c"}},"f410e94af377487eb15dccc05e9dbd1f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b509ba5220f74a0ebb5154c6c9efd7d0","placeholder":"​","style":"IPY_MODEL_ca535199c87e46b1a5afc8719a107288","value":"100%"}},"f69d770baa0f4c798b13a980a5c9a7b3":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_9a4dece219b143e890775caa21f4d9d7","max":8433,"min":0,"orientation":"horizontal","style":"IPY_MODEL_dd868393dc1448199decf276d16433b7","value":8433}},"16387c180e1846049308803627c3a9f1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6469a8019eca4eeabc4fd4e069e8c6af","placeholder":"​","style":"IPY_MODEL_c7818afcb3f84d1b815eb2c1fd684b72","value":" 8433/8433 [00:00&lt;00:00, 44367.14it/s]"}},"ee1cc0adcf6b4fd7af79525004fffe9c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b509ba5220f74a0ebb5154c6c9efd7d0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ca535199c87e46b1a5afc8719a107288":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9a4dece219b143e890775caa21f4d9d7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dd868393dc1448199decf276d16433b7":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"6469a8019eca4eeabc4fd4e069e8c6af":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c7818afcb3f84d1b815eb2c1fd684b72":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"507977436fff4bc9910e498ce27a206e":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7e2bc8b499c64f67befcf2627f174c72","IPY_MODEL_caf50d5f77234e8d8bde0bd64bcd61e1","IPY_MODEL_1d21915c594e4902a507070a4dc51a64"],"layout":"IPY_MODEL_53ea1b21670142e29a3ca4711ca54374"}},"7e2bc8b499c64f67befcf2627f174c72":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b2f399360ee7470591588bdbc1104234","placeholder":"​","style":"IPY_MODEL_0e557c0698164754929b96d41796e71f","value":"100%"}},"caf50d5f77234e8d8bde0bd64bcd61e1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_da77fd8192ea4646af72bc5eb3ae3b31","max":8433,"min":0,"orientation":"horizontal","style":"IPY_MODEL_d0216e0292c4497cb4118c2ce194df2f","value":8433}},"1d21915c594e4902a507070a4dc51a64":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_038a1d215b7944409fb4bf8ca7270a87","placeholder":"​","style":"IPY_MODEL_58d10b029416452b9aee009a1428e5a6","value":" 8433/8433 [00:00&lt;00:00, 40473.44it/s]"}},"53ea1b21670142e29a3ca4711ca54374":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b2f399360ee7470591588bdbc1104234":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0e557c0698164754929b96d41796e71f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"da77fd8192ea4646af72bc5eb3ae3b31":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d0216e0292c4497cb4118c2ce194df2f":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"038a1d215b7944409fb4bf8ca7270a87":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"58d10b029416452b9aee009a1428e5a6":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2d1b47eacde8411a8f795afc3c3188f5":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_da0d9ce8602541148f573c66891985ac","IPY_MODEL_518e24d8966a4be4bd5fcff606bba49c","IPY_MODEL_46ec9df482f2406baba9ed4228e835cc"],"layout":"IPY_MODEL_c0fc13542c1f4c4f8b560fbb7f7d8581"}},"da0d9ce8602541148f573c66891985ac":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_335dd67693854b84a97b14b91cabfdb1","placeholder":"​","style":"IPY_MODEL_42f11698d4ba49c882366ece07292d8f","value":"100%"}},"518e24d8966a4be4bd5fcff606bba49c":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_de93c75b7bdd4a2ca066c0749fd67190","max":5,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7deabb06869b401f849b5dfa4bd1f58d","value":5}},"46ec9df482f2406baba9ed4228e835cc":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8d1e656a199d422c9b11b5da943293cb","placeholder":"​","style":"IPY_MODEL_7d3dba1b720845d9941275e14066f7ad","value":" 5/5 [01:16&lt;00:00, 13.44s/it]"}},"c0fc13542c1f4c4f8b560fbb7f7d8581":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"335dd67693854b84a97b14b91cabfdb1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"42f11698d4ba49c882366ece07292d8f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"de93c75b7bdd4a2ca066c0749fd67190":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7deabb06869b401f849b5dfa4bd1f58d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8d1e656a199d422c9b11b5da943293cb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7d3dba1b720845d9941275e14066f7ad":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5a32f1c3aa2445578205fd19db115ff0":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_425aed8ff1e04c3bb75056fe3d94c65f","IPY_MODEL_d780eb2f01a147dcb47fbe3dc64045d3","IPY_MODEL_8ab1abd2052d4776a5e2c2ac2fee3bbb"],"layout":"IPY_MODEL_3d163edd49d54fa8b6bf0607d1b4bed0"}},"425aed8ff1e04c3bb75056fe3d94c65f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_dee40cf47f1a43df9ad499dec32f9fdc","placeholder":"​","style":"IPY_MODEL_c853f4fa74b24f4bb9b8c82dd874191d","value":"100%"}},"d780eb2f01a147dcb47fbe3dc64045d3":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_55028b0cd3964150856db5b5afc7fc74","max":5,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c614cbf9b0124374a469f1c5b46a8c92","value":5}},"8ab1abd2052d4776a5e2c2ac2fee3bbb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_377742744d0748d1a7a44669aac403c9","placeholder":"​","style":"IPY_MODEL_9a9b6736054e4166a48b53ebf46bf5ad","value":" 5/5 [04:22&lt;00:00, 47.31s/it]"}},"3d163edd49d54fa8b6bf0607d1b4bed0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dee40cf47f1a43df9ad499dec32f9fdc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c853f4fa74b24f4bb9b8c82dd874191d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"55028b0cd3964150856db5b5afc7fc74":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c614cbf9b0124374a469f1c5b46a8c92":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"377742744d0748d1a7a44669aac403c9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9a9b6736054e4166a48b53ebf46bf5ad":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a3052ce4e6474a318d4d51c8ddf709b9":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_1c51ce5b7ab04d2e82254e49fb743047","IPY_MODEL_ef52e47755614d4688209783adb79354","IPY_MODEL_bd1f36f9b22f42a8b47f3635d21755e0"],"layout":"IPY_MODEL_52194bd6bf8445d48352f0308c2b5c59"}},"1c51ce5b7ab04d2e82254e49fb743047":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a658acab045f4bc0b632e438d2a138bb","placeholder":"​","style":"IPY_MODEL_cd553e27e65343c4886ac802b1c584f3","value":""}},"ef52e47755614d4688209783adb79354":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_e294e6d1f8f6481eb93a0053f457216e","max":8,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5a1677792cd34496b1e4592f10b16962","value":8}},"bd1f36f9b22f42a8b47f3635d21755e0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7361fc0fc42f44e4bb433752ba385294","placeholder":"​","style":"IPY_MODEL_09b904830086415697a3400c8640b362","value":" 9/? [06:43&lt;00:00, 40.90s/it]"}},"52194bd6bf8445d48352f0308c2b5c59":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a658acab045f4bc0b632e438d2a138bb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cd553e27e65343c4886ac802b1c584f3":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e294e6d1f8f6481eb93a0053f457216e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5a1677792cd34496b1e4592f10b16962":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7361fc0fc42f44e4bb433752ba385294":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"09b904830086415697a3400c8640b362":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0}