i2b2_2012_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2cd6ada5",
   "metadata": {},
   "source": [
    "## Step 1: convert xml to BRAT\n",
    "Code from Xi Yang (University of Florida)\n",
    "\n",
    "Modified by Lavender Jiang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cba6ca7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "from pathlib import Path\n",
    "import xmltodict\n",
    "import os\n",
    "import sys\n",
    "from collections import defaultdict\n",
    "import numpy as np\n",
    "import pathlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "5f8d43be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "making dir...\n"
     ]
    }
   ],
   "source": [
    "# folder for raw files\n",
    "infiles = {\"train\": \"/gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release\",\n",
    "           \"test\": \"/gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml\"}\n",
    "\n",
    "brat_out = {\"train\": \"brat-train\",\n",
    "            \"test\": \"brat-test\"}\n",
    "\n",
    "def get_paths(d, keys):\n",
    "    ptr = {}\n",
    "    for key in keys:\n",
    "        ptr[key] = Path(d[key])\n",
    "    return ptr\n",
    "\n",
    "def make_dir(d):\n",
    "    print(\"making dir...\")\n",
    "    for key in d.keys():\n",
    "        path = d[key]\n",
    "        if not type(path) == pathlib.PosixPath:\n",
    "            path = Path(path)\n",
    "        path.mkdir(parents=True, exist_ok=True)            \n",
    "\n",
    "splits = [\"train\", \"test\"]\n",
    "in_paths = get_paths(infiles, splits)\n",
    "out_paths = get_paths(brat_out, splits)\n",
    "make_dir(out_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "af4395dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copying text for split train. source path /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release, dest path brat-train\n",
      "copied doc 471 to brat-train/471.txt\n",
      "copied doc 596 to brat-train/596.txt\n",
      "copied doc 26 to brat-train/26.txt\n",
      "copied doc 192 to brat-train/192.txt\n",
      "copied doc 541 to brat-train/541.txt\n",
      "copied doc 682 to brat-train/682.txt\n",
      "copied doc 492 to brat-train/492.txt\n",
      "copied doc 331 to brat-train/331.txt\n",
      "copied doc 376 to brat-train/376.txt\n",
      "copied doc 86 to brat-train/86.txt\n",
      "copied doc 747 to brat-train/747.txt\n",
      "copied doc 776 to brat-train/776.txt\n",
      "copied doc 193 to brat-train/193.txt\n",
      "copied doc 462 to brat-train/462.txt\n",
      "copied doc 497 to brat-train/497.txt\n",
      "copied doc 11 to brat-train/11.txt\n",
      "copied doc 343 to brat-train/343.txt\n",
      "copied doc 307 to brat-train/307.txt\n",
      "copied doc 353 to brat-train/353.txt\n",
      "copied doc 267 to brat-train/267.txt\n",
      "copied doc 172 to brat-train/172.txt\n",
      "copied doc 636 to brat-train/636.txt\n",
      "copied doc 356 to brat-train/356.txt\n",
      "copied doc 191 to brat-train/191.txt\n",
      "copied doc 751 to brat-train/751.txt\n",
      "copied doc 256 to brat-train/256.txt\n",
      "copied doc 237 to brat-train/237.txt\n",
      "copied doc 121 to brat-train/121.txt\n",
      "copied doc 301 to brat-train/301.txt\n",
      "copied doc 6 to brat-train/6.txt\n",
      "copied doc 526 to brat-train/526.txt\n",
      "copied doc 676 to brat-train/676.txt\n",
      "copied doc 413 to brat-train/413.txt\n",
      "copied doc 252 to brat-train/252.txt\n",
      "copied doc 642 to brat-train/642.txt\n",
      "copied doc 151 to brat-train/151.txt\n",
      "copied doc 321 to brat-train/321.txt\n",
      "copied doc 707 to brat-train/707.txt\n",
      "copied doc 318 to brat-train/318.txt\n",
      "copied doc 458 to brat-train/458.txt\n",
      "copied doc 96 to brat-train/96.txt\n",
      "copied doc 612 to brat-train/612.txt\n",
      "copied doc 522 to brat-train/522.txt\n",
      "copied doc 351 to brat-train/351.txt\n",
      "copied doc 611 to brat-train/611.txt\n",
      "copied doc 47 to brat-train/47.txt\n",
      "copied doc 273 to brat-train/273.txt\n",
      "copied doc 637 to brat-train/637.txt\n",
      "copied doc 302 to brat-train/302.txt\n",
      "copied doc 291 to brat-train/291.txt\n",
      "copied doc 571 to brat-train/571.txt\n",
      "copied doc 452 to brat-train/452.txt\n",
      "copied doc 622 to brat-train/622.txt\n",
      "copied doc 218 to brat-train/218.txt\n",
      "copied doc 2 to brat-train/2.txt\n",
      "copied doc 1 to brat-train/1.txt\n",
      "copied doc 641 to brat-train/641.txt\n",
      "copied doc 92 to brat-train/92.txt\n",
      "copied doc 141 to brat-train/141.txt\n",
      "copied doc 426 to brat-train/426.txt\n",
      "copied doc 427 to brat-train/427.txt\n",
      "copied doc 186 to brat-train/186.txt\n",
      "copied doc 587 to brat-train/587.txt\n",
      "copied doc 388 to brat-train/388.txt\n",
      "copied doc 736 to brat-train/736.txt\n",
      "copied doc 626 to brat-train/626.txt\n",
      "copied doc 152 to brat-train/152.txt\n",
      "copied doc 42 to brat-train/42.txt\n",
      "copied doc 386 to brat-train/386.txt\n",
      "copied doc 577 to brat-train/577.txt\n",
      "copied doc 72 to brat-train/72.txt\n",
      "copied doc 756 to brat-train/756.txt\n",
      "copied doc 722 to brat-train/722.txt\n",
      "copied doc 791 to brat-train/791.txt\n",
      "copied doc 432 to brat-train/432.txt\n",
      "copied doc 143 to brat-train/143.txt\n",
      "copied doc 93 to brat-train/93.txt\n",
      "copied doc 433 to brat-train/433.txt\n",
      "copied doc 167 to brat-train/167.txt\n",
      "copied doc 197 to brat-train/197.txt\n",
      "copied doc 692 to brat-train/692.txt\n",
      "copied doc 236 to brat-train/236.txt\n",
      "copied doc 393 to brat-train/393.txt\n",
      "copied doc 313 to brat-train/313.txt\n",
      "copied doc 153 to brat-train/153.txt\n",
      "copied doc 496 to brat-train/496.txt\n",
      "copied doc 717 to brat-train/717.txt\n",
      "copied doc 481 to brat-train/481.txt\n",
      "copied doc 468 to brat-train/468.txt\n",
      "copied doc 396 to brat-train/396.txt\n",
      "copied doc 807 to brat-train/807.txt\n",
      "copied doc 122 to brat-train/122.txt\n",
      "copied doc 81 to brat-train/81.txt\n",
      "copied doc 178 to brat-train/178.txt\n",
      "copied doc 3 to brat-train/3.txt\n",
      "copied doc 591 to brat-train/591.txt\n",
      "copied doc 373 to brat-train/373.txt\n",
      "copied doc 557 to brat-train/557.txt\n",
      "copied doc 188 to brat-train/188.txt\n",
      "copied doc 43 to brat-train/43.txt\n",
      "copied doc 382 to brat-train/382.txt\n",
      "copied doc 532 to brat-train/532.txt\n",
      "copied doc 491 to brat-train/491.txt\n",
      "copied doc 582 to brat-train/582.txt\n",
      "copied doc 336 to brat-train/336.txt\n",
      "copied doc 362 to brat-train/362.txt\n",
      "copied doc 346 to brat-train/346.txt\n",
      "copied doc 721 to brat-train/721.txt\n",
      "copied doc 787 to brat-train/787.txt\n",
      "copied doc 417 to brat-train/417.txt\n",
      "copied doc 21 to brat-train/21.txt\n",
      "copied doc 567 to brat-train/567.txt\n",
      "copied doc 166 to brat-train/166.txt\n",
      "copied doc 23 to brat-train/23.txt\n",
      "copied doc 576 to brat-train/576.txt\n",
      "copied doc 107 to brat-train/107.txt\n",
      "copied doc 521 to brat-train/521.txt\n",
      "copied doc 367 to brat-train/367.txt\n",
      "copied doc 308 to brat-train/308.txt\n",
      "copied doc 511 to brat-train/511.txt\n",
      "copied doc 272 to brat-train/272.txt\n",
      "copied doc 786 to brat-train/786.txt\n",
      "copied doc 697 to brat-train/697.txt\n",
      "copied doc 387 to brat-train/387.txt\n",
      "copied doc 656 to brat-train/656.txt\n",
      "copied doc 51 to brat-train/51.txt\n",
      "copied doc 123 to brat-train/123.txt\n",
      "copied doc 463 to brat-train/463.txt\n",
      "copied doc 36 to brat-train/36.txt\n",
      "copied doc 482 to brat-train/482.txt\n",
      "copied doc 68 to brat-train/68.txt\n",
      "copied doc 332 to brat-train/332.txt\n",
      "copied doc 437 to brat-train/437.txt\n",
      "copied doc 247 to brat-train/247.txt\n",
      "copied doc 512 to brat-train/512.txt\n",
      "copied doc 681 to brat-train/681.txt\n",
      "copied doc 337 to brat-train/337.txt\n",
      "copied doc 162 to brat-train/162.txt\n",
      "copied doc 602 to brat-train/602.txt\n",
      "copied doc 348 to brat-train/348.txt\n",
      "copied doc 18 to brat-train/18.txt\n",
      "copied doc 87 to brat-train/87.txt\n",
      "copied doc 366 to brat-train/366.txt\n",
      "copied doc 311 to brat-train/311.txt\n",
      "copied doc 517 to brat-train/517.txt\n",
      "copied doc 357 to brat-train/357.txt\n",
      "copied doc 201 to brat-train/201.txt\n",
      "copied doc 28 to brat-train/28.txt\n",
      "copied doc 316 to brat-train/316.txt\n",
      "copied doc 438 to brat-train/438.txt\n",
      "copied doc 411 to brat-train/411.txt\n",
      "copied doc 168 to brat-train/168.txt\n",
      "copied doc 726 to brat-train/726.txt\n",
      "copied doc 701 to brat-train/701.txt\n",
      "copied doc 8 to brat-train/8.txt\n",
      "copied doc 711 to brat-train/711.txt\n",
      "copied doc 156 to brat-train/156.txt\n",
      "copied doc 502 to brat-train/502.txt\n",
      "copied doc 422 to brat-train/422.txt\n",
      "copied doc 647 to brat-train/647.txt\n",
      "copied doc 163 to brat-train/163.txt\n",
      "copied doc 16 to brat-train/16.txt\n",
      "copied doc 408 to brat-train/408.txt\n",
      "copied doc 38 to brat-train/38.txt\n",
      "copied doc 801 to brat-train/801.txt\n",
      "copied doc 212 to brat-train/212.txt\n",
      "copied doc 423 to brat-train/423.txt\n",
      "copied doc 472 to brat-train/472.txt\n",
      "copied doc 216 to brat-train/216.txt\n",
      "copied doc 126 to brat-train/126.txt\n",
      "copied doc 631 to brat-train/631.txt\n",
      "copied doc 546 to brat-train/546.txt\n",
      "copied doc 242 to brat-train/242.txt\n",
      "copied doc 777 to brat-train/777.txt\n",
      "copied doc 213 to brat-train/213.txt\n",
      "copied doc 271 to brat-train/271.txt\n",
      "copied doc 173 to brat-train/173.txt\n",
      "copied doc 473 to brat-train/473.txt\n",
      "copied doc 757 to brat-train/757.txt\n",
      "copied doc 248 to brat-train/248.txt\n",
      "copied doc 547 to brat-train/547.txt\n",
      "copied doc 203 to brat-train/203.txt\n",
      "copied doc 572 to brat-train/572.txt\n",
      "copied doc 116 to brat-train/116.txt\n",
      "copied doc 666 to brat-train/666.txt\n",
      "copied doc 182 to brat-train/182.txt\n",
      "copied doc 352 to brat-train/352.txt\n",
      "copied doc 407 to brat-train/407.txt\n",
      "copied doc 177 to brat-train/177.txt\n",
      "copied doc 797 to brat-train/797.txt\n",
      "copied doc 98 to brat-train/98.txt\n",
      "copying text for split test. source path /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml, dest path brat-test\n",
      "copied doc 442 to brat-test/442.txt\n",
      "copied doc 31 to brat-test/31.txt\n",
      "copied doc 233 to brat-test/233.txt\n",
      "copied doc 667 to brat-test/667.txt\n",
      "copied doc 333 to brat-test/333.txt\n",
      "copied doc 73 to brat-test/73.txt\n",
      "copied doc 283 to brat-test/283.txt\n",
      "copied doc 287 to brat-test/287.txt\n",
      "copied doc 767 to brat-test/767.txt\n",
      "copied doc 487 to brat-test/487.txt\n",
      "copied doc 61 to brat-test/61.txt\n",
      "copied doc 431 to brat-test/431.txt\n",
      "copied doc 227 to brat-test/227.txt\n",
      "copied doc 78 to brat-test/78.txt\n",
      "copied doc 312 to brat-test/312.txt\n",
      "copied doc 262 to brat-test/262.txt\n",
      "copied doc 556 to brat-test/556.txt\n",
      "copied doc 66 to brat-test/66.txt\n",
      "copied doc 196 to brat-test/196.txt\n",
      "copied doc 592 to brat-test/592.txt\n",
      "copied doc 282 to brat-test/282.txt\n",
      "copied doc 326 to brat-test/326.txt\n",
      "copied doc 737 to brat-test/737.txt\n",
      "copied doc 222 to brat-test/222.txt\n",
      "copied doc 83 to brat-test/83.txt\n",
      "copied doc 561 to brat-test/561.txt\n",
      "copied doc 118 to brat-test/118.txt\n",
      "copied doc 297 to brat-test/297.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copied doc 358 to brat-test/358.txt\n",
      "copied doc 402 to brat-test/402.txt\n",
      "copied doc 562 to brat-test/562.txt\n",
      "copied doc 607 to brat-test/607.txt\n",
      "copied doc 261 to brat-test/261.txt\n",
      "copied doc 361 to brat-test/361.txt\n",
      "copied doc 368 to brat-test/368.txt\n",
      "copied doc 128 to brat-test/128.txt\n",
      "copied doc 383 to brat-test/383.txt\n",
      "copied doc 32 to brat-test/32.txt\n",
      "copied doc 293 to brat-test/293.txt\n",
      "copied doc 542 to brat-test/542.txt\n",
      "copied doc 617 to brat-test/617.txt\n",
      "copied doc 581 to brat-test/581.txt\n",
      "copied doc 347 to brat-test/347.txt\n",
      "copied doc 806 to brat-test/806.txt\n",
      "copied doc 712 to brat-test/712.txt\n",
      "copied doc 221 to brat-test/221.txt\n",
      "copied doc 687 to brat-test/687.txt\n",
      "copied doc 416 to brat-test/416.txt\n",
      "copied doc 263 to brat-test/263.txt\n",
      "copied doc 132 to brat-test/132.txt\n",
      "copied doc 327 to brat-test/327.txt\n",
      "copied doc 657 to brat-test/657.txt\n",
      "copied doc 208 to brat-test/208.txt\n",
      "copied doc 138 to brat-test/138.txt\n",
      "copied doc 171 to brat-test/171.txt\n",
      "copied doc 238 to brat-test/238.txt\n",
      "copied doc 792 to brat-test/792.txt\n",
      "copied doc 253 to brat-test/253.txt\n",
      "copied doc 317 to brat-test/317.txt\n",
      "copied doc 441 to brat-test/441.txt\n",
      "copied doc 298 to brat-test/298.txt\n",
      "copied doc 137 to brat-test/137.txt\n",
      "copied doc 392 to brat-test/392.txt\n",
      "copied doc 516 to brat-test/516.txt\n",
      "copied doc 88 to brat-test/88.txt\n",
      "copied doc 397 to brat-test/397.txt\n",
      "copied doc 21 to brat-test/21.txt\n",
      "copied doc 752 to brat-test/752.txt\n",
      "copied doc 762 to brat-test/762.txt\n",
      "copied doc 781 to brat-test/781.txt\n",
      "copied doc 288 to brat-test/288.txt\n",
      "copied doc 207 to brat-test/207.txt\n",
      "copied doc 206 to brat-test/206.txt\n",
      "copied doc 812 to brat-test/812.txt\n",
      "copied doc 601 to brat-test/601.txt\n",
      "copied doc 652 to brat-test/652.txt\n",
      "copied doc 527 to brat-test/527.txt\n",
      "copied doc 371 to brat-test/371.txt\n",
      "copied doc 33 to brat-test/33.txt\n",
      "copied doc 53 to brat-test/53.txt\n",
      "copied doc 296 to brat-test/296.txt\n",
      "copied doc 108 to brat-test/108.txt\n",
      "copied doc 716 to brat-test/716.txt\n",
      "copied doc 71 to brat-test/71.txt\n",
      "copied doc 46 to brat-test/46.txt\n",
      "copied doc 223 to brat-test/223.txt\n",
      "copied doc 772 to brat-test/772.txt\n",
      "copied doc 257 to brat-test/257.txt\n",
      "copied doc 142 to brat-test/142.txt\n",
      "copied doc 277 to brat-test/277.txt\n",
      "copied doc 677 to brat-test/677.txt\n",
      "copied doc 796 to brat-test/796.txt\n",
      "copied doc 258 to brat-test/258.txt\n",
      "copied doc 727 to brat-test/727.txt\n",
      "copied doc 378 to brat-test/378.txt\n",
      "copied doc 103 to brat-test/103.txt\n",
      "copied doc 113 to brat-test/113.txt\n",
      "copied doc 447 to brat-test/447.txt\n",
      "copied doc 742 to brat-test/742.txt\n",
      "copied doc 127 to brat-test/127.txt\n",
      "copied doc 412 to brat-test/412.txt\n",
      "copied doc 276 to brat-test/276.txt\n",
      "copied doc 398 to brat-test/398.txt\n",
      "copied doc 101 to brat-test/101.txt\n",
      "copied doc 627 to brat-test/627.txt\n",
      "copied doc 802 to brat-test/802.txt\n",
      "copied doc 436 to brat-test/436.txt\n",
      "copied doc 552 to brat-test/552.txt\n",
      "copied doc 303 to brat-test/303.txt\n",
      "copied doc 446 to brat-test/446.txt\n",
      "copied doc 817 to brat-test/817.txt\n",
      "copied doc 157 to brat-test/157.txt\n",
      "copied doc 537 to brat-test/537.txt\n",
      "copied doc 342 to brat-test/342.txt\n",
      "copied doc 243 to brat-test/243.txt\n",
      "copied doc 391 to brat-test/391.txt\n",
      "copied doc 732 to brat-test/732.txt\n",
      "copied doc 466 to brat-test/466.txt\n",
      "copied doc 202 to brat-test/202.txt\n",
      "copied doc 131 to brat-test/131.txt\n"
     ]
    }
   ],
   "source": [
    "def copy_text(d_src, d_dest):\n",
    "    for key in d_src:\n",
    "        path_src = d_src[key]\n",
    "        path_dest = d_dest[key]\n",
    "        print(f\"copying text for split {key}. source path {path_src}, dest path {path_dest}\")\n",
    "        gen = path_src.glob(\"*.txt\")\n",
    "        if len(list(gen)) == 0:\n",
    "            raise RunTimeError(f\"empty directory! please make sure you have .txt file in {path_src}\")\n",
    "        for each in path_src.glob(\"*.txt\"):\n",
    "            fid = each.stem.split(\".\")[0]\n",
    "            new_file = path_dest / f\"{fid}.txt\"\n",
    "            shutil.copyfile(each, new_file)\n",
    "            print(f\"copied doc {fid} to {new_file}\")\n",
    "\n",
    "copy_text(in_paths, out_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed9ff19b",
   "metadata": {},
   "outputs": [],
   "source": [
    "BRAT_TEMP = \"T{}\\t{} {} {}\\t{}\"\n",
    "EVENTS = {'PROBLEM', 'TEST', 'TREATMENT', 'CLINICAL_DEPT', 'EVIDENTIAL', 'OCCURRENCE'}\n",
    "SPEC = {'&': 'AAMMPP'}\n",
    "\n",
    "def load_file(file):\n",
    "    with open(file, \"r\") as f:\n",
    "        cont = f.read()\n",
    "    return cont\n",
    "\n",
    "def write_to_file(data, file):\n",
    "    with open(file, \"w\") as f:\n",
    "        f.write(data)\n",
    "\n",
    "def xml2brat(p1, p2, verbose=False):\n",
    "    # offset need to -1 on the number\n",
    "    for each in p1.glob(\"*.xml\"):\n",
    "        brat_anns = []\n",
    "        idx = 1\n",
    "        print(f'file name is {each}')\n",
    "        ofn = p2 / (each.stem.split(\".\")[0] + \".ann\")\n",
    "        xml = load_file(each)\n",
    "        if verbose:\n",
    "            print(xml)\n",
    "        xml = xml.replace('&', 'AAMMPP')\n",
    "        tags = xmltodict.parse(xml)['ClinicalNarrativeTemporalAnnotation']['TAGS']\n",
    "        try:\n",
    "            for k, v in tags.items():\n",
    "                # only keep event tags\n",
    "                if k == 'EVENT':\n",
    "                    for d in v:\n",
    "                        typ = d['@type']\n",
    "                        if typ in EVENTS:\n",
    "                            s = int(d['@start']) - 1 # convert from 1-index to 0-index\n",
    "                            e = int(d['@end']) - 1\n",
    "                            txt = d['@text']\n",
    "                            brat_anns.append(BRAT_TEMP.format(idx, typ, s, e, txt))\n",
    "                            idx += 1\n",
    "        except:\n",
    "            print(xml)\n",
    "            print(tags)\n",
    "            assert(False)\n",
    "        ot = \"\\n\".join(brat_anns)\n",
    "        ot = ot.replace('AAMMPP', '&')\n",
    "        # break # added for debug\n",
    "        write_to_file(ot, ofn)\n",
    "\n",
    "def dataset_xml2brat(d_in, d_out):\n",
    "    for key in d_in:\n",
    "        print(f\"in path is {d_in[key]}, out path is {d_out[key]}\")\n",
    "        xml2brat(d_in[key], d_out[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "133a8e19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in path is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release, out path is brat-train\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/481.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/547.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/11.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/546.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/591.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/122.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/656.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/121.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/36.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/408.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/407.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/452.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/701.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/517.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/267.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/2.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/166.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/626.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/376.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/43.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/736.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/413.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/201.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/596.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/311.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/173.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/422.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/16.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/522.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/92.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/172.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/532.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/86.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/237.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/348.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/236.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/458.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/641.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/707.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/346.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/751.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/567.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/38.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/143.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/676.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/787.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/353.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/602.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/93.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/141.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/252.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/786.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/318.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/87.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/463.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/72.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/362.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/611.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/316.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/178.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/427.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/472.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/721.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/177.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/426.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/471.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/582.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/647.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/692.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/213.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/186.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/757.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/18.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/756.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/42.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/433.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/332.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/331.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/193.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/192.xml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/801.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/191.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/98.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/156.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/256.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/367.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/366.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/726.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/587.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/302.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/301.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/163.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/697.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/512.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/3.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/162.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/411.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/68.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/273.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/373.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/622.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/116.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/272.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/521.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/126.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/47.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/271.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/337.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/382.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/631.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/336.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/492.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/26.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/807.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/197.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/491.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/557.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/123.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/96.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/81.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/666.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/23.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/777.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/343.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/242.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/776.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/308.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/1.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/307.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/352.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/168.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/351.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/417.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/462.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/711.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/167.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/572.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/8.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/51.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/526.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/388.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/571.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/637.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/682.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/203.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/387.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/636.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/681.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/747.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/313.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/386.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/497.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/212.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/791.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/423.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/396.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/496.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/321.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/432.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/182.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/248.xml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/393.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/642.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/28.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/247.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/541.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/291.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/357.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/6.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/468.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/717.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/356.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/577.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/576.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/153.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/502.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/107.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/152.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/218.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/797.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/612.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/151.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/511.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/473.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/722.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/216.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/438.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/188.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/437.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-07-15.original-annotation.release/482.xml\n",
      "in path is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml, out path is brat-test\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/436.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/298.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/592.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/113.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/297.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/657.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/223.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/196.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/767.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/333.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/88.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/342.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/802.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/157.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/562.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/516.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/378.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/561.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/627.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/772.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/737.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/303.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/276.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/487.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/202.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/781.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/312.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/128.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/127.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/238.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/283.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/383.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/171.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/282.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/71.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/742.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/347.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/392.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/391.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/817.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/677.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/243.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/142.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/208.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/207.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/712.xml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/206.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/317.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/66.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/361.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/288.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/537.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/103.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/287.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/398.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/581.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/792.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/397.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/296.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/222.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/221.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/73.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/21.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/442.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/258.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/441.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/552.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/652.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/257.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/46.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/368.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/617.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/762.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/727.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/118.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/412.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/263.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/53.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/227.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/732.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/447.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/78.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/446.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/806.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/556.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/667.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/233.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/132.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/131.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/601.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/812.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/33.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/416.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/527.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/277.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/101.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/61.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/138.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/137.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/431.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/293.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/542.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/358.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/607.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/752.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/83.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/31.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/716.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/466.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/108.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/687.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/402.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/253.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/262.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/796.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/261.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/327.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/326.xml\n",
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/371.xml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file name is /gpfs/data/oermannlab/public_data/i2b2_2012/2012-08-08.test-data.event-timex-groundtruth/xml/32.xml\n"
     ]
    }
   ],
   "source": [
    "dataset_xml2brat(in_paths, out_paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "526c51f3",
   "metadata": {},
   "source": [
    "## Step 2: convert BRAT to BIO\n",
    "Code from Xi Yang (University of Florida)\n",
    "\n",
    "Modified by Lavender Jiang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c53a0c26",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from shutil import copyfile\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3b0cedbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacy.pipeline.sentencizer.Sentencizer at 0x7f4277c80900>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from spacy.lang.en import English\n",
    "nlp = English()\n",
    "sentencizer = nlp.create_pipe(\"sentencizer\")\n",
    "nlp.add_pipe(\"sentencizer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "75382da9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "making dir...\n"
     ]
    }
   ],
   "source": [
    "# folder for brat files\n",
    "infiles = {\"train\": \"brat-train\",\n",
    "           \"test\": \"brat-test\"}\n",
    "bio_out = {\"train\": \"bio-train\",\n",
    "           \"test\": \"bio-test\"}\n",
    "\n",
    "def get_ann_files(d_path, splits):\n",
    "    res = {}\n",
    "    for key in splits:\n",
    "        in_path = d_path[key]\n",
    "        inputfiles = set()\n",
    "        for f in os.listdir(in_path):\n",
    "            if f.endswith('.ann'):\n",
    "                inputfiles.add(f.split('.')[0].split('_')[0])\n",
    "        res[key] = inputfiles\n",
    "    return res\n",
    "\n",
    "ann_files_d = get_ann_files(infiles, [\"train\", \"test\"])\n",
    "make_dir(bio_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1e0f095b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of entities to retain\n",
    "select_types = ['PROBLEM', 'TEST', 'TREATMENT', 'CLINICAL_DEPT', 'EVIDENTIAL', 'OCCURRENCE']\n",
    "\n",
    "# convert Brat format into BIO format\n",
    "# function for getting entity annotations from the annotation file\n",
    "def get_annotation_entities(ann_file, select_types=None):\n",
    "    entities = []\n",
    "    with open(ann_file, \"r\", encoding=\"utf-8\") as f:\n",
    "        for line in f:\n",
    "            if line.startswith('T'):\n",
    "                term = line.strip().split('\\t')[1].split()\n",
    "                if (select_types != None) and (term[0] not in select_types): continue\n",
    "                if int(term[-1]) <= int(term[1]): continue\n",
    "                entities.append((int(term[1]), int(term[-1]), term[0]))\n",
    "    return sorted(entities, key=lambda x: (x[0], x[1]))\n",
    "\n",
    "# function for handling overlap by keeping the entity with largest text span\n",
    "def remove_overlap_entities(sorted_entities):\n",
    "    keep_entities = []\n",
    "    for idx, entity in enumerate(sorted_entities):\n",
    "        if idx == 0:\n",
    "            keep_entities.append(entity)\n",
    "            last_keep = entity\n",
    "            continue\n",
    "        if entity[0] < last_keep[1]:\n",
    "            if entity[1]-entity[0] > last_keep[1]-last_keep[0]:\n",
    "                last_keep = entity\n",
    "                keep_entities[-1] = last_keep\n",
    "        elif entity[0] == last_keep[1]:\n",
    "            last_keep = (last_keep[0], entity[1], last_keep[-1])\n",
    "            keep_entities[-1] = last_keep\n",
    "        else:\n",
    "            last_keep = entity\n",
    "            keep_entities.append(entity)\n",
    "    return keep_entities\n",
    "\n",
    "# inverse index of entity annotations\n",
    "def entity_dictionary(keep_entities, txt_file):\n",
    "    #print(f\"txt_file is {txt_file}\")\n",
    "    file_name = os.path.basename(txt_file)\n",
    "    #print(f\"file name is {file_name}\")\n",
    "    f_ann = {}\n",
    "    with open(txt_file, \"r\", encoding=\"utf-8\") as f:\n",
    "        text = f.readlines()\n",
    "        text = ''.join([i for i in text])\n",
    "    for entity in keep_entities:\n",
    "        entity_text = text[entity[0]:entity[1]]\n",
    "        doc = nlp(entity_text)\n",
    "        token_starts = [(i, doc[i:].start_char) for i in range(len(doc))]\n",
    "        term_type = entity[-1]\n",
    "        term_offset = entity[0]\n",
    "        for i, token in enumerate(doc):\n",
    "            ann_offset = token_starts[i][1]+term_offset\n",
    "            if ann_offset not in f_ann:\n",
    "                f_ann[ann_offset] = [i, token.text, term_type]\n",
    "    return f_ann"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "adfe5ff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def brat2bio(inputfiles, inputpath, outputpath, verbose=False):\n",
    "    # Brat -> BIO format conversion\n",
    "    print(f\"converting brat2bio {inputfiles}\")\n",
    "    for infile in inputfiles:\n",
    "        file = f\"{infile}\"\n",
    "        ann_file = f\"{inputpath}/{file}.ann\"\n",
    "        txt_file = f\"{inputpath}/{file}.txt\"\n",
    "        out_file = f\"{outputpath}/{file}.bio.txt\"\n",
    "        if verbose:\n",
    "            print(f'infile is {infile}')\n",
    "            print(f'outfile is {out_file}')\n",
    "        sorted_entities = get_annotation_entities(ann_file, select_types)\n",
    "        keep_entities = remove_overlap_entities(sorted_entities)\n",
    "        f_ann = entity_dictionary(keep_entities, txt_file)\n",
    "        \n",
    "        with open(out_file, \"w\", encoding=\"utf-8\") as f_out:\n",
    "            with open(txt_file, \"r\", encoding=\"utf-8\") as f:\n",
    "                sent_offset = 0\n",
    "                prev_label = \"O\"\n",
    "                for line in f:\n",
    "                    if '⁄' in line:\n",
    "                        line = line.replace('⁄', '/') # replace non unicode characters\n",
    "                    doc = nlp(line.strip())\n",
    "                    # list of tuples, first value is token index, second value is char idx\n",
    "                    token_starts = [(i, doc[i:].start_char) for i in range(len(doc))]\n",
    "                    for token in doc:\n",
    "                        token_sent_offset = token_starts[token.i][1] # sentence level local index\n",
    "                        token_doc_offset = token_starts[token.i][1] + sent_offset # document level global index\n",
    "                        if token_doc_offset in f_ann:\n",
    "                            if prev_label == \"O\" or not (prev_label.split(\"-\")[1] == f_ann[token_doc_offset][2]): # or prev_label == f\"I-{f_ann[token_doc_offset][2]}\" or prev_label == f\"B-{f_ann[token_doc_offset][2]}\":#f_ann[token_doc_offset][0] == 0: # changed edge case to I-tag according to wikipedia\n",
    "                                label = f\"B-{f_ann[token_doc_offset][2]}\"\n",
    "                            else:\n",
    "                                label = f\"I-{f_ann[token_doc_offset][2]}\"\n",
    "                            if not (f_ann[token_doc_offset][1] == token.text_with_ws.rstrip()):\n",
    "                                print('{} does not match {}'.format(f_ann[token_doc_offset][1], token.text_with_ws.rstrip()))\n",
    "                                assert(False)\n",
    "                        else:\n",
    "                            label = f\"O\"\n",
    "                        prev_label = label # update prev_label\n",
    "                        f_out.write(f\"{token.text} {token_sent_offset} {token_sent_offset+len(token.text)} {token_doc_offset} {token_doc_offset+len(token.text)} {label}\\n\")\n",
    "                    f_out.write('\\n')\n",
    "                    sent_offset += (len(line))  \n",
    "\n",
    "def brat2bio_dict(ann_files_d, infiles_d, bio_out_d):\n",
    "    for key in ann_files_d:\n",
    "        ann_file = ann_files_d[key]\n",
    "        infiles = infiles_d[key]\n",
    "        bio_out = bio_out_d[key]\n",
    "        brat2bio(ann_file, infiles, bio_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d9a6d149",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "converting brat2bio {'273', '247', '308', '722', '252', '366', '422', '676', '96', '462', '721', '42', '577', '302', '472', '726', '346', '337', '321', '496', '491', '72', '313', '291', '426', '386', '116', '212', '152', '517', '612', '756', '36', '791', '236', '458', '123', '191', '681', '272', '482', '173', '156', '237', '201', '332', '86', '666', '168', '502', '107', '411', '1', '582', '432', '193', '407', '433', '717', '576', '636', '427', '417', '188', '786', '167', '126', '697', '512', '248', '348', '622', '777', '707', '166', '186', '522', '481', '6', '16', '757', '637', '492', '311', '367', '801', '642', '471', '256', '438', '192', '611', '682', '271', '357', '307', '807', '81', '388', '413', '547', '38', '242', '647', '92', '23', '2', '596', '318', '177', '316', '736', '151', '93', '47', '343', '203', '396', '182', '51', '8', '352', '373', '121', '153', '218', '452', '546', '701', '656', '141', '98', '521', '797', '497', '331', '26', '336', '382', '68', '163', '353', '437', '387', '747', '43', '18', '11', '541', '532', '356', '776', '122', '473', '87', '162', '216', '28', '172', '572', '463', '631', '626', '591', '751', '178', '641', '267', '143', '393', '468', '692', '362', '511', '526', '587', '423', '408', '3', '602', '351', '571', '376', '787', '711', '567', '557', '213', '197', '301'}\n",
      "converting brat2bio {'416', '287', '627', '282', '562', '297', '326', '303', '667', '727', '233', '196', '581', '617', '66', '441', '157', '298', '592', '127', '487', '206', '262', '556', '113', '137', '802', '397', '392', '71', '361', '53', '258', '132', '446', '276', '402', '716', '333', '238', '138', '732', '466', '142', '442', '31', '812', '371', '103', '431', '358', '243', '412', '742', '131', '767', '312', '516', '261', '752', '78', '383', '227', '762', '652', '561', '21', '368', '552', '61', '83', '772', '436', '542', '128', '347', '378', '221', '293', '607', '398', '296', '712', '253', '327', '208', '342', '391', '223', '88', '73', '317', '601', '657', '806', '288', '277', '537', '817', '108', '257', '32', '207', '33', '46', '283', '677', '118', '781', '792', '447', '101', '527', '202', '222', '687', '737', '263', '171', '796'}\n"
     ]
    }
   ],
   "source": [
    "brat2bio_dict(ann_files_d, infiles, bio_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb2f900c",
   "metadata": {},
   "source": [
    "## Step 3: Combine files to dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "094d8ab6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size 133, val size 57, test size 120\n"
     ]
    }
   ],
   "source": [
    "import random \n",
    "\n",
    "random_state = 13\n",
    "train_ids, dev_ids = train_test_split(list(ann_files_d['train']), train_size=0.7, random_state=random_state, shuffle=True)\n",
    "test_ids = list(ann_files_d['test'])\n",
    "random.Random(random_state).shuffle(test_ids)\n",
    "print(f\"train size {len(train_ids)}, val size {len(dev_ids)}, test size {len(test_ids)}\")\n",
    "i2b2_datasets = {\"train\":train_ids, \"dev\":dev_ids, \"test\":test_ids}\n",
    "json.dump(i2b2_datasets, open(\"i2b2_2012_datasets.json\", \"w\", encoding=\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "9094614d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_if_nonexist(dir_s):\n",
    "    if not os.path.exists(dir_s):\n",
    "        os.makedirs(dir_s)\n",
    "\n",
    "data_dir = \"dataset\"\n",
    "make_if_nonexist(data_dir)\n",
    "\n",
    "# Merge BIO format train, validation and test datasets\n",
    "for split in [\"train\", \"dev\", \"test\"]:\n",
    "    if split in [\"train\", \"dev\"]:\n",
    "        outputpath = bio_out[\"train\"]\n",
    "    else:\n",
    "        outputpath = bio_out[\"test\"]\n",
    "    split_dir = os.path.join(data_dir, split)\n",
    "    make_if_nonexist(split_dir)\n",
    "    with open(os.path.join(data_dir, f\"{split}.txt\"), \"w\", encoding=\"utf-8\") as f:\n",
    "        for fid in i2b2_datasets[split]:\n",
    "            copyfile(f\"{outputpath}/{fid}.bio.txt\", os.path.join(split_dir,f\"{fid}.bio.txt\"))\n",
    "            with open(f\"{outputpath}/{fid}.bio.txt\", \"r\", encoding=\"utf-8\") as fr:\n",
    "                txt = fr.read().strip()\n",
    "                if txt != '':\n",
    "                    f.write(txt)\n",
    "                    f.write(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b74639c",
   "metadata": {},
   "source": [
    "## Step 4: Bio 2 Nemo\n",
    "Code from nVidia NeMo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2627a0c7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-05-06 13:10:54 bio2nemo:119] Processing dataset/train.txt\n",
      "[NeMo I 2022-05-06 13:10:54 bio2nemo:124] Processing of the dataset/train.txt is complete\n"
     ]
    }
   ],
   "source": [
    "!python bio2nemo.py --data_file dataset/train.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2a79f7e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-05-06 13:11:51 bio2nemo:119] Processing dataset/dev.txt\n",
      "[NeMo I 2022-05-06 13:11:51 bio2nemo:124] Processing of the dataset/dev.txt is complete\n"
     ]
    }
   ],
   "source": [
    "!python bio2nemo.py --data_file dataset/dev.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75d31050",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NeMo I 2022-05-06 13:11:57 bio2nemo:119] Processing dataset/test.txt\n",
      "[NeMo I 2022-05-06 13:11:58 bio2nemo:124] Processing of the dataset/test.txt is complete\n"
     ]
    }
   ],
   "source": [
    "!python bio2nemo.py --data_file dataset/test.txt"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-nemo] *",
   "language": "python",
   "name": "conda-env-.conda-nemo-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}