parse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "superior-mustang",
   "metadata": {},
   "outputs": [],
   "source": [
    "%config Completer.use_jedi = False\n",
    "import glob\n",
    "import os\n",
    "import re\n",
    "import pandas\n",
    "import numpy\n",
    "import sqlite3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "clinical-wayne",
   "metadata": {},
   "outputs": [],
   "source": [
    "dbfile=\"parsedata.db\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "special-disposal",
   "metadata": {},
   "outputs": [],
   "source": [
    "inpfiles=glob.glob(r\"examples\\*.inp\")\n",
    "inpfiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "humanitarian-curtis",
   "metadata": {},
   "outputs": [],
   "source": [
    "inpfile=inpfiles[2]\n",
    "inpfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "french-layer",
   "metadata": {},
   "outputs": [],
   "source": [
    "basename,_=os.path.splitext(os.path.basename(inpfile))\n",
    "basename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hazardous-proxy",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(inpfile) as f:\n",
    "    txt=f.read()\n",
    "len(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "certain-investment",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,line in enumerate(txt.splitlines()):\n",
    "    print(i,line)\n",
    "    if i > 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cardiovascular-works",
   "metadata": {},
   "outputs": [],
   "source": [
    "txt=\"\"\n",
    "with open(inpfile) as f:\n",
    "    for line in f:\n",
    "        l = line[:80]\n",
    "        l = re.sub(r\"\\$.*$\",\"\",re.sub(r\"\\$.*?\\$\",\"\",l)).strip()\n",
    "        if len(l) > 0:\n",
    "            txt += l + \"\\n\"\n",
    "len(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "elementary-beijing",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,line in enumerate(txt.splitlines()):\n",
    "    print(i,line)\n",
    "    if i > 100:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "disabled-arctic",
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in re.finditer(\"^.*?\\.\\.\",txt,re.DOTALL):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "special-contest",
   "metadata": {},
   "outputs": [],
   "source": [
    "m.group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "needed-ground",
   "metadata": {},
   "outputs": [],
   "source": [
    "command_list=[]\n",
    "arg_list=[]\n",
    "\n",
    "for i,m1 in enumerate(re.finditer(r\"\\s*(.*?)\\s*\\.\\.\\s*\", txt, re.DOTALL)):\n",
    "        \n",
    "    g1=m1.group(1)\n",
    "    print(\"==========\")\n",
    "    print(i)\n",
    "    print(g1)\n",
    "    print(\"==========\")\n",
    "    \n",
    "    m2=re.match(r\"(?P<uname>.*?)\\s*=\\s*(?P<tcmd>.*)|(?P<icmd>.*)\",g1)\n",
    "    kind = m2.lastgroup\n",
    "    cmd,uname=None,None\n",
    "    if kind == 'icmd':\n",
    "        cmd=m2.group('icmd')\n",
    "        print(\"Command: \", cmd)\n",
    "        print(\"Command takes no U-NAME\")\n",
    "    elif kind == 'tcmd':\n",
    "        cmd=m2.group('tcmd')\n",
    "        uname0=m2.group('uname')\n",
    "        mu0=re.match(r'\"(?P<quoted>.*)\"|(?P<unquoted>.*)',uname0)\n",
    "        uname=mu0.group(mu0.lastgroup)\n",
    "        print(\"Command:\", cmd)\n",
    "        print(\"U-NAME: \", uname)\n",
    "    if len(cmd) == 0:\n",
    "        cmd = \"$noop\"\n",
    "    command_list.append((i,cmd,uname))\n",
    "    tail2=g1[m2.end():]\n",
    "    \n",
    "    j,m3=-1,None\n",
    "    for j,m3 in enumerate(re.finditer(r'\\s*(.+?)\\s*=\\s*(\\((?:(?:\\s*\"[^\"]+?\"|\\s*[^\\s\\)]+?)[,\\s]*)+\\)|(?:.*))\\s*', tail2)):\n",
    "        key,val=m3.group(1),m3.group(2)\n",
    "        arg_list.append((i,j,key,val))\n",
    "        print(f\"  {j:2}\",key)\n",
    "        print(f\"    \",val)\n",
    "    tail3 = tail2[m3.end():] if j>=0 else tail2\n",
    "    if len(tail3) > 0:\n",
    "        arg_list.append((i,j+1,\"$tail\",tail3))\n",
    "    print(\"Tail length: \",len(tail3))\n",
    "    ignore_tail = (len(tail3) == 0) \\\n",
    "        or cmd.startswith(\"RUN-PERIOD\") \\\n",
    "        or cmd.startswith(\"SCHEDULE\")\\\n",
    "        or tail3.startswith(\"COEF \")\n",
    "    \n",
    "    print()\n",
    "    #if i >= 90:\n",
    "    if not ignore_tail:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "advance-keyboard",
   "metadata": {},
   "outputs": [],
   "source": [
    "command_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "affiliated-defendant",
   "metadata": {},
   "outputs": [],
   "source": [
    "arg_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "subject-commitment",
   "metadata": {},
   "outputs": [],
   "source": [
    "cl=pandas.DataFrame(command_list,columns=['i','cmd','uname']).set_index('i')\n",
    "cl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "million-status",
   "metadata": {},
   "outputs": [],
   "source": [
    "al=pandas.DataFrame(arg_list,columns=['i','j','key','val']).set_index(['i','j'])\n",
    "al"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "necessary-nickel",
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.Connection(dbfile)\n",
    "try:\n",
    "    cl.to_sql(\"command_list\",conn)\n",
    "    al.to_sql(\"arg_list\",conn)\n",
    "finally:\n",
    "    conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "welcome-speed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abstract-chaos",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amber-bleeding",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "marked-potato",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "certain-smoke",
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.Connection(dbfile)\n",
    "try:\n",
    "    df1=pandas.read_sql(\"\"\"select \"i\", \"cmd\", \"uname\", \"j\", \"key\", \"val\" FROM \"arg_extended_refg_network\";\"\"\",conn)\n",
    "finally:\n",
    "    conn.close()\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "downtown-fifty",
   "metadata": {},
   "outputs": [],
   "source": [
    "edges=[]\n",
    "for row in df1.itertuples():\n",
    "    uname2=None\n",
    "    m1=re.match(r'\"(?P<quoted>.*)\"|(?P<unquoted>.*)',row.uname)\n",
    "    uname2=m1.group(m1.lastgroup)\n",
    "    \n",
    "    val2=None\n",
    "    m2=re.match(r'\"(?P<quoted>.*)\"|\\(\\s*(?P<list>.+?)\\s*\\)|(?P<unquoted>[^\\(\\)]*)',row.val)\n",
    "    val2=m2.group(m2.lastgroup)\n",
    "    \n",
    "    print(f'{row.Index} \"{uname2}\" -> \"{val2}\"')\n",
    "    edges.append((uname2, val2))\n",
    "    #if row.Index == 15:\n",
    "    #    break\n",
    "el=pandas.DataFrame(edges,columns=['uname','val'])\n",
    "el"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "consolidated-spectrum",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_value(val):\n",
    "    m=re.match(r'\"(?P<quoted>.*)\"|\\(\\s*(?P<list>.+?)\\s*\\)|(?P<unquoted>[^\\(\\)]*)',val)\n",
    "    return m.group(m.lastgroup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "worth-making",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['val'].apply(clean_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adopted-creativity",
   "metadata": {},
   "outputs": [],
   "source": [
    "m1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "disabled-placement",
   "metadata": {},
   "outputs": [],
   "source": [
    "m1.lastgroup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cross-newark",
   "metadata": {},
   "outputs": [],
   "source": [
    "m2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "plain-curtis",
   "metadata": {},
   "outputs": [],
   "source": [
    "m2.lastgroup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "swiss-preservation",
   "metadata": {},
   "outputs": [],
   "source": [
    "el['uname']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "uniform-relay",
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes=numpy.union1d(el.uname,el.val)\n",
    "nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "maritime-creativity",
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "valid-player",
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in nodes:\n",
    "    if n != n.strip():\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "welcome-fundamental",
   "metadata": {},
   "outputs": [],
   "source": [
    "n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "located-ballot",
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.Connection(dbfile)\n",
    "try:\n",
    "    nodes2=pandas.read_sql(\"\"\"select \"cmd\", \"uname\", \"shape\", \"color\" FROM \"refg_nodes\";\"\"\",conn)\n",
    "finally:\n",
    "    conn.close()\n",
    "nodes2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "extra-delaware",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "young-words",
   "metadata": {},
   "outputs": [],
   "source": [
    "import graphviz\n",
    "print(graphviz.ENGINES)\n",
    "print(graphviz.FORMATS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "labeled-corrections",
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "\n",
    "dot = Digraph(comment=basename,engine='dot',format='pdf')\n",
    "# Digraph(name='pet-shop', node_attr={'shape': 'plaintext'})\n",
    "\n",
    "for row in nodes2.itertuples():\n",
    "    #dot.attr('node', shape=row.shape)\n",
    "    dot.node(row.uname, shape=row.shape, fillcolor=row.color, style=\"filled\")\n",
    "for row in el.itertuples():\n",
    "    dot.edge(row.uname,row.val)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "invisible-hospital",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "noble-quick",
   "metadata": {},
   "outputs": [],
   "source": [
    "dot.format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "noted-blond",
   "metadata": {},
   "outputs": [],
   "source": [
    "dot.render(f'output/{basename}.gv', view=True)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "surprising-latvia",
   "metadata": {},
   "outputs": [],
   "source": [
    "?pandas.read_sql"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "digital-ultimate",
   "metadata": {},
   "outputs": [],
   "source": [
    "def datatable_for_command(cmd):\n",
    "    conn = sqlite3.Connection(dbfile)\n",
    "    try:\n",
    "        df2=pandas.read_sql(\"\"\"select \"uname\", \"key\", \"val\" FROM \"arg_extended\" where \"cmd\" == ?;\"\"\",\n",
    "                            conn,\n",
    "                            index_col=['uname','key'],\n",
    "                            params=(cmd,))\n",
    "    finally:\n",
    "        conn.close()\n",
    "    df3=df2.unstack()\n",
    "    df3.columns=df3.columns.droplevel(0)\n",
    "    return df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "north-mainland",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3=datatable_for_command(\"SPACE\")\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "homeless-mirror",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3=datatable_for_command(\"PROCESS-LOAD\")\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fallen-opportunity",
   "metadata": {},
   "outputs": [],
   "source": [
    "?object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "concerned-guinea",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3.astype(df3.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pleasant-anatomy",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt=df3.dtypes\n",
    "for c in df3:\n",
    "    print(c)\n",
    "    try:\n",
    "        df3[c].astype(float)\n",
    "        dt[c] = 'float'\n",
    "        print(\"OK to float\")\n",
    "    except:\n",
    "        print(\"Not a float!\")\n",
    "dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lucky-progressive",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3=df3.astype(dt)\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "southwest-swaziland",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3.to_excel(f\"output/{basename}.xlsx\",sheet_name=\"PROCESS-LOAD\",freeze_panes=(1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "great-quality",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3.to_clipboard()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "smooth-assessment",
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://www.graphviz.org/doc/info/shapes.html#html\n",
    "# let codes=$(\"body > div > div > div > main > div > figure code\")\n",
    "# codes.map(function(x,i){return i.innerText;})\n",
    "\n",
    "shapeopts=[\"box\", \"polygon\", \"ellipse\", \"oval\", \"circle\", \"point\", \"egg\", \"triangle\", \"plaintext\", \"plain\", \"diamond\",\n",
    " \"trapezium\", \"parallelogram\", \"house\", \"pentagon\", \"hexagon\", \"septagon\", \"octagon\", \"doublecircle\",\n",
    " \"doubleoctagon\", \"tripleoctagon\", \"invtriangle\", \"invtrapezium\", \"invhouse\", \"Mdiamond\", \"Msquare\",\n",
    " \"Mcircle\", \"rect\", \"rectangle\", \"square\", \"star\", \"none\", \"underline\", \"cylinder\", \"note\", \"tab\",\n",
    " \"folder\", \"box3d\", \"component\", \"promoter\", \"cds\", \"terminator\", \"utr\", \"primersite\", \"restrictionsite\",\n",
    " \"fivepoverhang\", \"threepoverhang\", \"noverhang\", \"assembly\", \"signature\", \"insulator\", \"ribosite\",\n",
    " \"rnastab\", \"proteasesite\", \"proteinstab\", \"rpromoter\", \"rarrow\", \"larrow\", \"lpromoter\"]\n",
    "len(shapeopts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "accurate-potential",
   "metadata": {},
   "outputs": [],
   "source": [
    "?df3.to_excel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "several-colors",
   "metadata": {},
   "outputs": [],
   "source": [
    "?conn.executescript"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "roman-sandwich",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}