old_french_rnntagger.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "old_french_rnntagger.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "ykw77chJ3Xng",
        "tnwzyJXdI9fU"
      ],
      "mount_file_id": "1P6ebzXQ5VVHftN_W-7G-ZCFp9mpWtqBY",
      "authorship_tag": "ABX9TyMt7JKxPALShZyfXeo7n46G",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CristinaGHolgado/old-french-with-pie-rnntagger/blob/master/old_french_rnntagger.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j7V_rJ0zM9kd"
      },
      "source": [
        "# **RNNTagger (tagger)**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z0xgm8PONHgL"
      },
      "source": [
        "### **Log in to Google Drive to access RNNTagger**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OYljOBJS7FOi"
      },
      "source": [
        "1. Upload **RNNTagger** (tagger) unziped folder into your GDrive.\n",
        "  (https://www.cis.uni-muenchen.de/~schmid/tools/RNNTagger/data/RNNTagger-1.2.1.zip)\n",
        "2. Upload the files to be tagged into a folder inside the **RNNTagger/** folder.\n",
        "3. Set path to GDrive **RNNTagger/** folder.\n",
        "4. Mount to your gdrive in GColab."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TwsT4wTTSQwj",
        "outputId": "df16bb89-01ed-40bd-be9b-dc52d96416b4"
      },
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "\n",
        "rnntagger_path = 'gdrive/My Drive/RNNTagger(tagger)'\n",
        "drive.mount('/content/gdrive')\n",
        "os.chdir(rnntagger_path)\n",
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n",
            "/content/gdrive/My Drive/RNNTagger(tagger)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dfkVNgaHuKXT"
      },
      "source": [
        "* Check PyTorch&CUDA (required)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jN55jnMQOMmo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0ccfc633-467b-4fee-b12b-d1634ca2d94f"
      },
      "source": [
        "import torch\n",
        "print(f\"Torch version : {torch.__version__}\")\n",
        "print(f\"Device name : {torch.cuda.get_device_name(0)}\")\n",
        "print(f\"Current path : {os.getcwd()}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Torch version : 1.7.0+cu101\n",
            "Device name : Tesla T4\n",
            "Current path : /content/gdrive/My Drive/RNNTagger(tagger)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jk7bA84Uwg32"
      },
      "source": [
        "### **Annotate input corpus**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KgPGElmluqgc"
      },
      "source": [
        "* Set permissions to *lemma-lookup.pl*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CMzyxgOwykz5"
      },
      "source": [
        "!chmod 755 -R ./scripts/lemma-lookup.pl"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d_AP-agZc5NN"
      },
      "source": [
        "from IPython.display import display, HTML\n",
        "import pandas as pd\n",
        "import subprocess\n",
        "import glob\n",
        "import csv\n",
        "import os, sys\n",
        "\n",
        "def getTokens():\n",
        "  \"\"\"Save tokens column for each input corpus file\n",
        "  \"\"\"\n",
        "  for filename in all_files:\n",
        "      df = pd.read_csv(filename, sep=\"\\t\", encoding=\"utf8\", names=[\"token\",\"nan_lemme\",\"nan_tag\",\"src\"], quoting=csv.QUOTE_NONE)\n",
        "      display(HTML(df.head().to_html()))\n",
        "      tokenscol = df[\"token\"]\n",
        "      name = filename.replace(files_extension,\"_tokens.csv\")\n",
        "      tokenscol.to_csv(name, encoding=\"utf8\", sep=\"\\t\", index=False, header=None)\n",
        "      print(filename,\"\\n\")\n",
        "\n",
        "def tagFiles():\n",
        "  \"\"\"Tag tokens files\n",
        "  \"\"\"\n",
        "  all_files = glob.glob(path_tsv + \"/*_tokens.csv\")\n",
        "  print(f\"Nombre de fichiers à étiquetter : {len(all_files)}*n\")\n",
        "  if not os.path.exists('tagged_files'):\n",
        "    os.makedirs('tagged_files')\n",
        "  nbFile = 0\n",
        "  for file in all_files:\n",
        "    file = file.replace(\"/\",\"//\")\n",
        "    output_name = file.split(\"//\")[-1].replace(\"tokens.csv\",\"_tagged(rnntagger).csv\")\n",
        "    command = \"bash cmd/rnn-tagger-old-french.sh '\"+file+\"' > '\"+\"tagged_files//\"+output_name+\"'\"\n",
        "    nbFile += 1\n",
        "    print(nbFile)\n",
        "    print(command)\n",
        "    !{command} "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Vipw-7Pz_wb"
      },
      "source": [
        "* Set path to the corpus folder\n",
        "* Set files extension"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OYQMJ5JXz9HI",
        "outputId": "1f1874af-ed01-4578-8d5f-d9bfe1c2d111"
      },
      "source": [
        "path_tsv = 'tsv_files' # path to folder containing tsv files\n",
        "files_extension = \".tsv\"\n",
        "\n",
        "all_files = glob.glob(path_tsv + \"/*\"+ files_extension)\n",
        "print(\"Total files : \", len(all_files))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Nombre de fichiers :  3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N_U__MYHmOoa"
      },
      "source": [
        "getTokens()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R7ogwFZJ2qJj",
        "outputId": "ab5e423a-d221-474b-e69a-8aecf461a077"
      },
      "source": [
        "tagFiles()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Nombre de fichiers à étiquetter : 12*n\n",
            "1\n",
            "bash cmd/rnn-tagger-old-french.sh 'tsv_files//thebes2_tokens.csv' > 'tagged_files//thebes2__tagged(rnntagger).csv'\n",
            "2\n",
            "bash cmd/rnn-tagger-old-french.sh 'tsv_files//CommPsia1a_tokens.csv' > 'tagged_files//CommPsia1a__tagged(rnntagger).csv'\n",
            "3\n",
            "bash cmd/rnn-tagger-old-french.sh 'tsv_files//strasbBfm_tokens.csv' > 'tagged_files//strasbBfm__tagged(rnntagger).csv'\n",
            "4\n",
            "bash cmd/rnn-tagger-old-french.sh 'tsv_files//saintre_tokens.csv' > 'tagged_files//saintre__tagged(rnntagger).csv'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LksJSgX532cJ"
      },
      "source": [
        "### If tagging a single file, run :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TY1W4Yqi1iwO",
        "outputId": "7535abdd-c602-43c7-80f5-5dd944180b35"
      },
      "source": [
        "%%shell\n",
        "bash cmd/rnn-tagger-old-french.sh 'tsv_files//tokens//Berin1_tokens.csv' > 'Berin1__tagged(rnntagger).csv'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zrOee8W36XbW"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ykw77chJ3Xng"
      },
      "source": [
        "# **RNNTagger (trainer)**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MzkEAQEZDW30"
      },
      "source": [
        "0. GColab Menu : Exécution > Réinitialiser l'environnement d'exécution\n",
        "\n",
        "1.   Upload the RNNTagger (trainer) folder into your GDrive.\n",
        "2.   Modify the next files :\n",
        " *   ***Tagger-Data/prepare-data.sh*** : replace BFMGOLD and BFMGOLDLEM corpus paths (Sharedocs : right click in the .zip > partager > copier le lien).\n",
        " *   ***train-tagger.sh*** : Change *rnn-train.py* in line 46 to *PyRNN/rnn-train.py*. Otherwise, the bash script won't find the training python script.\n",
        " * ***train-tagger.sh*** : check the first line matches the GCollab python path (*#env/python*). Otherwise, *#!/usr/bin/python3*.\n",
        "3. Set the RNNTagger folder path. Then, mount to your drive :\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z6z3Phu83Wmm",
        "outputId": "bb723e8c-e75a-4324-b001-637a601d537b"
      },
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "rnntrainer_path = 'gdrive/My Drive/RNNTagger(trainer)' # set rnntagger folder in gdrive (training folder)\n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "os.chdir(rnntrainer_path)\n",
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n",
            "/content/gdrive/My Drive/RNNTagger(trainer)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vs378edjGmwE"
      },
      "source": [
        "4. Activate utf8::all for Perl :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OM4zcMTISWWp"
      },
      "source": [
        "!cpan utf8::all"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_HcRPTcUGyke"
      },
      "source": [
        "5. Go to **/Tagger-Data **folder and run *prepare-data.sh* (POS tags) :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mnIkFaHPVhMC"
      },
      "source": [
        "os.chdir('Tagger-Data')\n",
        "!chmod 755 -R ./split-corpus.pl\n",
        "!chmod 755 -R ./extract-data.pl\n",
        "!bash prepare-data.sh"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kKrwXTUmHfV2"
      },
      "source": [
        "6. Go to **/Lemmatizer-Data** and call *prepare-data.sh* (lemmas) :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PkLl0DrF47SL",
        "outputId": "e8f01391-6ae8-45d2-e74e-25ebb7541aad"
      },
      "source": [
        "%cd ..\n",
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/gdrive/My Drive/RNNTagger(trainer)\n",
            "/content/gdrive/My Drive/RNNTagger(trainer)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LLcol4eenkxE",
        "outputId": "06bbde26-c387-4d42-d993-446f67b29c66"
      },
      "source": [
        "os.chdir('Lemmatizer-Data')\n",
        "!chmod 755 -R ./prepare-data.pl\n",
        "!chmod 755 -R ./split.pl\n",
        "!chmod 755 -R ./filter.pl\n",
        "!chmod 755 -R ./make-lex.pl\n",
        "!bash prepare-data.sh"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "900000"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oxp7iDmX7EDx",
        "outputId": "45ac60df-d47c-44a2-fdb7-c464992bf1ad"
      },
      "source": [
        "%cd ..\n",
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/gdrive/My Drive/RNNTagger(trainer)\n",
            "/content/gdrive/My Drive/RNNTagger(trainer)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cYr_Y-7B7LOn"
      },
      "source": [
        "##Train tagger"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5RPUnBSxF2os"
      },
      "source": [
        "1.   In ***train-tagger.sh*** modify the line 46 (***rnn-train.py*** > ***PyRNN/rnn-train.py***). Otherwise, without that path specification the script won't be found.\n",
        "2.   In  ***rnn-train.py*** : check out that ***#!/usr/bin/python3*** is specified.\n",
        "3.   Check on top of the script that the execution path of python is correct. \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SpyKX-3kCDOG",
        "outputId": "9e3bc7bb-6ee9-4fec-eaaf-907f4ca6d18c"
      },
      "source": [
        "# ! echo $PYTHONPATH\n",
        "# !chmod 755 -R ./train-tagger.sh\n",
        "# # import os\n",
        "# os.environ['PYTHONPATH'] = 'usr/bin/python3'\n",
        "!echo $PYTHONPATH"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/env/python\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kMAmMCeWZ4ka"
      },
      "source": [
        "### ~4h training (tags)\n",
        "import os\n",
        "print(os.getcwd())\n",
        "!chmod u+x PyRNN/rnn-train.py\n",
        "!bash train-tagger.sh Tagger-Training\n",
        "\n",
        "import subprocess\n",
        "# subprocess.call([\"train-tagger.sh\", \"Tagger-Training\"], env={\"PATH\": \"\"})\n",
        "# subprocess.call([\"bash\", \"train-lemmatizer.sh\", \"Lemmatizer-Training\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tnwzyJXdI9fU"
      },
      "source": [
        "## Train lemmatizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b2QybpiLJFea"
      },
      "source": [
        "1.   Same previous steps : oath ***PyNMT/nmt-train.py *** specified in ***train-lemmatizer.sh***\n",
        "2.   Check on top of the script that the execution path of python is correct. \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qtGFdGO_IQ-B",
        "outputId": "c4271f3b-5534-4f41-a2a8-f95f2ef44d61"
      },
      "source": [
        "import os\n",
        "print(os.getcwd())\n",
        "!chmod u+x PyNMT/nmt-train.py\n",
        "# import os\n",
        "# os.environ['PYTHONPATH'] = 'usr/bin/python3'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/gdrive/My Drive/RNNTagger(trainer)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZgJzImAMMTL-",
        "outputId": "52e24e6e-d92f-4645-96c8-5f9c293301d4"
      },
      "source": [
        "# !CUDA_VISIBLE_DEVICES=1 python3 PyNMT/nmt-train.py\n",
        "# import torch\n",
        "# torch.cuda.device_count()\n",
        "# torch.cuda.set_device(0)\n",
        "\n",
        "!bash ./train-lemmatizer.sh Lemmatizer-Training\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "translation examples\n",
            "src: p a l u d ## N O M c o m\n",
            "ref: p a l u d\n",
            "tgt: p a l u d\n",
            "\n",
            "src: a l u e t ## N O M c o m\n",
            "ref: a l l e u\n",
            "tgt: a l l e u\n",
            "\n",
            "src: G r i t e ## N O M p r o\n",
            "ref: G r i t e\n",
            "tgt: G r i t e\n",
            "\n",
            "Training Loss: 0.13670659136082977\n",
            "Evaluation on dev data\n",
            "storing parameters\n",
            "translation examples\n",
            "src: s a i s i ## V E R p p e\n",
            "ref: s a i s i r\n",
            "tgt: s a i s i r\n",
            "\n",
            "src: c o n m e ## C O N s u b\n",
            "ref: c o m m e\n",
            "tgt: c o m m e\n",
            "\n",
            "src: j a u d e ## N O M c o m\n",
            "ref: g u i l d e\n",
            "tgt: g u i l d e\n",
            "\n",
            "Training Loss: 0.02435993800293654\n",
            "Evaluation on dev data\n",
            "storing parameters\n",
            "translation examples\n",
            "src: f i k i é ## V E R p p e\n",
            "ref: f i c h e r\n",
            "tgt: f i c h e r\n",
            "\n",
            "src: D I E N T ## V E R c j g\n",
            "ref: d i r e\n",
            "tgt: d i r e\n",
            "\n",
            "src: r e o n t ## A D J q u a\n",
            "ref: r o n d\n",
            "tgt: r o n d\n",
            "\n",
            "Training Loss: 0.013831932018103543\n",
            "Evaluation on dev data\n",
            "translation examples\n",
            "src: d e v r a ## V E R c j g\n",
            "ref: d e v o i r\n",
            "tgt: d e v o i r\n",
            "\n",
            "src: q u e r t ## V E R c j g\n",
            "ref: q u é r i r\n",
            "tgt: q u é r i r\n",
            "\n",
            "src: j u g e r ## V E R i n f\n",
            "ref: j u g e r\n",
            "tgt: j u g e r\n",
            "\n",
            "Training Loss: 0.009778315154992742\n",
            "Evaluation on dev data\n",
            "translation examples\n",
            "src: c r u p e ## N O M c o m\n",
            "ref: c r o u p e\n",
            "tgt: c r o u p e\n",
            "\n",
            "src: c a u p e ## V E R c j g\n",
            "ref: c o u p e r\n",
            "tgt: c o u p e r\n",
            "\n",
            "src: p l u i e ## N O M c o m\n",
            "ref: p l u i e\n",
            "tgt: p l u i e\n",
            "\n",
            "Training Loss: 0.007521410490987182\n",
            "Evaluation on dev data\n",
            "translation examples\n",
            "src: d i o n s ## V E R c j g\n",
            "ref: d i r e\n",
            "tgt: d i r e\n",
            "\n",
            "src: A ï r i é ## V E R c j g\n",
            "ref: a r r h e r\n",
            "tgt: a r r h e r\n",
            "\n",
            "src: n o u e r ## V E R i n f\n",
            "ref: n o e r\n",
            "tgt: n o e r\n",
            "\n",
            "Training Loss: 0.006069676057294419\n",
            "Evaluation on dev data\n",
            "translation examples\n",
            "src: m e ï s t ## V E R c j g\n",
            "ref: m e t t r e\n",
            "tgt: m e t t r e\n",
            "\n",
            "src: a u c i d ## V E R c j g\n",
            "ref: o c c i r e\n",
            "tgt: o c c i r e\n",
            "\n",
            "src: m i s s e ## V E R p p e\n",
            "ref: m e t t r e\n",
            "tgt: m e t t r e\n",
            "\n",
            "Training Loss: 0.005042397673396044\n",
            "Evaluation on dev data\n",
            "translation examples\n",
            "src: p a r d i ## V E R c j g\n",
            "ref: p e r d r e\n",
            "tgt: p e r d r e\n",
            "\n",
            "src: f e s e l ## N O M c o m\n",
            "ref: f a i s c e a u\n",
            "tgt: f a i s c e a u\n",
            "\n",
            "src: c o n u i ## V E R c j g\n",
            "ref: c o n n a î t r e\n",
            "tgt: c o n n a î t r e\n",
            "\n",
            "Training Loss: 0.00425329150554171\n",
            "Evaluation on dev data\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}