pie_train_dev_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pie_train_dev_data.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN8svu9t2k2t1qz7I2AdXLS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/CristinaGHolgado/old-french-with-pie-rnntagger/blob/master/pie_train_dev_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JZuqiJbqgvnN"
      },
      "source": [
        "## Make training/dev data for pie (with conllu files)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nFYY5PKKhFNq",
        "outputId": "b84343ae-106a-4798-83dc-df6581d4b99c"
      },
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "\n",
        "path_train_data = \"gdrive/MyDrive/RNNTagger(trainer)/BFMGOLD/\" # path to the corpus"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iOTvnUgjhvcl",
        "outputId": "c828e4c0-82f1-4bc0-b209-09db6fb1782b"
      },
      "source": [
        "import pandas as pd\n",
        "import csv\n",
        "import glob\n",
        "import pprint\n",
        "\n",
        "# adding header for pie\n",
        "\n",
        "def make_train_dev_data(dev=None):\n",
        "  '''Create training/dev dataset for Pie: header, exclude files without lemmas, merge\n",
        "  Parameters\n",
        "  ----------\n",
        "  dev : bool(optional)\n",
        "    if True, it creates the devset with the selected corpus files\n",
        "    if False, it creates the training data excluding dev files\n",
        "  '''\n",
        "  _columns = [\"id\",\"token\",\"lemma\",\"ud_pos\",\"cattex_pos\",\n",
        "             \"morph1\",\"morph2\",\"morph3\",\"morph4\",\"src\"]\n",
        "  files_with_lemmas = 0\n",
        "  data_df_list = []\n",
        "  \n",
        "  corpus = sorted(glob.glob(os.path.join(path_train_data, \"*.conllu\")))\n",
        "  \n",
        "  if dev == True:\n",
        "    c = [f for f in corpus if f.split(\"/\")[-1].replace('.conllu','') in dev_files]\n",
        "    corpus = c\n",
        "  else:\n",
        "    c = [f for f in corpus if f.split(\"/\")[-1].replace('.conllu','') not in dev_files]\n",
        "    corpus = c\n",
        "  \n",
        "  print(\"\\nFiles selected:\\n\")\n",
        "  for f in corpus:\n",
        "    print(f.split(\"/\")[-1])\n",
        "    data = pd.read_csv(f, sep='\\t', encoding='utf-8',\n",
        "                              quoting=csv.QUOTE_NONE,\n",
        "                              names = _columns)\n",
        "\n",
        "    data = data[data.lemma != \"_\"]\n",
        "\n",
        "    if len(data) > 5:\n",
        "      files_with_lemmas+=1\n",
        "      data_df_list.append(data)\n",
        "  \n",
        "  data = pd.concat(data_df_list, axis=0, ignore_index=True) # merge dataframes into a single one\n",
        "  data.columns = _columns\n",
        "\n",
        "  output_name = ''.join([\"devdata.tsv\" if dev == True else \"traindata.tsv\"])\n",
        "  print(output_name)\n",
        "  \n",
        "  data.to_csv(output_name, sep='\\t', encoding='utf-8',\n",
        "            quoting=csv.QUOTE_NONE, index=None, header=True)\n",
        "  print()\n",
        "\n",
        "  print(\">> Files with lemmas:\", files_with_lemmas)\n",
        "  print(\"Files not contaning lemmas have been excluded\")\n",
        "\n",
        "\n",
        "##\n",
        "dev_files = ['aucassin','chartes_hain13']\n",
        "make_train_dev_data(dev=True) # make dev dataset\n",
        "make_train_dev_data(dev=False)  # make train dataset"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Files selected:\n",
            "\n",
            "aucassin.conllu\n",
            "chartes_hain13.conllu\n",
            "devdata.tsv\n",
            "\n",
            ">> Files with lemmas: 2\n",
            "Files nos contaning lemmas have been excluded\n",
            "\n",
            "Files selected:\n",
            "\n",
            "AlexisRaM.conllu\n",
            "CharretteKu.conllu\n",
            "CligesKu.conllu\n",
            "DescrEngl.conllu\n",
            "DialGreg2.conllu\n",
            "ErecKu.conllu\n",
            "Lapidfp.conllu\n",
            "PercevalKu.conllu\n",
            "SBath1.conllu\n",
            "SEustPr1.conllu\n",
            "SGenPr1.conllu\n",
            "YvainKu.conllu\n",
            "adgar.conllu\n",
            "amiamil.conllu\n",
            "artu.conllu\n",
            "beauma1-cat.conllu\n",
            "becket-cat.conllu\n",
            "beroul.conllu\n",
            "brut2.conllu\n",
            "cambps.conllu\n",
            "clari.conllu\n",
            "commyn1.conllu\n",
            "comput.conllu\n",
            "dole.conllu\n",
            "eneas1.conllu\n",
            "gcoin1.conllu\n",
            "jehpar.conllu\n",
            "maniere1396.conllu\n",
            "maniere1399.conllu\n",
            "maniere1415.conllu\n",
            "moree-cat.conllu\n",
            "passion.conllu\n",
            "qgraal_cm.conllu\n",
            "qjm.conllu\n",
            "regcrim1-cat.conllu\n",
            "roland.conllu\n",
            "rosel.conllu\n",
            "rosem2-cat.conllu\n",
            "slethgier.conllu\n",
            "stbrend.conllu\n",
            "traindata.tsv\n",
            "\n",
            ">> Files with lemmas: 19\n",
            "Files nos contaning lemmas have been excluded\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}