extractive-summarization/preprocessing-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "# !pip3 install malaya\n",
    "\n",
    "import malaya\n",
    "import re\n",
    "from malaya.texts._text_functions import split_into_sentences\n",
    "from malaya.texts import _regex\n",
    "\n",
    "tokenizer = malaya.preprocessing._tokenizer\n",
    "splitter = split_into_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "92579"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "stories = glob.glob('cnn/stories/*.story')\n",
    "len(stories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_number_regex(s):\n",
    "    if re.match(\"^\\d+?\\.\\d+?$\", s) is None:\n",
    "        return s.isdigit()\n",
    "    return True\n",
    "\n",
    "def preprocessing(string):\n",
    "    string = re.sub('[^\\'\"A-Za-z\\-(),.$0-9 ]+', ' ', string.lower())\n",
    "    tokenized = tokenizer(string)\n",
    "    tokens = []\n",
    "    for w in tokenized:\n",
    "        if is_number_regex(w):\n",
    "            tokens.append('<NUM>')\n",
    "        elif re.match(_regex._money, w):\n",
    "            tokens.append('<MONEY>')\n",
    "        elif re.match(_regex._date, w):\n",
    "            tokens.append('<DATE>')\n",
    "        else:\n",
    "            tokens.append(w)\n",
    "    return tokens\n",
    "\n",
    "def split_story(doc):\n",
    "    index = doc.find('@highlight')\n",
    "    story, highlights = doc[:index], doc[index:].split('@highlight')\n",
    "    highlights = [h.strip() for h in highlights if len(h) > 0]\n",
    "    stories = []\n",
    "    for s in splitter(story):\n",
    "        stories.append(preprocessing(s))\n",
    "    summaries = []\n",
    "    for s in highlights:\n",
    "        summaries.append(preprocessing(s))\n",
    "    return stories, summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_src_nsents = 3\n",
    "max_src_nsents = 20\n",
    "min_src_ntokens_per_sent = 5\n",
    "max_src_ntokens_per_sent = 30\n",
    "min_tgt_ntokens = 5\n",
    "max_tgt_ntokens = 500\n",
    "sep_token = '[SEP]'\n",
    "cls_token = '[CLS]'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(stories[0]) as fopen:\n",
    "    story = fopen.read()\n",
    "story, highlights = split_story(story)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_ngrams(n, text):\n",
    "    ngram_set = set()\n",
    "    text_length = len(text)\n",
    "    max_index_ngram_start = text_length - n\n",
    "    for i in range(max_index_ngram_start + 1):\n",
    "        ngram_set.add(tuple(text[i:i + n]))\n",
    "    return ngram_set\n",
    "\n",
    "\n",
    "def _get_word_ngrams(n, sentences):\n",
    "    assert len(sentences) > 0\n",
    "    assert n > 0\n",
    "\n",
    "    words = sum(sentences, [])\n",
    "    return _get_ngrams(n, words)\n",
    "\n",
    "def cal_rouge(evaluated_ngrams, reference_ngrams):\n",
    "    reference_count = len(reference_ngrams)\n",
    "    evaluated_count = len(evaluated_ngrams)\n",
    "\n",
    "    overlapping_ngrams = evaluated_ngrams.intersection(reference_ngrams)\n",
    "    overlapping_count = len(overlapping_ngrams)\n",
    "\n",
    "    if evaluated_count == 0:\n",
    "        precision = 0.0\n",
    "    else:\n",
    "        precision = overlapping_count / evaluated_count\n",
    "\n",
    "    if reference_count == 0:\n",
    "        recall = 0.0\n",
    "    else:\n",
    "        recall = overlapping_count / reference_count\n",
    "\n",
    "    f1_score = 2.0 * ((precision * recall) / (precision + recall + 1e-8))\n",
    "    return {\"f\": f1_score, \"p\": precision, \"r\": recall}\n",
    "\n",
    "\n",
    "def greedy_selection(doc_sent_list, abstract_sent_list, summary_size):\n",
    "    def _rouge_clean(s):\n",
    "        return re.sub(r'[^a-zA-Z0-9 ]', '', s)\n",
    "\n",
    "    max_rouge = 0.0\n",
    "    abstract = sum(abstract_sent_list, [])\n",
    "    abstract = _rouge_clean(' '.join(abstract)).split()\n",
    "    sents = [_rouge_clean(' '.join(s)).split() for s in doc_sent_list]\n",
    "    evaluated_1grams = [_get_word_ngrams(1, [sent]) for sent in sents]\n",
    "    reference_1grams = _get_word_ngrams(1, [abstract])\n",
    "    evaluated_2grams = [_get_word_ngrams(2, [sent]) for sent in sents]\n",
    "    reference_2grams = _get_word_ngrams(2, [abstract])\n",
    "\n",
    "    selected = []\n",
    "    for s in range(summary_size):\n",
    "        cur_max_rouge = max_rouge\n",
    "        cur_id = -1\n",
    "        for i in range(len(sents)):\n",
    "            if (i in selected):\n",
    "                continue\n",
    "            c = selected + [i]\n",
    "            candidates_1 = [evaluated_1grams[idx] for idx in c]\n",
    "            candidates_1 = set.union(*map(set, candidates_1))\n",
    "            candidates_2 = [evaluated_2grams[idx] for idx in c]\n",
    "            candidates_2 = set.union(*map(set, candidates_2))\n",
    "            rouge_1 = cal_rouge(candidates_1, reference_1grams)['f']\n",
    "            rouge_2 = cal_rouge(candidates_2, reference_2grams)['f']\n",
    "            rouge_score = rouge_1 + rouge_2\n",
    "            if rouge_score > cur_max_rouge:\n",
    "                cur_max_rouge = rouge_score\n",
    "                cur_id = i\n",
    "        if (cur_id == -1):\n",
    "            return selected\n",
    "        selected.append(cur_id)\n",
    "        max_rouge = cur_max_rouge\n",
    "\n",
    "    return sorted(selected)\n",
    "\n",
    "def get_xy(story, highlights):\n",
    "    idxs = [i for i, s in enumerate(story) if (len(s) > min_src_ntokens_per_sent)]\n",
    "    \n",
    "    idxs = [i for i, s in enumerate(story) if (len(s) > min_src_ntokens_per_sent)]\n",
    "\n",
    "    src = [story[i][:max_src_ntokens_per_sent] for i in idxs]\n",
    "    src = src[:max_src_nsents]\n",
    "\n",
    "    sent_labels = greedy_selection(src, highlights, 3)\n",
    "\n",
    "    _sent_labels = [0] * len(src)\n",
    "    for l in sent_labels:\n",
    "        _sent_labels[l] = 1\n",
    "    _sent_labels\n",
    "    \n",
    "    src_txt = [' '.join(sent) for sent in src]\n",
    "    text = ' {} {} '.format(sep_token, cls_token).join(src_txt)\n",
    "    text = '[CLS] %s [SEP]'%(text)\n",
    "    cls_ids = [i for i, t in enumerate(text.split()) if t == cls_token]\n",
    "    \n",
    "    return text, cls_ids, _sent_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import json\n",
    "\n",
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from dask import delayed\n",
    "# import dask\n",
    "\n",
    "# def process(i):\n",
    "#     with open(stories[i]) as fopen:\n",
    "#         story = fopen.read()\n",
    "#     story, highlights = split_story(story)\n",
    "#     return get_xy(story, highlights)\n",
    "\n",
    "# train = []\n",
    "# for i in range(len(stories)):\n",
    "#     im = delayed(process)(i)\n",
    "#     train.append(im)\n",
    "    \n",
    "# train = dask.compute(*train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(stories[1]) as fopen:\n",
    "    story = fopen.read()\n",
    "story, highlights = split_story(story)\n",
    "text, cls_ids, sent_labels = get_xy(story, highlights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 20, 560)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sent_labels), len(cls_ids), len(text.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 92579/92579 [16:41<00:00, 92.41it/s] \n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "texts, clss, labels = [], [], []\n",
    "\n",
    "for i in tqdm(range(len(stories))):\n",
    "    with open(stories[i]) as fopen:\n",
    "        story = fopen.read()\n",
    "    story, highlights = split_story(story)\n",
    "    text, cls_ids, sent_labels = get_xy(story, highlights)\n",
    "    if len(cls_ids) != len(sent_labels):\n",
    "        continue\n",
    "    texts.append(text)\n",
    "    clss.append(cls_ids)\n",
    "    labels.append(sent_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 118356\n",
      "Most common words [('the', 1974502), (',', 1740960), ('[CLS]', 1668596), ('[SEP]', 1668596), ('.', 1284463), ('to', 844716)]\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(texts).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "_, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size, atleast = 2)\n",
    "print('vocab from size: %d'%(len(dictionary)))\n",
    "print('Most common words', count[4:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_texts, test_texts, train_clss, test_clss, train_labels, test_labels = \\\n",
    "train_test_split(texts, clss, labels, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('dataset.pkl', 'wb') as fopen:\n",
    "    pickle.dump({'train_texts': train_texts,\n",
    "                'test_texts': test_texts,\n",
    "                'train_clss': train_clss,\n",
    "                'test_clss': test_clss,\n",
    "                'train_labels': train_labels,\n",
    "                'test_labels': test_labels}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dictionary.pkl', 'wb') as fopen:\n",
    "    pickle.dump({'dictionary': dictionary, 'rev_dictionary': rev_dictionary}, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}