aihub_json_preprocessing_summary_and_report_generation_data.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AIHub Json Preprocessing"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Development Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install kss==3.7.3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS Argument Error: Restart Jupyter Kernel Runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install python-mecab-ko"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS 3.7.3 matches python-mecab-ko"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%pip install ray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import kss\n",
    "import ray\n",
    "import json\n",
    "import time\n",
    "import inspect\n",
    "from time import sleep\n",
    "from tqdm import tqdm\n",
    "from mecab import MeCab\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c:\\\\Users\\\\MinSeok\\\\Documents\\\\text-preprocessing\\\\sentence-segmentation'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AIHUB 요약문 및 레포트 생성 데이터"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Source](https://www.aihub.or.kr/aihubdata/data/view.do?currMenu=115&topMenu=100&aihubDataSe=realm&dataSetSn=582)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert JSON File to TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_preprocessing import make_topic_json_txt_file_path_list\n",
    "from data_preprocessing import count_number_of_txt_file_with_batch_list\n",
    "from data_preprocessing import write_jsontext_to_txt_file_with_batch_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_folder_list = ['AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/',\n",
    "             'AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/']\n",
    "\n",
    "topic_name_list = ['01.news_r', '02.briefing', '03.his_cul', '04.paper', '05.minute',\n",
    "                   '06.edit', '07.public', '08.speech', '09.literature', '10.narration']\n",
    "\n",
    "txt_path_list = [\"exploration/summary_and_report_generation_data_pro/AIHUB_summary_and_report_generation_data_train_\", \n",
    "                 \"exploration/summary_and_report_generation_data_pro/AIHUB_summary_and_report_generation_data_valid_\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of file:  165071\n"
     ]
    }
   ],
   "source": [
    "the_number_of_topic_json_file_df = make_topic_json_txt_file_path_list(json_folder_list, topic_name_list, txt_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 26765\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_01 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_01, the_number_of_train_txt_file_list_01 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_01, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 13017\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_02 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_02, the_number_of_train_txt_file_list_02 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_02, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 4518\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name +  \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_03 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_03, the_number_of_train_txt_file_list_03 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_03, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 6241\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_04 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_04, the_number_of_train_txt_file_list_04 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_04, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 23034\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_05 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_05, the_number_of_train_txt_file_list_05 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_05, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 6479\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_06 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_06, the_number_of_train_txt_file_list_06 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_06, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 6274\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_07 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_07, the_number_of_train_txt_file_list_07 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_07, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 24581\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_08 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_08, the_number_of_train_txt_file_list_08 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_08, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 8107\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_09 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_09, the_number_of_train_txt_file_list_09 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_09, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 3552\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_10 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_train_txt_file_10, the_number_of_train_txt_file_list_10 = count_number_of_txt_file_with_batch_list(train_json_file_path_list_10, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_file_by_batch_train_01_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_01.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_02_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_02.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_03_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_03.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_04_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_04.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_05_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_05.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_06_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_06.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_07_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_07.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_08_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_08.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_09_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_09.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_10_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_train_10.xlsx', engine='openpyxl')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>File</th>\n",
       "      <th>Length of Source List</th>\n",
       "      <th>The Number of txt File</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1271</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1208</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1240</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1231</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1244</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21595</th>\n",
       "      <td>21595</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1221</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21596</th>\n",
       "      <td>21596</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1220</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21597</th>\n",
       "      <td>21597</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1332</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21598</th>\n",
       "      <td>21598</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1276</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21599</th>\n",
       "      <td>21599</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...</td>\n",
       "      <td>1251</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21600 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0                                               File  \\\n",
       "0               0  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "1               1  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "2               2  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "3               3  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "4               4  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "...           ...                                                ...   \n",
       "21595       21595  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "21596       21596  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "21597       21597  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "21598       21598  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "21599       21599  AIHUB_요약문 및 레포트 생성 데이터/Training/원천데이터/TS1/01.n...   \n",
       "\n",
       "       Length of Source List  The Number of txt File  Description  \n",
       "0                       1271                       1          NaN  \n",
       "1                       1208                       1          NaN  \n",
       "2                       1240                       1          NaN  \n",
       "3                       1231                       1          NaN  \n",
       "4                       1244                       1          NaN  \n",
       "...                      ...                     ...          ...  \n",
       "21595                   1221                       1          NaN  \n",
       "21596                   1220                       1          NaN  \n",
       "21597                   1332                       1          NaN  \n",
       "21598                   1276                       1          NaN  \n",
       "21599                   1251                       1          NaN  \n",
       "\n",
       "[21600 rows x 5 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_file_by_batch_train_01_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 3344\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_01 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_01, the_number_of_valid_txt_file_list_01 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_01, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 1666\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_02 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_02, the_number_of_valid_txt_file_list_02 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_02, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 546\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_03 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_03, the_number_of_valid_txt_file_list_03 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_03, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 785\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_04 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_04, the_number_of_valid_txt_file_list_04 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_04, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 2883\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_05 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_05, the_number_of_valid_txt_file_list_05 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_05, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 809\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_06 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_06, the_number_of_valid_txt_file_list_06 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_06, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 767\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_07 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_07, the_number_of_valid_txt_file_list_07 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_07, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 3073\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_08 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_08, the_number_of_valid_txt_file_list_08 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_08, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 999\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_09 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_09, the_number_of_valid_txt_file_list_09 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_09, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 433\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_10 = source_file_index_df['source_file_path'].values.tolist()\n",
    "the_number_of_valid_txt_file_10, the_number_of_valid_txt_file_list_10 = count_number_of_txt_file_with_batch_list(valid_json_file_path_list_10, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_file_by_batch_valid_01_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_01.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_02_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_02.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_03_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_03.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_04_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_04.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_05_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_05.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_06_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_06.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_07_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_07.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_08_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_08.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_09_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_09.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_10_df = pd.read_excel('source_file_by_batch/summary_and_report_generation_data_valid_10.xlsx', engine='openpyxl')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>File</th>\n",
       "      <th>Length of Source List</th>\n",
       "      <th>The Number of txt File</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1261</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1229</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1211</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1250</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1223</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2695</th>\n",
       "      <td>2695</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1210</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2696</th>\n",
       "      <td>2696</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1232</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2697</th>\n",
       "      <td>2697</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1236</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2698</th>\n",
       "      <td>2698</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1221</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2699</th>\n",
       "      <td>2699</td>\n",
       "      <td>AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...</td>\n",
       "      <td>1242</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2700 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                               File  \\\n",
       "0              0  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "1              1  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "2              2  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "3              3  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "4              4  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "...          ...                                                ...   \n",
       "2695        2695  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "2696        2696  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "2697        2697  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "2698        2698  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "2699        2699  AIHUB_요약문 및 레포트 생성 데이터/Validation/원천데이터/VS1/01...   \n",
       "\n",
       "      Length of Source List  The Number of txt File  Description  \n",
       "0                      1261                       1          NaN  \n",
       "1                      1229                       1          NaN  \n",
       "2                      1211                       1          NaN  \n",
       "3                      1250                       1          NaN  \n",
       "4                      1223                       1          NaN  \n",
       "...                     ...                     ...          ...  \n",
       "2695                   1210                       1          NaN  \n",
       "2696                   1232                       1          NaN  \n",
       "2697                   1236                       1          NaN  \n",
       "2698                   1221                       1          NaN  \n",
       "2699                   1242                       1          NaN  \n",
       "\n",
       "[2700 rows x 5 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_file_by_batch_valid_01_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_01 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_01 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_01, train_txt_file_path_list_01, batch_size, the_number_of_train_txt_file_list_01, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_02 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_02 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_02, train_txt_file_path_list_02, batch_size, the_number_of_train_txt_file_list_02, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_03 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_03 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_03, train_txt_file_path_list_03, batch_size, the_number_of_train_txt_file_list_03, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_04 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_04 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_04, train_txt_file_path_list_04, batch_size, the_number_of_train_txt_file_list_04, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_05 = source_file_index_df['source_file_name'].values.tolist()\n",
    "train_txt_file_path_list_05 = source_file_index_df['txt_file_name'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_05, train_txt_file_path_list_05, batch_size, the_number_of_train_txt_file_list_05, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_06 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_06 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_06, train_txt_file_path_list_06, batch_size, the_number_of_train_txt_file_list_06, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_07 = source_file_index_df['source_file_name'].values.tolist()\n",
    "train_txt_file_path_list_07 = source_file_index_df['txt_file_name'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_07, train_txt_file_path_list_07, batch_size, the_number_of_train_txt_file_list_07, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_08 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_08 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_08, train_txt_file_path_list_08, batch_size, the_number_of_train_txt_file_list_08, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_09 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_09 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_09, train_txt_file_path_list_09, batch_size, the_number_of_train_txt_file_list_09, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_train\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "train_json_file_path_list_10 = source_file_index_df['source_file_path'].values.tolist()\n",
    "train_txt_file_path_list_10 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_path_list_10, train_txt_file_path_list_10, batch_size, the_number_of_train_txt_file_list_10, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_01 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_01 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_01, valid_txt_file_path_list_01, batch_size, the_number_of_valid_txt_file_list_01, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_02 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_02 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_02, valid_txt_file_path_list_02, batch_size, the_number_of_valid_txt_file_list_02, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_03 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_03 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_03, valid_txt_file_path_list_03, batch_size, the_number_of_valid_txt_file_list_03, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_04 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_04 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_04, valid_txt_file_path_list_04, batch_size, the_number_of_valid_txt_file_list_04, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_05 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_05 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_05, valid_txt_file_path_list_05, batch_size, the_number_of_valid_txt_file_list_05, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_06 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_06 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_06, valid_txt_file_path_list_06, batch_size, the_number_of_valid_txt_file_list_06, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_07 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_07 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_07, valid_txt_file_path_list_07, batch_size, the_number_of_valid_txt_file_list_07, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_08 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_08 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_08, valid_txt_file_path_list_08, batch_size, the_number_of_valid_txt_file_list_08, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_09 = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_09 = source_file_index_df['txt_file_path'].values.tolist()\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_09, valid_txt_file_path_list_09, batch_size, the_number_of_valid_txt_file_list_09, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = \"summary_and_report_generation_data\"\n",
    "source_file_index_xlsx_path = \"source_file_index/\" + corpus_name + \"_valid\" + \"_source_file_index.xlsx\"\n",
    "source_file_index_df = pd.read_excel(source_file_index_xlsx_path, engine='openpyxl')\n",
    "valid_json_file_path_list_10  = source_file_index_df['source_file_path'].values.tolist()\n",
    "valid_txt_file_path_list_10  = source_file_index_df['txt_file_path'].values.tolist()batch_size = 1000\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_path_list_10, valid_txt_file_path_list_10, batch_size, the_number_of_valid_txt_file_list_10, corpus_name)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_segmentation import preprocessing_text\n",
    "from data_preprocessing import make_pro_post_txt_file_path_list\n",
    "from data_preprocessing import make_topic_pro_post_txt_file_name_list\n",
    "from data_preprocessing import merge_and_deduplicate_topic_corpus_txt\n",
    "from reading_data import reading_txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_corpus_name = \"AIHUB_corpus/exploration/summary_and_report_generation_data_pro/AIHUB_summary_and_report_generation_data_\"\n",
    "\n",
    "topic_name_list = ['01.news_r', '02.briefing', '03.his_cul', '04.paper', '05.minute',\n",
    "                '06.edit', '07.public', '08.speech', '09.literature', '10.narration']\n",
    "\n",
    "make_topic_pro_post_txt_file_name_list(pro_corpus_name, topic_name_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_post_xlsx_path = \"pro_post_corpus_path/summary_and_report_generation_data_\" + \"train_\" + \"01\" + \".xlsx\"\n",
    "pro_post_total_corpus_path_df = pd.read_excel(pro_post_xlsx_path, engine='openpyxl')\n",
    "pro_total_corpus_path_list = pro_post_total_corpus_path_df['Pro'].values.tolist()\n",
    "post_total_corpus_path_list = pro_post_total_corpus_path_df['Post'].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "168"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pro_total_corpus_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 이러한 이야기는 단재가 어떤 고대의 사서를 참고해 기록한 것인지는 명확하게 밝혀져 있지 않다. 그러나 김부식의 《삼국사기》에는 생략되어 있는 부분이다.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"source\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "이러한 이야기는 단재가 어떤 고대의 사서를 참고해 기록한 것인지는 명확하게 밝혀져 있지 않다.\n",
      "\n",
      "그러나 김부식의 삼국사기 에는 생략되어 있는 부분이다.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"preprocessing\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-05-31 10:00:22,317\tINFO worker.py:1625 -- Started a local Ray instance.\n"
     ]
    }
   ],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '01' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '02' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '03' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '04' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '05' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '06' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '07' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '08' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '09' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'train_' + '10' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '01' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '02' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '03' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '04' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '05' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '06' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '07' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '08' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '09' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/summary_and_report_generation_data_' + 'valid_' + '10' + '.xlsx' \\\n",
    "    --start_index 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessing_corpus_path = \"AIHUB_corpus/exploration/summary_and_report_generation_data_post/AIHUB_summary_and_report_generation_data_\" +\"*.txt\"\n",
    "merge_corpus_path = 'AIHUB_corpus/duplicate/AIHUB_summary_and_report_generation_data_'\n",
    "deduplicate_corpus_path = 'AIHUB_corpus/AIHUB_automatic_patent_classification_data.txt'\n",
    "\n",
    "topic_name_list = ['01.news_r', '02.briefing', '03.his_cul', '04.paper', '05.minute',\n",
    "                   '06.edit', '07.public', '08.speech', '09.literature', '10.narration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_and_deduplicate_topic_corpus_txt(preprocessing_corpus_path, merge_corpus_path,\n",
    "                                        deduplicate_corpus_path, topic_name_list)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "corpus_06",
   "language": "python",
   "name": "corpus_06"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}