ASR/whisper-small.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:04:00.824791800Z",
     "start_time": "2023-05-19T07:03:53.105870300Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alc/venvs/til2023/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['HF_HOME'] = 'huggingface'\n",
    "os.environ['HF_HUB_DISABLE_TELEMETRY'] = '1'\n",
    "os.environ['HF_HUB_ENABLE_HF_TRANSFER'] = 'True'\n",
    "import math\n",
    "from datasets import load_dataset\n",
    "from transformers import WhisperProcessor, WhisperForConditionalGeneration, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "import torch\n",
    "from dataclasses import dataclass, field\n",
    "from typing import Any, Dict, List, Union\n",
    "import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:04:00.838791400Z",
     "start_time": "2023-05-19T07:04:00.821791500Z"
    }
   },
   "outputs": [],
   "source": [
    "model_name = 'openai/whisper-small.en'\n",
    "checkpoint_name = 'whisper-checkpoints/checkpoint-750/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:04:01.398692Z",
     "start_time": "2023-05-19T07:04:00.836808200Z"
    }
   },
   "outputs": [],
   "source": [
    "processor = WhisperProcessor.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:04:13.435825Z",
     "start_time": "2023-05-19T07:04:01.394691100Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Resolving data files: 100%|██████████| 3750/3750 [00:00<00:00, 141412.81it/s]\n",
      "Found cached dataset audiofolder (/home/alc/TIL-2023/ASR/huggingface/datasets/audiofolder/default-111e26b5add33029/0.0.0/6cbdd16f8688354c63b4e2a36e1585d05de285023ee6443ffd71c4182055c0fc)\n"
     ]
    }
   ],
   "source": [
    "ds = load_dataset('audiofolder', data_dir='TIL_data_folder', split='train')  # specify split to return a Dataset object instead of a DatasetDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:04:13.575387500Z",
     "start_time": "2023-05-19T07:04:13.434826400Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ds = ds.train_test_split(test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:07:54.055577100Z",
     "start_time": "2023-05-19T07:04:13.576902Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                             \r"
     ]
    }
   ],
   "source": [
    "def prepare_dataset(batch):\n",
    "    model_name = 'openai/whisper-small.en'\n",
    "    from transformers import WhisperProcessor\n",
    "    processor = WhisperProcessor.from_pretrained(model_name)\n",
    "    batch[\"input_features\"] =[processor(audio[\"array\"], sampling_rate=audio[\"sampling_rate\"]).input_features[0] for audio in batch[\"audio\"]]\n",
    "    batch[\"input_length\"] = [len(b) for b in batch[\"input_features\"]]\n",
    "    batch[\"labels\"] = processor(text=batch[\"annotation\"]).input_ids\n",
    "    batch['length'] = batch[\"input_length\"]\n",
    "    return batch\n",
    "\n",
    "ds = ds.map(prepare_dataset, num_proc=8, batched=True, batch_size=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:08:11.793311800Z",
     "start_time": "2023-05-19T07:08:11.771310700Z"
    }
   },
   "outputs": [],
   "source": [
    "# purpose of the data collator is to ensure that the inputs and labels are padded correctly\n",
    "\n",
    "@dataclass\n",
    "class DataCollatorSpeechSeq2SeqWithPadding:\n",
    "    processor: WhisperProcessor\n",
    "\n",
    "    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n",
    "        # split inputs and labels since they have to be of different lengths and need different padding methods\n",
    "        \n",
    "        # first treat the audio inputs by simply returning torch tensors\n",
    "        input_features = [{\"input_features\": feature[\"input_features\"]} for feature in features]\n",
    "        batch = self.processor.feature_extractor.pad(input_features, return_tensors=\"pt\")\n",
    "\n",
    "        # get the tokenized label sequences\n",
    "        label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n",
    "        # pad the labels to max length\n",
    "        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors=\"pt\")\n",
    "\n",
    "        # replace padding with -100 to ignore loss correctly\n",
    "        labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n",
    "\n",
    "        # if bos token is appended in previous tokenization step,\n",
    "        # cut bos token here as it's append later anyways\n",
    "        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():\n",
    "            labels = labels[:, 1:]\n",
    "\n",
    "        batch[\"labels\"] = labels\n",
    "\n",
    "        return batch\n",
    "\n",
    "data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:08:13.616042100Z",
     "start_time": "2023-05-19T07:08:12.258678500Z"
    }
   },
   "outputs": [],
   "source": [
    "metric = evaluate.load(\"wer\")\n",
    "\n",
    "def compute_metrics(pred):\n",
    "    pred_ids = pred.predictions\n",
    "    label_ids = pred.label_ids\n",
    "\n",
    "    # replace -100 with the pad_token_id\n",
    "    label_ids[label_ids == -100] = processor.tokenizer.pad_token_id\n",
    "\n",
    "    # we do not want to group tokens when computing the metrics\n",
    "    pred_str = processor.tokenizer.batch_decode(pred_ids, skip_special_tokens=True)\n",
    "    label_str = processor.tokenizer.batch_decode(label_ids, skip_special_tokens=True)\n",
    "\n",
    "    wer = metric.compute(predictions=pred_str, references=label_str)\n",
    "\n",
    "    return {\"wer\": wer}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:08:18.024710500Z",
     "start_time": "2023-05-19T07:08:13.617039500Z"
    }
   },
   "outputs": [],
   "source": [
    "model = WhisperForConditionalGeneration.from_pretrained(\n",
    "    model_name,  # checkpoint_name\n",
    "    pad_token_id=processor.tokenizer.pad_token_id,\n",
    "    mask_time_prob=0.5,  # 0.05\n",
    "    mask_time_length=10, # 10\n",
    "    mask_feature_prob=0.5, # 0\n",
    "    mask_feature_length=10, # 10\n",
    "    apply_spec_augment=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:08:18.036711400Z",
     "start_time": "2023-05-19T07:08:18.022711100Z"
    }
   },
   "outputs": [],
   "source": [
    "model.freeze_encoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:11:13.069961300Z",
     "start_time": "2023-05-19T07:11:13.057404Z"
    }
   },
   "outputs": [],
   "source": [
    "per_gpu_bs = 3\n",
    "effective_bs = 32\n",
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"whisper-checkpoints\",\n",
    "    overwrite_output_dir =True,\n",
    "    per_device_train_batch_size=per_gpu_bs,\n",
    "    gradient_accumulation_steps=math.ceil(effective_bs/per_gpu_bs),\n",
    "    learning_rate=1e-4,\n",
    "    num_train_epochs=20,\n",
    "    gradient_checkpointing=False,\n",
    "    fp16=True,\n",
    "    # bf16=True,  # for A100\n",
    "    fp16_full_eval=True,\n",
    "    # bf16_full_eval=True,  # for A100\n",
    "    group_by_length=True,  # slows down\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy='epoch',  # epoch\n",
    "    save_safetensors=True,\n",
    "    per_device_eval_batch_size=4,\n",
    "    save_steps=1,\n",
    "    eval_steps=1,\n",
    "    logging_steps=100,\n",
    "    save_total_limit=3,\n",
    "    lr_scheduler_type='cosine',\n",
    "    load_best_model_at_end=True,  # True\n",
    "    adam_beta1=0.9,\n",
    "    adam_beta2=0.98,  # follow fairseq fintuning config\n",
    "    warmup_ratio=0.22, # follow Ranger21\n",
    "    weight_decay=1e-4,  # follow Ranger21\n",
    "    metric_for_best_model=\"wer\",\n",
    "    greater_is_better=False,\n",
    "    report_to=['tensorboard'],\n",
    "    dataloader_num_workers=24 if os.name != 'nt' else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:08:18.098709400Z",
     "start_time": "2023-05-19T07:08:18.086710100Z"
    }
   },
   "outputs": [],
   "source": [
    "class CustomWhisperTrainer(Seq2SeqTrainer):\n",
    "    def training_step(self, model: torch.nn.Module, inputs: Dict[str, Union[torch.Tensor, Any]]) -> torch.Tensor:\n",
    "        \"\"\"\n",
    "        Perform a training step on a batch of inputs.\n",
    "\n",
    "        Subclass and override to inject custom behavior.\n",
    "\n",
    "        Args:\n",
    "            model (:obj:`nn.Module`):\n",
    "                The model to train.\n",
    "            inputs (:obj:`Dict[str, Union[torch.Tensor, Any]]`):\n",
    "                The inputs and targets of the model.\n",
    "\n",
    "                The dictionary will be unpacked before being fed to the model. Most models expect the targets under the\n",
    "                argument :obj:`labels`. Check your model's documentation for all accepted arguments.\n",
    "\n",
    "        Return:\n",
    "            :obj:`torch.Tensor`: The tensor with training loss on this batch.\n",
    "        \"\"\"\n",
    "\n",
    "        model.train()\n",
    "        inputs = self._prepare_inputs(inputs)\n",
    "        loss = self.compute_loss(model, inputs)\n",
    "\n",
    "        if self.args.gradient_accumulation_steps > 1:\n",
    "            loss = loss / self.args.gradient_accumulation_steps\n",
    "\n",
    "        if os.name != 'nt':\n",
    "            accelerator.backward(self.scaler.scale(loss))\n",
    "            # self.scaler.scale(loss).backward()\n",
    "        else:\n",
    "            self.scaler.scale(loss).backward()\n",
    "        return loss.detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:08:21.285482Z",
     "start_time": "2023-05-19T07:08:21.272629300Z"
    }
   },
   "outputs": [],
   "source": [
    "if os.name != 'nt':\n",
    "    from accelerate import Accelerator\n",
    "    accelerator = Accelerator(mixed_precision='fp16', dynamo_backend='eager')  # FP8 needs transformer_engine package which is only on Linux with Hopper GPUs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-19T07:08:22.337997500Z",
     "start_time": "2023-05-19T07:08:22.318481400Z"
    }
   },
   "outputs": [],
   "source": [
    "def tri_stage_schedule(epoch: int, max_epoch = training_args.num_train_epochs, stage_ratio = [0.1, 0.4, 0.5], peak_lr = training_args.learning_rate, initial_lr_scale=0.01, final_lr_scale=0.05):\n",
    "    \"\"\"https://github.com/facebookresearch/fairseq/blob/5ecbbf58d6e80b917340bcbf9d7bdbb539f0f92b/fairseq/optim/lr_scheduler/tri_stage_lr_scheduler.py#L51\"\"\"\n",
    "    assert sum(stage_ratio) == 1\n",
    "    current_ratio = epoch / max_epoch\n",
    "    if current_ratio < stage_ratio[0]:  # linear warmup\n",
    "        lrs = torch.linspace(initial_lr_scale * peak_lr, peak_lr, int(stage_ratio[0] * max_epoch))\n",
    "        return lrs[epoch]\n",
    "    elif stage_ratio[0] <= current_ratio <= stage_ratio[1]:  # constant\n",
    "        return peak_lr\n",
    "    else:  # exponential decay\n",
    "        decay_factor = -math.log(final_lr_scale) / (stage_ratio[2] * max_epoch)\n",
    "        return peak_lr * math.exp(-decay_factor * stage_ratio[2] * max_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alc/venvs/til2023/lib/python3.10/site-packages/torch/utils/data/dataloader.py:560: UserWarning: This DataLoader will create 24 worker processes in total. Our suggested max number of worker in current system is 16, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
      "  warnings.warn(_create_warning_msg(\n",
      "/home/alc/venvs/til2023/lib/python3.10/site-packages/transformers/optimization.py:407: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "[2023-05-21 01:32:33,666] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64)\n",
      "   function: 'forward' (/home/alc/venvs/til2023/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:485)\n",
      "   reasons:  ___check_obj_id(self, 140666950594000)\n",
      "to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html.\n",
      "[2023-05-21 01:32:40,029] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64)\n",
      "   function: 'forward' (/home/alc/venvs/til2023/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:267)\n",
      "   reasons:  ___check_obj_id(self, 140666950726368)\n",
      "to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html.\n",
      "[2023-05-21 01:32:40,254] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64)\n",
      "   function: '_shape' (/home/alc/venvs/til2023/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:263)\n",
      "   reasons:  ___check_obj_id(self, 140666950594192)\n",
      "to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html.\n",
      "[2023-05-21 01:32:42,508] torch._dynamo.convert_frame: [WARNING] torch._dynamo hit config.cache_size_limit (64)\n",
      "   function: 'forward' (/home/alc/venvs/til2023/lib/python3.10/site-packages/transformers/activations.py:77)\n",
      "   reasons:  ___check_obj_id(self, 140666950724928)\n",
      "to diagnose recompilation issues, see https://pytorch.org/docs/master/dynamo/troubleshooting.html.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='94' max='1860' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [  94/1860 02:13 < 42:50, 0.69 it/s, Epoch 0.99/20]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "    <div>\n",
       "      \n",
       "      <progress value='38' max='188' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 38/188 00:05 < 00:20, 7.26 it/s]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alc/venvs/til2023/lib/python3.10/site-packages/torch/utils/data/dataloader.py:560: UserWarning: This DataLoader will create 24 worker processes in total. Our suggested max number of worker in current system is 16, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
      "  warnings.warn(_create_warning_msg(\n"
     ]
    },
    {
     "ename": "OutOfMemoryError",
     "evalue": "CUDA out of memory. Tried to allocate 916.00 MiB (GPU 0; 23.69 GiB total capacity; 19.54 GiB already allocated; 859.00 MiB free; 21.44 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[15], line 22\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[39mif\u001b[39;00m os\u001b[39m.\u001b[39mname \u001b[39m!=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mnt\u001b[39m\u001b[39m'\u001b[39m:  \u001b[39m# windows does not support torch.compile yet\u001b[39;00m\n\u001b[1;32m     20\u001b[0m     \u001b[39m# pass\u001b[39;00m\n\u001b[1;32m     21\u001b[0m     trainer\u001b[39m.\u001b[39mmodel_wrapped, trainer\u001b[39m.\u001b[39moptimizer, trainer\u001b[39m.\u001b[39mlr_scheduler \u001b[39m=\u001b[39m accelerator\u001b[39m.\u001b[39mprepare(trainer\u001b[39m.\u001b[39mmodel_wrapped, trainer\u001b[39m.\u001b[39moptimizer, trainer\u001b[39m.\u001b[39mlr_scheduler)\n\u001b[0;32m---> 22\u001b[0m trainer\u001b[39m.\u001b[39;49mtrain()\n\u001b[1;32m     23\u001b[0m \u001b[39mif\u001b[39;00m os\u001b[39m.\u001b[39mname \u001b[39m!=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mnt\u001b[39m\u001b[39m'\u001b[39m:\n\u001b[1;32m     24\u001b[0m     accelerator\u001b[39m.\u001b[39mwait_for_everyone()\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer.py:1664\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1659\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel_wrapped \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\n\u001b[1;32m   1661\u001b[0m inner_training_loop \u001b[39m=\u001b[39m find_executable_batch_size(\n\u001b[1;32m   1662\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_inner_training_loop, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_train_batch_size, args\u001b[39m.\u001b[39mauto_find_batch_size\n\u001b[1;32m   1663\u001b[0m )\n\u001b[0;32m-> 1664\u001b[0m \u001b[39mreturn\u001b[39;00m inner_training_loop(\n\u001b[1;32m   1665\u001b[0m     args\u001b[39m=\u001b[39;49margs,\n\u001b[1;32m   1666\u001b[0m     resume_from_checkpoint\u001b[39m=\u001b[39;49mresume_from_checkpoint,\n\u001b[1;32m   1667\u001b[0m     trial\u001b[39m=\u001b[39;49mtrial,\n\u001b[1;32m   1668\u001b[0m     ignore_keys_for_eval\u001b[39m=\u001b[39;49mignore_keys_for_eval,\n\u001b[1;32m   1669\u001b[0m )\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer.py:2034\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2031\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcontrol\u001b[39m.\u001b[39mshould_training_stop \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m   2033\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcontrol \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_handler\u001b[39m.\u001b[39mon_epoch_end(args, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcontrol)\n\u001b[0;32m-> 2034\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)\n\u001b[1;32m   2036\u001b[0m \u001b[39mif\u001b[39;00m DebugOption\u001b[39m.\u001b[39mTPU_METRICS_DEBUG \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mdebug:\n\u001b[1;32m   2037\u001b[0m     \u001b[39mif\u001b[39;00m is_torch_tpu_available():\n\u001b[1;32m   2038\u001b[0m         \u001b[39m# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)\u001b[39;00m\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer.py:2300\u001b[0m, in \u001b[0;36mTrainer._maybe_log_save_evaluate\u001b[0;34m(self, tr_loss, model, trial, epoch, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2298\u001b[0m         metrics\u001b[39m.\u001b[39mupdate(dataset_metrics)\n\u001b[1;32m   2299\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2300\u001b[0m     metrics \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mevaluate(ignore_keys\u001b[39m=\u001b[39;49mignore_keys_for_eval)\n\u001b[1;32m   2301\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_report_to_hp_search(trial, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mglobal_step, metrics)\n\u001b[1;32m   2303\u001b[0m \u001b[39m# Run delayed LR scheduler now that metrics are populated\u001b[39;00m\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer_seq2seq.py:159\u001b[0m, in \u001b[0;36mSeq2SeqTrainer.evaluate\u001b[0;34m(self, eval_dataset, ignore_keys, metric_key_prefix, **gen_kwargs)\u001b[0m\n\u001b[1;32m    154\u001b[0m gen_kwargs[\u001b[39m\"\u001b[39m\u001b[39mnum_beams\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m (\n\u001b[1;32m    155\u001b[0m     gen_kwargs[\u001b[39m\"\u001b[39m\u001b[39mnum_beams\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39mif\u001b[39;00m gen_kwargs\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mnum_beams\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mgeneration_num_beams\n\u001b[1;32m    156\u001b[0m )\n\u001b[1;32m    157\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_gen_kwargs \u001b[39m=\u001b[39m gen_kwargs\n\u001b[0;32m--> 159\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49mevaluate(eval_dataset, ignore_keys\u001b[39m=\u001b[39;49mignore_keys, metric_key_prefix\u001b[39m=\u001b[39;49mmetric_key_prefix)\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer.py:3029\u001b[0m, in \u001b[0;36mTrainer.evaluate\u001b[0;34m(self, eval_dataset, ignore_keys, metric_key_prefix)\u001b[0m\n\u001b[1;32m   3026\u001b[0m start_time \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mtime()\n\u001b[1;32m   3028\u001b[0m eval_loop \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mprediction_loop \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39muse_legacy_prediction_loop \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mevaluation_loop\n\u001b[0;32m-> 3029\u001b[0m output \u001b[39m=\u001b[39m eval_loop(\n\u001b[1;32m   3030\u001b[0m     eval_dataloader,\n\u001b[1;32m   3031\u001b[0m     description\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mEvaluation\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[1;32m   3032\u001b[0m     \u001b[39m# No point gathering the predictions if there are no metrics, otherwise we defer to\u001b[39;49;00m\n\u001b[1;32m   3033\u001b[0m     \u001b[39m# self.args.prediction_loss_only\u001b[39;49;00m\n\u001b[1;32m   3034\u001b[0m     prediction_loss_only\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m \u001b[39mif\u001b[39;49;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcompute_metrics \u001b[39mis\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m \u001b[39melse\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m   3035\u001b[0m     ignore_keys\u001b[39m=\u001b[39;49mignore_keys,\n\u001b[1;32m   3036\u001b[0m     metric_key_prefix\u001b[39m=\u001b[39;49mmetric_key_prefix,\n\u001b[1;32m   3037\u001b[0m )\n\u001b[1;32m   3039\u001b[0m total_batch_size \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39meval_batch_size \u001b[39m*\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mworld_size\n\u001b[1;32m   3040\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mmetric_key_prefix\u001b[39m}\u001b[39;00m\u001b[39m_jit_compilation_time\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m output\u001b[39m.\u001b[39mmetrics:\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer.py:3235\u001b[0m, in \u001b[0;36mTrainer.evaluation_loop\u001b[0;34m(self, dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix)\u001b[0m\n\u001b[1;32m   3233\u001b[0m         logits \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpreprocess_logits_for_metrics(logits, labels)\n\u001b[1;32m   3234\u001b[0m     logits \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_nested_gather(logits)\n\u001b[0;32m-> 3235\u001b[0m     preds_host \u001b[39m=\u001b[39m logits \u001b[39mif\u001b[39;00m preds_host \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m nested_concat(preds_host, logits, padding_index\u001b[39m=\u001b[39;49m\u001b[39m-\u001b[39;49m\u001b[39m100\u001b[39;49m)\n\u001b[1;32m   3236\u001b[0m \u001b[39mif\u001b[39;00m labels \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m   3237\u001b[0m     labels \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_nested_gather(labels)\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer_pt_utils.py:114\u001b[0m, in \u001b[0;36mnested_concat\u001b[0;34m(tensors, new_tensors, padding_index)\u001b[0m\n\u001b[1;32m    110\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mtype\u001b[39m(tensors) \u001b[39m==\u001b[39m \u001b[39mtype\u001b[39m(\n\u001b[1;32m    111\u001b[0m     new_tensors\n\u001b[1;32m    112\u001b[0m ), \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mExpected `tensors` and `new_tensors` to have the same type but found \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mtype\u001b[39m(tensors)\u001b[39m}\u001b[39;00m\u001b[39m and \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mtype\u001b[39m(new_tensors)\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    113\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(tensors, (\u001b[39mlist\u001b[39m, \u001b[39mtuple\u001b[39m)):\n\u001b[0;32m--> 114\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mtype\u001b[39;49m(tensors)(nested_concat(t, n, padding_index\u001b[39m=\u001b[39;49mpadding_index) \u001b[39mfor\u001b[39;49;00m t, n \u001b[39min\u001b[39;49;00m \u001b[39mzip\u001b[39;49m(tensors, new_tensors))\n\u001b[1;32m    115\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(tensors, torch\u001b[39m.\u001b[39mTensor):\n\u001b[1;32m    116\u001b[0m     \u001b[39mreturn\u001b[39;00m torch_pad_and_concatenate(tensors, new_tensors, padding_index\u001b[39m=\u001b[39mpadding_index)\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer_pt_utils.py:114\u001b[0m, in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    110\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mtype\u001b[39m(tensors) \u001b[39m==\u001b[39m \u001b[39mtype\u001b[39m(\n\u001b[1;32m    111\u001b[0m     new_tensors\n\u001b[1;32m    112\u001b[0m ), \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mExpected `tensors` and `new_tensors` to have the same type but found \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mtype\u001b[39m(tensors)\u001b[39m}\u001b[39;00m\u001b[39m and \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mtype\u001b[39m(new_tensors)\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    113\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(tensors, (\u001b[39mlist\u001b[39m, \u001b[39mtuple\u001b[39m)):\n\u001b[0;32m--> 114\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mtype\u001b[39m(tensors)(nested_concat(t, n, padding_index\u001b[39m=\u001b[39;49mpadding_index) \u001b[39mfor\u001b[39;00m t, n \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(tensors, new_tensors))\n\u001b[1;32m    115\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(tensors, torch\u001b[39m.\u001b[39mTensor):\n\u001b[1;32m    116\u001b[0m     \u001b[39mreturn\u001b[39;00m torch_pad_and_concatenate(tensors, new_tensors, padding_index\u001b[39m=\u001b[39mpadding_index)\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer_pt_utils.py:116\u001b[0m, in \u001b[0;36mnested_concat\u001b[0;34m(tensors, new_tensors, padding_index)\u001b[0m\n\u001b[1;32m    114\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mtype\u001b[39m(tensors)(nested_concat(t, n, padding_index\u001b[39m=\u001b[39mpadding_index) \u001b[39mfor\u001b[39;00m t, n \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(tensors, new_tensors))\n\u001b[1;32m    115\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(tensors, torch\u001b[39m.\u001b[39mTensor):\n\u001b[0;32m--> 116\u001b[0m     \u001b[39mreturn\u001b[39;00m torch_pad_and_concatenate(tensors, new_tensors, padding_index\u001b[39m=\u001b[39;49mpadding_index)\n\u001b[1;32m    117\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(tensors, Mapping):\n\u001b[1;32m    118\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mtype\u001b[39m(tensors)(\n\u001b[1;32m    119\u001b[0m         {k: nested_concat(t, new_tensors[k], padding_index\u001b[39m=\u001b[39mpadding_index) \u001b[39mfor\u001b[39;00m k, t \u001b[39min\u001b[39;00m tensors\u001b[39m.\u001b[39mitems()}\n\u001b[1;32m    120\u001b[0m     )\n",
      "File \u001b[0;32m~/venvs/til2023/lib/python3.10/site-packages/transformers/trainer_pt_utils.py:75\u001b[0m, in \u001b[0;36mtorch_pad_and_concatenate\u001b[0;34m(tensor1, tensor2, padding_index)\u001b[0m\n\u001b[1;32m     72\u001b[0m tensor2 \u001b[39m=\u001b[39m atleast_1d(tensor2)\n\u001b[1;32m     74\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(tensor1\u001b[39m.\u001b[39mshape) \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m \u001b[39mor\u001b[39;00m tensor1\u001b[39m.\u001b[39mshape[\u001b[39m1\u001b[39m] \u001b[39m==\u001b[39m tensor2\u001b[39m.\u001b[39mshape[\u001b[39m1\u001b[39m]:\n\u001b[0;32m---> 75\u001b[0m     \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39;49mcat((tensor1, tensor2), dim\u001b[39m=\u001b[39;49m\u001b[39m0\u001b[39;49m)\n\u001b[1;32m     77\u001b[0m \u001b[39m# Let's figure out the new shape\u001b[39;00m\n\u001b[1;32m     78\u001b[0m new_shape \u001b[39m=\u001b[39m (tensor1\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m] \u001b[39m+\u001b[39m tensor2\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m], \u001b[39mmax\u001b[39m(tensor1\u001b[39m.\u001b[39mshape[\u001b[39m1\u001b[39m], tensor2\u001b[39m.\u001b[39mshape[\u001b[39m1\u001b[39m])) \u001b[39m+\u001b[39m tensor1\u001b[39m.\u001b[39mshape[\u001b[39m2\u001b[39m:]\n",
      "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 916.00 MiB (GPU 0; 23.69 GiB total capacity; 19.54 GiB already allocated; 859.00 MiB free; 21.44 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
     ]
    }
   ],
   "source": [
    "# max_steps = math.ceil(training_args.num_train_epochs * len(ds['train']) / training_args.gradient_accumulation_steps / min(training_args.per_device_train_batch_size, len(ds['train'])))\n",
    "# optimizer = Ranger21(model.parameters(), num_iterations=max_steps, lr=1e-4)\n",
    "# optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, betas=(0.9, 0.98), eps=1e-8, foreach=False)  # https://github.com/facebookresearch/fairseq/blob/main/examples/wav2vec/config/finetuning/base_960h.yaml\n",
    "# scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max_steps)\n",
    "# scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=tri_stage_schedule)  # following FAIR finetuning settings\n",
    "# scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda x: x)  # constant LR, stays same throughout, for Ranger21\n",
    "# torch._dynamo.config.suppress_errors = True\n",
    "# torch._dynamo.config.verbose=True\n",
    "trainer = CustomWhisperTrainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=ds['train'],\n",
    "    eval_dataset=ds['test'],\n",
    "    tokenizer=processor.feature_extractor,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    "    # optimizers=(optimizer, scheduler),\n",
    ")\n",
    "if os.name != 'nt':  # windows does not support torch.compile yet\n",
    "    trainer.model_wrapped, trainer.optimizer, trainer.lr_scheduler = accelerator.prepare(trainer.model_wrapped, trainer.optimizer, trainer.lr_scheduler)\n",
    "trainer.train()\n",
    "if os.name != 'nt':\n",
    "    accelerator.wait_for_everyone()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# INFERENCE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-15T10:54:22.540466800Z",
     "start_time": "2023-05-15T10:54:10.025365900Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alc/venvs/til2023/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['whisper-checkpoints/checkpoint-1031/preprocessor_config.json']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import WhisperProcessor\n",
    "processor = WhisperProcessor.from_pretrained('openai/whisper-small.en')\n",
    "processor.tokenizer.save_pretrained('whisper-checkpoints/checkpoint-1031/')\n",
    "processor.feature_extractor.save_pretrained('whisper-checkpoints/checkpoint-1031/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T10:52:01.892552800Z",
     "start_time": "2023-05-20T10:51:54.231627400Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alc/venvs/til2023/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "# Infer\n",
    "import os\n",
    "os.environ['HF_HOME'] = 'huggingface'\n",
    "os.environ['HF_HUB_DISABLE_TELEMETRY'] = '1'\n",
    "os.environ['HF_HUB_ENABLE_HF_TRANSFER'] = 'True'\n",
    "import torch\n",
    "import datasets\n",
    "from transformers import WhisperProcessor, WhisperForConditionalGeneration\n",
    "from transformers import pipeline\n",
    "from transformers.pipelines.pt_utils import KeyDataset\n",
    "from tqdm.auto import tqdm\n",
    "import pandas as pd\n",
    "from torch.utils.data import DataLoader\n",
    "torch.multiprocessing.set_start_method('spawn')\n",
    "\n",
    "def clean(annotation):\n",
    "    if \"'\" in annotation:\n",
    "        # print(annotation, f'has \\' in {annotation}, removing')\n",
    "        annotation = annotation.split(\"'\")[0] + annotation.split(\"'\")[1][1:]  # Tokenizer includes \"'\" but TIL dataset does not, remove the S following '\n",
    "    return annotation\n",
    "\n",
    "transcriber = pipeline(\"automatic-speech-recognition\", model='whisper-checkpoints/checkpoint-1031', device=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-05-20T10:52:10.814054900Z",
     "start_time": "2023-05-20T10:52:10.799056200Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Resolving data files: 100%|██████████| 12000/12000 [00:00<00:00, 26027.75it/s]\n",
      "Found cached dataset audiofolder (/home/alc/TIL-2023/ASR/huggingface/datasets/audiofolder/test-b719a705ed310f32/0.0.0/6cbdd16f8688354c63b4e2a36e1585d05de285023ee6443ffd71c4182055c0fc)\n"
     ]
    }
   ],
   "source": [
    "dataset = datasets.load_dataset(\"test\", split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/12000 [00:00<?, ?it/s]/home/alc/venvs/til2023/lib/python3.10/site-packages/transformers/generation/utils.py:1346: UserWarning: Using `max_length`'s default (448) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n",
      "100%|██████████| 12000/12000 [08:50<00:00, 22.61it/s]\n"
     ]
    }
   ],
   "source": [
    "test_ds = pd.read_csv('Test_Advanced.csv')\n",
    "results = []\n",
    "for out in tqdm(transcriber(KeyDataset(dataset, \"audio\"), batch_size=80), total=len(test_ds)):\n",
    "    results.append(clean(out['text']))\n",
    "test_ds['annotation'] = results\n",
    "test_ds['path'] = test_ds['path'].apply(lambda x: x.split('/')[-1])\n",
    "test_ds.to_csv('whisper-submissions/1031 build in aug raw 0.00336.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}