guides/eval_benchmark/kerascv-kinetics-400-evaluation-in-pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1eaf876c",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-04-03T06:51:23.922069Z",
     "iopub.status.busy": "2024-04-03T06:51:23.921347Z",
     "iopub.status.idle": "2024-04-03T06:51:38.385947Z",
     "shell.execute_reply": "2024-04-03T06:51:38.384845Z"
    },
    "papermill": {
     "duration": 14.478032,
     "end_time": "2024-04-03T06:51:38.388525",
     "exception": false,
     "start_time": "2024-04-03T06:51:23.910493",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install decord -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4dbec267",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:51:38.408525Z",
     "iopub.status.busy": "2024-04-03T06:51:38.408119Z",
     "iopub.status.idle": "2024-04-03T06:52:09.447505Z",
     "shell.execute_reply": "2024-04-03T06:52:09.446420Z"
    },
    "papermill": {
     "duration": 31.052127,
     "end_time": "2024-04-03T06:52:09.449916",
     "exception": false,
     "start_time": "2024-04-03T06:51:38.397789",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'keras-cv'...\r\n",
      "remote: Enumerating objects: 13782, done.\u001b[K\r\n",
      "remote: Counting objects: 100% (1919/1919), done.\u001b[K\r\n",
      "remote: Compressing objects: 100% (769/769), done.\u001b[K\r\n",
      "remote: Total 13782 (delta 1337), reused 1628 (delta 1134), pack-reused 11863\u001b[K\r\n",
      "Receiving objects: 100% (13782/13782), 25.65 MiB | 27.53 MiB/s, done.\r\n",
      "Resolving deltas: 100% (9788/9788), done.\r\n",
      "/kaggle/working/keras-cv\n"
     ]
    }
   ],
   "source": [
    "!git clone --branch video_swin https://github.com/innat/keras-cv.git\n",
    "%cd keras-cv\n",
    "!pip install -q -e ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1d893c5b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:52:09.472818Z",
     "iopub.status.busy": "2024-04-03T06:52:09.472498Z",
     "iopub.status.idle": "2024-04-03T06:52:09.477540Z",
     "shell.execute_reply": "2024-04-03T06:52:09.476554Z"
    },
    "papermill": {
     "duration": 0.018822,
     "end_time": "2024-04-03T06:52:09.479574",
     "exception": false,
     "start_time": "2024-04-03T06:52:09.460752",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os, warnings\n",
    "os.environ[\"KERAS_BACKEND\"] = \"torch\" \n",
    "warnings.simplefilter(action=\"ignore\")\n",
    "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ef3dd5d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:52:09.502723Z",
     "iopub.status.busy": "2024-04-03T06:52:09.502106Z",
     "iopub.status.idle": "2024-04-03T06:52:30.729858Z",
     "shell.execute_reply": "2024-04-03T06:52:30.728816Z"
    },
    "papermill": {
     "duration": 21.241686,
     "end_time": "2024-04-03T06:52:30.732079",
     "exception": false,
     "start_time": "2024-04-03T06:52:09.490393",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('3.0.5', '2.1.2')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, sys\n",
    "import cv2\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from tqdm import tqdm\n",
    "from decord import VideoReader\n",
    "from decord import cpu, gpu\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "import keras\n",
    "from keras import ops\n",
    "from keras_cv.models import VideoSwinBackbone\n",
    "from keras_cv.models import VideoClassifier\n",
    "\n",
    "keras.__version__, torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "156d3160",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:52:30.754915Z",
     "iopub.status.busy": "2024-04-03T06:52:30.754366Z",
     "iopub.status.idle": "2024-04-03T06:52:30.760351Z",
     "shell.execute_reply": "2024-04-03T06:52:30.759474Z"
    },
    "papermill": {
     "duration": 0.019592,
     "end_time": "2024-04-03T06:52:30.762442",
     "exception": false,
     "start_time": "2024-04-03T06:52:30.742850",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch.autograd.grad_mode.set_grad_enabled at 0x7d9cfdd7eb60>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.set_grad_enabled(False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c58e1f41",
   "metadata": {
    "papermill": {
     "duration": 0.010719,
     "end_time": "2024-04-03T06:52:30.784957",
     "exception": false,
     "start_time": "2024-04-03T06:52:30.774238",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "57a701bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:52:30.807733Z",
     "iopub.status.busy": "2024-04-03T06:52:30.807420Z",
     "iopub.status.idle": "2024-04-03T06:52:31.980326Z",
     "shell.execute_reply": "2024-04-03T06:52:31.979189Z"
    },
    "papermill": {
     "duration": 1.186849,
     "end_time": "2024-04-03T06:52:31.982494",
     "exception": false,
     "start_time": "2024-04-03T06:52:30.795645",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>abseiling</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>air drumming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>answering questions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>applauding</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>applying cream</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                 name\n",
       "0   0            abseiling\n",
       "1   1         air drumming\n",
       "2   2  answering questions\n",
       "3   3           applauding\n",
       "4   4       applying cream"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/innat/VideoSwin/main/data/kinetics_400_labels.csv -q\n",
    "labels = pd.read_csv('kinetics_400_labels.csv')\n",
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8d4f3c4c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:52:32.007009Z",
     "iopub.status.busy": "2024-04-03T06:52:32.006111Z",
     "iopub.status.idle": "2024-04-03T06:52:32.012778Z",
     "shell.execute_reply": "2024-04-03T06:52:32.011933Z"
    },
    "papermill": {
     "duration": 0.02052,
     "end_time": "2024-04-03T06:52:32.014633",
     "exception": false,
     "start_time": "2024-04-03T06:52:31.994113",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "id2label = dict(zip(labels.id.tolist(), labels.name.tolist()))\n",
    "label2id = dict(zip(labels.name.tolist(), labels.id.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c43413c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:52:32.038344Z",
     "iopub.status.busy": "2024-04-03T06:52:32.037721Z",
     "iopub.status.idle": "2024-04-03T06:53:51.132927Z",
     "shell.execute_reply": "2024-04-03T06:53:51.131886Z"
    },
    "papermill": {
     "duration": 79.144924,
     "end_time": "2024-04-03T06:53:51.170549",
     "exception": false,
     "start_time": "2024-04-03T06:52:32.025625",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19796it [00:30, 659.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19796, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_path</th>\n",
       "      <th>label</th>\n",
       "      <th>string_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/kaggle/input/k4testset/videos_val/jf7RDuUTrsQ...</td>\n",
       "      <td>325</td>\n",
       "      <td>somersaulting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/kaggle/input/k4testset/videos_val/JTlatknwOrY...</td>\n",
       "      <td>233</td>\n",
       "      <td>playing harmonica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/kaggle/input/k4testset/videos_val/8UxlDNur-Z0...</td>\n",
       "      <td>262</td>\n",
       "      <td>pushing cart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/kaggle/input/k4testset/videos_val/y9r115bgfNk...</td>\n",
       "      <td>320</td>\n",
       "      <td>sniffing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/kaggle/input/k4testset/videos_val/ZnIDviwA8CE...</td>\n",
       "      <td>244</td>\n",
       "      <td>playing saxophone</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          video_path  label       string_label\n",
       "0  /kaggle/input/k4testset/videos_val/jf7RDuUTrsQ...    325      somersaulting\n",
       "1  /kaggle/input/k4testset/videos_val/JTlatknwOrY...    233  playing harmonica\n",
       "2  /kaggle/input/k4testset/videos_val/8UxlDNur-Z0...    262       pushing cart\n",
       "3  /kaggle/input/k4testset/videos_val/y9r115bgfNk...    320           sniffing\n",
       "4  /kaggle/input/k4testset/videos_val/ZnIDviwA8CE...    244  playing saxophone"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process_data(text_file_path, data_folder_path, n=10):\n",
    "    video_paths = []\n",
    "    labels = []\n",
    "    string_labels = []\n",
    "\n",
    "    # Get all video filenames from the data folder\n",
    "    all_files_in_data_folder = [\n",
    "        f for f in os.listdir(data_folder_path) \n",
    "        if os.path.isfile(os.path.join(data_folder_path, f))\n",
    "    ]\n",
    "    with open(text_file_path, 'r') as f:\n",
    "        for line in tqdm(f):\n",
    "            parts = line.strip().split()\n",
    "            if len(parts) == 2:\n",
    "                filename, label = parts\n",
    "                search_string = filename[-n:]\n",
    "                matching_file = next(\n",
    "                    (\n",
    "                        f for f in all_files_in_data_folder \n",
    "                        if f.endswith(search_string)\n",
    "                    ), None\n",
    "                )\n",
    "                if matching_file:\n",
    "                    abs_path = os.path.join(data_folder_path, matching_file)\n",
    "                    video_paths.append(abs_path)\n",
    "                    labels.append(int(label))\n",
    "                    string_labels.append(id2label[int(label)])\n",
    "                    \n",
    "    df = pd.DataFrame({\n",
    "        'video_path': video_paths,\n",
    "        'label': labels,\n",
    "        'string_label': string_labels\n",
    "    })\n",
    "    return df\n",
    "\n",
    "# Example usage:\n",
    "text_file_path = \"/kaggle/input/k4testset/kinetics400_val_list_videos.txt\"\n",
    "data_folder_path = \"/kaggle/input/k4testset/videos_val\"\n",
    "df = process_data(text_file_path, data_folder_path)\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27bd6782",
   "metadata": {
    "papermill": {
     "duration": 0.033667,
     "end_time": "2024-04-03T06:53:51.238227",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.204560",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data Loader\n",
    "\n",
    "To build the dataloader, we will be using [mmaction](https://mmaction2.readthedocs.io/en/latest/index.html) recipe. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "27c2fd34",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:51.314733Z",
     "iopub.status.busy": "2024-04-03T06:53:51.313960Z",
     "iopub.status.idle": "2024-04-03T06:53:51.320470Z",
     "shell.execute_reply": "2024-04-03T06:53:51.319317Z"
    },
    "papermill": {
     "duration": 0.047102,
     "end_time": "2024-04-03T06:53:51.322763",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.275661",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class VideoInit:\n",
    "    def transform(self, results):\n",
    "        container = VideoReader(results['filename'])\n",
    "        results['total_frames'] = len(container)\n",
    "        results['video_reader'] = container\n",
    "        results['avg_fps'] = container.get_avg_fps()\n",
    "        results['start_index'] = 0\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1091a7da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:51.403976Z",
     "iopub.status.busy": "2024-04-03T06:53:51.403564Z",
     "iopub.status.idle": "2024-04-03T06:53:51.438391Z",
     "shell.execute_reply": "2024-04-03T06:53:51.437394Z"
    },
    "papermill": {
     "duration": 0.079147,
     "end_time": "2024-04-03T06:53:51.440585",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.361438",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class VideoSample:\n",
    "    \"\"\"Sample frames from the video.\n",
    "\n",
    "    Required keys are \"total_frames\", \"start_index\" , added or modified keys\n",
    "    are \"frame_inds\", \"frame_interval\" and \"num_clips\".\n",
    "\n",
    "    Args:\n",
    "        clip_len (int): Frames of each sampled output clip.\n",
    "        frame_interval (int): Temporal interval of adjacent sampled frames.\n",
    "            Default: 1.\n",
    "        num_clips (int): Number of clips to be sampled. Default: 1.\n",
    "        temporal_jitter (bool): Whether to apply temporal jittering.\n",
    "            Default: False.\n",
    "        twice_sample (bool): Whether to use twice sample when testing.\n",
    "            If set to True, it will sample frames with and without fixed shift,\n",
    "            which is commonly used for testing in TSM model. Default: False.\n",
    "        out_of_bound_opt (str): The way to deal with out of bounds frame\n",
    "            indexes. Available options are 'loop', 'repeat_last'.\n",
    "            Default: 'loop'.\n",
    "        test_mode (bool): Store True when building test or validation dataset.\n",
    "            Default: False.\n",
    "        start_index (None): This argument is deprecated and moved to dataset\n",
    "            class (``BaseDataset``, ``VideoDatset``, ``RawframeDataset``, etc),\n",
    "            see this: https://github.com/open-mmlab/mmaction2/pull/89.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        clip_len,\n",
    "        frame_interval=1,\n",
    "        num_clips=1,\n",
    "        temporal_jitter=False,\n",
    "        twice_sample=False,\n",
    "        out_of_bound_opt='loop',\n",
    "        test_mode=False,\n",
    "        start_index=None,\n",
    "        frame_uniform=False,\n",
    "        multiview=1\n",
    "    ):\n",
    "        self.clip_len = clip_len\n",
    "        self.frame_interval = frame_interval\n",
    "        self.num_clips = num_clips\n",
    "        self.temporal_jitter = temporal_jitter\n",
    "        self.twice_sample = twice_sample\n",
    "        self.out_of_bound_opt = out_of_bound_opt\n",
    "        self.test_mode = test_mode\n",
    "        self.frame_uniform = frame_uniform\n",
    "        self.multiview=multiview\n",
    "        assert self.out_of_bound_opt in ['loop', 'repeat_last']\n",
    "\n",
    "        if start_index is not None:\n",
    "            warnings.warn(\n",
    "                'No longer support \"start_index\" in \"SampleFrames\", '\n",
    "                'it should be set in dataset class, see this pr: '\n",
    "                'https://github.com/open-mmlab/mmaction2/pull/89'\n",
    "            )\n",
    "\n",
    "    def _get_train_clips(self, num_frames):\n",
    "        \"\"\"Get clip offsets in train mode.\n",
    "\n",
    "        It will calculate the average interval for selected frames,\n",
    "        and randomly shift them within offsets between [0, avg_interval].\n",
    "        If the total number of frames is smaller than clips num or origin\n",
    "        frames length, it will return all zero indices.\n",
    "\n",
    "        Args:\n",
    "            num_frames (int): Total number of frame in the video.\n",
    "\n",
    "        Returns:\n",
    "            np.ndarray: Sampled frame indices in train mode.\n",
    "        \"\"\"\n",
    "        ori_clip_len = self.clip_len * self.frame_interval\n",
    "        avg_interval = (num_frames - ori_clip_len + 1) // self.num_clips\n",
    "\n",
    "        if avg_interval > 0:\n",
    "            base_offsets = np.arange(self.num_clips) * avg_interval\n",
    "            clip_offsets = base_offsets + np.random.randint(\n",
    "                avg_interval, size=self.num_clips)\n",
    "        elif num_frames > max(self.num_clips, ori_clip_len):\n",
    "            clip_offsets = np.sort(\n",
    "                np.random.randint(\n",
    "                    num_frames - ori_clip_len + 1, size=self.num_clips))\n",
    "        elif avg_interval == 0:\n",
    "            ratio = (num_frames - ori_clip_len + 1.0) / self.num_clips\n",
    "            clip_offsets = np.around(np.arange(self.num_clips) * ratio)\n",
    "        else:\n",
    "            clip_offsets = np.zeros((self.num_clips, ), dtype=np.int32)\n",
    "\n",
    "        return clip_offsets\n",
    "\n",
    "    def _get_test_clips(self, num_frames):\n",
    "        \"\"\"Get clip offsets in test mode.\n",
    "\n",
    "        Calculate the average interval for selected frames, and shift them\n",
    "        fixedly by avg_interval/2. If set twice_sample True, it will sample\n",
    "        frames together without fixed shift. If the total number of frames is\n",
    "        not enough, it will return all zero indices.\n",
    "\n",
    "        Args:\n",
    "            num_frames (int): Total number of frame in the video.\n",
    "\n",
    "        Returns:\n",
    "            np.ndarray: Sampled frame indices in test mode.\n",
    "        \"\"\"\n",
    "        ori_clip_len = self.clip_len * self.frame_interval\n",
    "        avg_interval = (num_frames - ori_clip_len + 1) / float(self.num_clips)\n",
    "        if num_frames > ori_clip_len - 1:\n",
    "            base_offsets = np.arange(self.num_clips) * avg_interval\n",
    "            clip_offsets = (base_offsets + avg_interval / 2.0).astype(np.int32)\n",
    "            if self.twice_sample:\n",
    "                clip_offsets = np.concatenate([clip_offsets, base_offsets])\n",
    "        else:\n",
    "            clip_offsets = np.zeros((self.num_clips, ), dtype=np.int32)\n",
    "        return clip_offsets\n",
    "\n",
    "    def _sample_clips(self, num_frames):\n",
    "        \"\"\"Choose clip offsets for the video in a given mode.\n",
    "\n",
    "        Args:\n",
    "            num_frames (int): Total number of frame in the video.\n",
    "\n",
    "        Returns:\n",
    "            np.ndarray: Sampled frame indices.\n",
    "        \"\"\"\n",
    "        if self.test_mode:\n",
    "            clip_offsets = self._get_test_clips(num_frames)\n",
    "        else:\n",
    "            if self.multiview == 1:\n",
    "                clip_offsets = self._get_train_clips(num_frames)\n",
    "            else:\n",
    "                clip_offsets = np.concatenate(\n",
    "                    [\n",
    "                        self._get_train_clips(num_frames)  \n",
    "                        for _ in range(self.multiview)\n",
    "                    ]\n",
    "                )\n",
    "        return clip_offsets\n",
    "\n",
    "    def get_seq_frames(self, num_frames):\n",
    "        seg_size = float(num_frames - 1) / self.clip_len\n",
    "        seq = []\n",
    "        for i in range(self.clip_len):\n",
    "            start = int(np.round(seg_size * i))\n",
    "            end = int(np.round(seg_size * (i + 1)))\n",
    "            if not self.test_mode:\n",
    "                seq.append(random.randint(start, end))\n",
    "            else:\n",
    "                seq.append((start + end) // 2)\n",
    "\n",
    "        return np.array(seq)\n",
    "\n",
    "    def transform(self, results):\n",
    "        \"\"\"Perform the SampleFrames loading.\n",
    "\n",
    "        Args:\n",
    "            results (dict): The resulting dict to be modified and passed\n",
    "                to the next transform in pipeline.\n",
    "        \"\"\"\n",
    "        total_frames = results['total_frames']\n",
    "        if self.frame_uniform:  # sthv2 sampling strategy\n",
    "            assert results['start_index'] == 0\n",
    "            frame_inds = self.get_seq_frames(total_frames)\n",
    "        else:\n",
    "            clip_offsets = self._sample_clips(total_frames)\n",
    "            frame_inds = clip_offsets[:, None] + np.arange(\n",
    "                self.clip_len)[None, :] * self.frame_interval\n",
    "            frame_inds = np.concatenate(frame_inds)\n",
    "\n",
    "            if self.temporal_jitter:\n",
    "                perframe_offsets = np.random.randint(\n",
    "                    self.frame_interval, size=len(frame_inds))\n",
    "                frame_inds += perframe_offsets\n",
    "\n",
    "            frame_inds = frame_inds.reshape((-1, self.clip_len))\n",
    "            if self.out_of_bound_opt == 'loop':\n",
    "                frame_inds = np.mod(frame_inds, total_frames)\n",
    "            elif self.out_of_bound_opt == 'repeat_last':\n",
    "                safe_inds = frame_inds < total_frames\n",
    "                unsafe_inds = 1 - safe_inds\n",
    "                last_ind = np.max(safe_inds * frame_inds, axis=1)\n",
    "                new_inds = (safe_inds * frame_inds + (unsafe_inds.T * last_ind).T)\n",
    "                frame_inds = new_inds\n",
    "            else:\n",
    "                raise ValueError('Illegal out_of_bound option.')\n",
    "\n",
    "            start_index = results['start_index']\n",
    "            frame_inds = np.concatenate(frame_inds) + start_index\n",
    "\n",
    "        results['frame_inds'] = frame_inds.astype(np.int32)\n",
    "        results['clip_len'] = self.clip_len\n",
    "        results['frame_interval'] = self.frame_interval\n",
    "        results['num_clips'] = self.num_clips\n",
    "        return results\n",
    "\n",
    "    def __repr__(self):\n",
    "        repr_str = (f'{self.__class__.__name__}('\n",
    "                    f'clip_len={self.clip_len}, '\n",
    "                    f'frame_interval={self.frame_interval}, '\n",
    "                    f'num_clips={self.num_clips}, '\n",
    "                    f'temporal_jitter={self.temporal_jitter}, '\n",
    "                    f'twice_sample={self.twice_sample}, '\n",
    "                    f'out_of_bound_opt={self.out_of_bound_opt}, '\n",
    "                    f'test_mode={self.test_mode})')\n",
    "        return repr_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "671e47f5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:51.521387Z",
     "iopub.status.busy": "2024-04-03T06:53:51.520881Z",
     "iopub.status.idle": "2024-04-03T06:53:51.527364Z",
     "shell.execute_reply": "2024-04-03T06:53:51.526390Z"
    },
    "papermill": {
     "duration": 0.049057,
     "end_time": "2024-04-03T06:53:51.529422",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.480365",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class VideoDecode:\n",
    "    def transform(self, results):\n",
    "        frame_inds = results['frame_inds']\n",
    "        container = results['video_reader']\n",
    "        imgs = container.get_batch(frame_inds).asnumpy()\n",
    "        imgs = list(imgs)\n",
    "        results['video_reader'] = None\n",
    "        del container\n",
    "        results['imgs'] = imgs\n",
    "        results['img_shape'] = imgs[0].shape[:2]\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d25b4c36",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:51.607530Z",
     "iopub.status.busy": "2024-04-03T06:53:51.607080Z",
     "iopub.status.idle": "2024-04-03T06:53:51.620583Z",
     "shell.execute_reply": "2024-04-03T06:53:51.619446Z"
    },
    "papermill": {
     "duration": 0.056298,
     "end_time": "2024-04-03T06:53:51.623345",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.567047",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def _scale_size(\n",
    "    size,\n",
    "    scale,\n",
    "):\n",
    "    if isinstance(scale, (float, int)):\n",
    "        scale = (scale, scale)\n",
    "    w, h = size\n",
    "    return int(w * float(scale[0]) + 0.5), int(h * float(scale[1]) + 0.5)\n",
    "\n",
    "def rescale_size(\n",
    "    old_size,\n",
    "    scale,\n",
    "    return_scale=False\n",
    "):\n",
    "    w, h = old_size\n",
    "    if isinstance(scale, (float, int)):\n",
    "        if scale <= 0:\n",
    "            raise ValueError(f'Invalid scale {scale}, must be positive.')\n",
    "        scale_factor = scale\n",
    "    elif isinstance(scale, tuple):\n",
    "        max_long_edge = max(scale)\n",
    "        max_short_edge = min(scale)\n",
    "        scale_factor = min(\n",
    "            max_long_edge / max(h, w),\n",
    "            max_short_edge / min(h, w)\n",
    "        )\n",
    "    else:\n",
    "        raise TypeError(\n",
    "            f'Scale must be a number or tuple of int, but got {type(scale)}'\n",
    "        )\n",
    "\n",
    "    new_size = _scale_size((w, h), scale_factor)\n",
    "\n",
    "    if return_scale:\n",
    "        return new_size, scale_factor\n",
    "    else:\n",
    "        return new_size\n",
    "\n",
    "class VideoResize:\n",
    "    def __init__(self, r_size):\n",
    "        self.r_size = (np.inf, r_size)\n",
    "\n",
    "    def transform(self, results):\n",
    "        img_h, img_w = results['img_shape']\n",
    "        new_w, new_h = rescale_size((img_w, img_h), self.r_size)\n",
    "\n",
    "        imgs = [\n",
    "            cv2.resize(img, (new_w, new_h))\n",
    "            for img in results['imgs']\n",
    "        ]\n",
    "        results['imgs'] = imgs\n",
    "        results['img_shape'] = imgs[0].shape[:2]\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "46d31ee2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:51.702400Z",
     "iopub.status.busy": "2024-04-03T06:53:51.701648Z",
     "iopub.status.idle": "2024-04-03T06:53:51.708951Z",
     "shell.execute_reply": "2024-04-03T06:53:51.707952Z"
    },
    "papermill": {
     "duration": 0.048056,
     "end_time": "2024-04-03T06:53:51.711094",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.663038",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class VideoCrop:\n",
    "    def __init__(self, c_size):\n",
    "        self.c_size = c_size\n",
    "\n",
    "    def transform(self, results):\n",
    "        img_h, img_w = results['img_shape']\n",
    "        center_x, center_y = img_w // 2, img_h // 2\n",
    "        x1, x2 = center_x - self.c_size // 2, center_x + self.c_size // 2\n",
    "        y1, y2 = center_y - self.c_size // 2, center_y + self.c_size // 2\n",
    "        imgs = [img[y1:y2, x1:x2] for img in results['imgs']]\n",
    "        results['imgs'] = imgs\n",
    "        results['img_shape'] = imgs[0].shape[:2]\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9c88cf88",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:51.782601Z",
     "iopub.status.busy": "2024-04-03T06:53:51.781764Z",
     "iopub.status.idle": "2024-04-03T06:53:51.787948Z",
     "shell.execute_reply": "2024-04-03T06:53:51.786987Z"
    },
    "papermill": {
     "duration": 0.04297,
     "end_time": "2024-04-03T06:53:51.789826",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.746856",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class VideoFormat:\n",
    "    def transform(self, results):\n",
    "        num_clips = results['num_clips']\n",
    "        clip_len = results['clip_len']\n",
    "        imgs = results['imgs']\n",
    "        # [num_clips*clip_len, H, W, C]\n",
    "        imgs = np.array(imgs)\n",
    "        # [num_clips, clip_len, H, W, C]\n",
    "        imgs = imgs.reshape((num_clips, clip_len) + imgs.shape[1:])\n",
    "        results['imgs'] = imgs\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a0124083",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:51.909478Z",
     "iopub.status.busy": "2024-04-03T06:53:51.909112Z",
     "iopub.status.idle": "2024-04-03T06:53:52.048494Z",
     "shell.execute_reply": "2024-04-03T06:53:52.047019Z"
    },
    "papermill": {
     "duration": 0.17638,
     "end_time": "2024-04-03T06:53:52.050476",
     "exception": false,
     "start_time": "2024-04-03T06:53:51.874096",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['filename', 'total_frames', 'video_reader', 'avg_fps', 'start_index', 'frame_inds', 'clip_len', 'frame_interval', 'num_clips', 'imgs', 'img_shape'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1, 16, 224, 224, 3)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item = dict()\n",
    "item['filename'] = '/kaggle/input/k4testset/videos_val/--07WQ2iBlw.mp4'\n",
    "v_init   = VideoInit().transform(item)\n",
    "v_sample = VideoSample(clip_len=16, num_clips=1, test_mode=True).transform(v_init)\n",
    "v_decode = VideoDecode().transform(v_sample)\n",
    "v_resize = VideoResize(r_size=256).transform(v_decode)\n",
    "v_crop   = VideoCrop(c_size=224).transform(v_resize)\n",
    "v_format = VideoFormat().transform(v_crop)\n",
    "print(v_format.keys())\n",
    "v_format['imgs'].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be48ff6d",
   "metadata": {
    "papermill": {
     "duration": 0.033437,
     "end_time": "2024-04-03T06:53:52.118351",
     "exception": false,
     "start_time": "2024-04-03T06:53:52.084914",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "[inference-config-video-swin](https://github.com/SwinTransformer/Video-Swin-Transformer/blob/db018fb8896251711791386bbd2127562fd8d6a6/configs/recognition/swin/swin_tiny_patch244_window877_kinetics400_1k.py#L45-L61)\n",
    "\n",
    "```python\n",
    "test_pipeline = [\n",
    "    dict(type='DecordInit'),\n",
    "    dict(\n",
    "        type='SampleFrames',\n",
    "        clip_len=32,\n",
    "        frame_interval=2,\n",
    "        num_clips=4,\n",
    "        test_mode=True),\n",
    "    dict(type='DecordDecode'),\n",
    "    dict(type='Resize', scale=(-1, 224)),\n",
    "    dict(type='ThreeCrop', crop_size=224),\n",
    "    dict(type='Flip', flip_ratio=0),\n",
    "    dict(type='Normalize', **img_norm_cfg),\n",
    "    dict(type='FormatShape', input_format='NCTHW'),\n",
    "    dict(type='Collect', keys=['imgs', 'label'], meta_keys=[]),\n",
    "    dict(type='ToTensor', keys=['imgs'])\n",
    "]\n",
    "```\n",
    "\n",
    "We will skip `ThreeCrop` and `Flip` at the moment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b62d1649",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:52.186989Z",
     "iopub.status.busy": "2024-04-03T06:53:52.186528Z",
     "iopub.status.idle": "2024-04-03T06:53:52.191589Z",
     "shell.execute_reply": "2024-04-03T06:53:52.190623Z"
    },
    "papermill": {
     "duration": 0.041923,
     "end_time": "2024-04-03T06:53:52.193583",
     "exception": false,
     "start_time": "2024-04-03T06:53:52.151660",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "num_classes=400\n",
    "batch_size=16\n",
    "num_clips=4\n",
    "frame_rate=2 \n",
    "input_frame=32\n",
    "h_crop_size=w_crop_size=224"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "38ad84b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:52.266322Z",
     "iopub.status.busy": "2024-04-03T06:53:52.265343Z",
     "iopub.status.idle": "2024-04-03T06:53:52.275520Z",
     "shell.execute_reply": "2024-04-03T06:53:52.274464Z"
    },
    "papermill": {
     "duration": 0.049125,
     "end_time": "2024-04-03T06:53:52.277732",
     "exception": false,
     "start_time": "2024-04-03T06:53:52.228607",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class VideoDataset(Dataset):\n",
    "    def __init__(self, dataframe, clip_len=1, frame_sample_rate=8):\n",
    "        self.dataframe = dataframe\n",
    "        self.clip_len = clip_len\n",
    "        self.frame_sample_rate = frame_sample_rate\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataframe)\n",
    "\n",
    "    def get_frames(self, video_path):\n",
    "        item = dict()\n",
    "        item['filename'] = video_path\n",
    "        v_init   = VideoInit().transform(item)\n",
    "        v_sample = VideoSample(\n",
    "            clip_len=input_frame, \n",
    "            num_clips=num_clips, \n",
    "            frame_interval=frame_rate,\n",
    "            test_mode=True\n",
    "        ).transform(v_init)\n",
    "        v_decode = VideoDecode().transform(v_sample)\n",
    "        v_resize = VideoResize(r_size=256).transform(v_decode)\n",
    "        v_crop   = VideoCrop(c_size=224).transform(v_resize)\n",
    "        v_format = VideoFormat().transform(v_crop)\n",
    "        frames = v_format['imgs']\n",
    "        return frames\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        video_path = self.dataframe.iloc[idx, 0]\n",
    "        label = self.dataframe.iloc[idx, 1]\n",
    "        video = self.get_frames(video_path)\n",
    "        return torch.tensor(video).to(torch.float32), torch.tensor(label).to(torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "094a3a1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:52.350322Z",
     "iopub.status.busy": "2024-04-03T06:53:52.349529Z",
     "iopub.status.idle": "2024-04-03T06:53:52.355074Z",
     "shell.execute_reply": "2024-04-03T06:53:52.354169Z"
    },
    "papermill": {
     "duration": 0.044263,
     "end_time": "2024-04-03T06:53:52.357228",
     "exception": false,
     "start_time": "2024-04-03T06:53:52.312965",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset = VideoDataset(\n",
    "    dataframe=df,  \n",
    "    clip_len=num_clips, frame_sample_rate=frame_rate\n",
    ")\n",
    "dataloader = DataLoader(\n",
    "    dataset, \n",
    "    batch_size=batch_size, \n",
    "    shuffle=False, \n",
    "    pin_memory=True, \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "063e4cf8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:53:52.428604Z",
     "iopub.status.busy": "2024-04-03T06:53:52.427956Z",
     "iopub.status.idle": "2024-04-03T06:54:15.228214Z",
     "shell.execute_reply": "2024-04-03T06:54:15.227198Z"
    },
    "papermill": {
     "duration": 22.839482,
     "end_time": "2024-04-03T06:54:15.230535",
     "exception": false,
     "start_time": "2024-04-03T06:53:52.391053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 4, 32, 224, 224, 3]) torch.Size([16])\n",
      "torch.Size([16, 4, 32, 224, 224, 3]) torch.Size([16])\n",
      "torch.Size([16, 4, 32, 224, 224, 3]) torch.Size([16])\n"
     ]
    }
   ],
   "source": [
    "for i, (videos, labels) in enumerate(dataloader):\n",
    "    print(videos.shape, labels.shape)\n",
    "    if i == 2 :\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1a09bd1",
   "metadata": {
    "papermill": {
     "duration": 0.034174,
     "end_time": "2024-04-03T06:54:15.299292",
     "exception": false,
     "start_time": "2024-04-03T06:54:15.265118",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f6472d84",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:54:15.370405Z",
     "iopub.status.busy": "2024-04-03T06:54:15.369675Z",
     "iopub.status.idle": "2024-04-03T06:54:17.324247Z",
     "shell.execute_reply": "2024-04-03T06:54:17.323102Z"
    },
    "papermill": {
     "duration": 1.993356,
     "end_time": "2024-04-03T06:54:17.326792",
     "exception": false,
     "start_time": "2024-04-03T06:54:15.333436",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!wget https://github.com/innat/VideoSwin/releases/download/v2.0/videoswin_tiny_kinetics400_classifier.weights.h5 -q\n",
    "\n",
    "def vswin_tiny():\n",
    "    backbone=VideoSwinBackbone(\n",
    "        input_shape=(32, 224, 224, 3), \n",
    "        embed_dim=96,\n",
    "        depths=[2, 2, 6, 2],\n",
    "        num_heads=[3, 6, 12, 24],\n",
    "        include_rescaling=True, \n",
    "    )\n",
    "    keras_model = VideoClassifier(\n",
    "        backbone=backbone,\n",
    "        num_classes=num_classes,\n",
    "        activation=None,\n",
    "        pooling='avg',\n",
    "    )\n",
    "    keras_model.load_weights(\n",
    "        'videoswin_tiny_kinetics400_classifier.weights.h5'\n",
    "    )\n",
    "    return keras_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a0a1a37c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:54:17.398302Z",
     "iopub.status.busy": "2024-04-03T06:54:17.397925Z",
     "iopub.status.idle": "2024-04-03T06:54:18.517080Z",
     "shell.execute_reply": "2024-04-03T06:54:18.516189Z"
    },
    "papermill": {
     "duration": 1.15803,
     "end_time": "2024-04-03T06:54:18.519232",
     "exception": false,
     "start_time": "2024-04-03T06:54:17.361202",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"video_classifier\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"video_classifier\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ videos (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">224</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">224</span>,   │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "│                                 │ <span style=\"color: #00af00; text-decoration-color: #00af00\">3</span>)                     │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ video_swin_backbone             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">7</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)  │    <span style=\"color: #00af00; text-decoration-color: #00af00\">27,850,470</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">VideoSwinBackbone</span>)             │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ avg_pool                        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">768</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalAveragePooling3D</span>)        │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ predictions (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">400</span>)            │       <span style=\"color: #00af00; text-decoration-color: #00af00\">307,600</span> │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
       "│ videos (\u001b[38;5;33mInputLayer\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m, \u001b[38;5;34m224\u001b[0m, \u001b[38;5;34m224\u001b[0m,   │             \u001b[38;5;34m0\u001b[0m │\n",
       "│                                 │ \u001b[38;5;34m3\u001b[0m)                     │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ video_swin_backbone             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m7\u001b[0m, \u001b[38;5;34m768\u001b[0m)  │    \u001b[38;5;34m27,850,470\u001b[0m │\n",
       "│ (\u001b[38;5;33mVideoSwinBackbone\u001b[0m)             │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ avg_pool                        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m768\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n",
       "│ (\u001b[38;5;33mGlobalAveragePooling3D\u001b[0m)        │                        │               │\n",
       "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
       "│ predictions (\u001b[38;5;33mDense\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m400\u001b[0m)            │       \u001b[38;5;34m307,600\u001b[0m │\n",
       "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">28,158,070</span> (107.41 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m28,158,070\u001b[0m (107.41 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">28,158,070</span> (107.41 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m28,158,070\u001b[0m (107.41 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = vswin_tiny()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ea5b058",
   "metadata": {
    "papermill": {
     "duration": 0.035071,
     "end_time": "2024-04-03T06:54:18.591038",
     "exception": false,
     "start_time": "2024-04-03T06:54:18.555967",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Training API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5b1fbf80",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:54:18.664301Z",
     "iopub.status.busy": "2024-04-03T06:54:18.663523Z",
     "iopub.status.idle": "2024-04-03T06:54:18.669902Z",
     "shell.execute_reply": "2024-04-03T06:54:18.669012Z"
    },
    "papermill": {
     "duration": 0.045088,
     "end_time": "2024-04-03T06:54:18.671800",
     "exception": false,
     "start_time": "2024-04-03T06:54:18.626712",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class AverageMeter:\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2c17e3e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:54:18.742637Z",
     "iopub.status.busy": "2024-04-03T06:54:18.742319Z",
     "iopub.status.idle": "2024-04-03T06:54:18.752982Z",
     "shell.execute_reply": "2024-04-03T06:54:18.752268Z"
    },
    "papermill": {
     "duration": 0.048423,
     "end_time": "2024-04-03T06:54:18.755033",
     "exception": false,
     "start_time": "2024-04-03T06:54:18.706610",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    model.cuda().eval()\n",
    "else:\n",
    "    model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "25e18510",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:54:18.827094Z",
     "iopub.status.busy": "2024-04-03T06:54:18.826304Z",
     "iopub.status.idle": "2024-04-03T06:54:18.830842Z",
     "shell.execute_reply": "2024-04-03T06:54:18.829964Z"
    },
    "papermill": {
     "duration": 0.042234,
     "end_time": "2024-04-03T06:54:18.832775",
     "exception": false,
     "start_time": "2024-04-03T06:54:18.790541",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "acc1_meter, acc5_meter = AverageMeter(), AverageMeter()\n",
    "log_print_freq = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1fdfd2e0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-04-03T06:54:18.904983Z",
     "iopub.status.busy": "2024-04-03T06:54:18.904128Z",
     "iopub.status.idle": "2024-04-03T10:54:14.344321Z",
     "shell.execute_reply": "2024-04-03T10:54:14.343208Z"
    },
    "papermill": {
     "duration": 14395.478546,
     "end_time": "2024-04-03T10:54:14.346498",
     "exception": false,
     "start_time": "2024-04-03T06:54:18.867952",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████| 1238/1238 [3:59:55<00:00, 11.63s/it, Acc1=77.690, Acc5=93.297]\n"
     ]
    }
   ],
   "source": [
    "pbar = tqdm(enumerate(dataloader), total=len(dataloader), ncols=80, leave=True)\n",
    "\n",
    "with torch.no_grad():\n",
    "    for idx, (image, label) in pbar:\n",
    "        label_id = label\n",
    "        label_id = label_id.reshape(-1)\n",
    "        \n",
    "        b, n, t, h, w, c = image.size() # batch, clip, time-dim, channel, height, width\n",
    "        tot_similarity = torch.zeros((b,num_classes)).cuda()\n",
    "        \n",
    "        for i in range(n):\n",
    "            image_input = image[:, i, :, :, :, :] # [b,t,h,w,c]\n",
    "            label_id = label_id.cuda(non_blocking=True)\n",
    "            image_input = image_input.cuda(non_blocking=True)\n",
    "            output = model(image_input)\n",
    "            similarity = output.view(b, -1).softmax(dim=-1)\n",
    "            tot_similarity += similarity\n",
    "            \n",
    "        values_1, indices_1 = tot_similarity.topk(1, dim=-1)\n",
    "        values_5, indices_5 = tot_similarity.topk(5, dim=-1)\n",
    "        acc1, acc5 = 0, 0\n",
    "        \n",
    "        for i in range(b):\n",
    "            if indices_1[i] == label_id[i]:\n",
    "                acc1 += 1\n",
    "            if label_id[i] in indices_5[i]:\n",
    "                acc5 += 1\n",
    "                \n",
    "        acc1_meter.update(float(acc1) / b * 100, b)\n",
    "        acc5_meter.update(float(acc5) / b * 100, b)\n",
    "        \n",
    "        if idx % log_print_freq == 0:\n",
    "            pbar.set_postfix(\n",
    "                Acc1=f\"{acc1_meter.avg:.3f}\", Acc5=f\"{acc5_meter.avg:.3f}\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c50c36d7",
   "metadata": {
    "papermill": {
     "duration": 0.129947,
     "end_time": "2024-04-03T10:54:14.609281",
     "exception": false,
     "start_time": "2024-04-03T10:54:14.479334",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "datasetId": 3721472,
     "sourceId": 6446831,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30673,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 14578.408157,
   "end_time": "2024-04-03T10:54:19.452722",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-04-03T06:51:21.044565",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}