using_transformers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1620bd82-9234-4c66-8e3f-1b77763251e2",
   "metadata": {},
   "source": [
    "# -> Behind the pipeline ->"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "998a00fa-4e0b-438c-ae10-bf8e1cfb5f50",
   "metadata": {},
   "source": [
    "## Example pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f740ad7d-c58e-49b7-a532-98c787881fb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english).\n",
      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'label': 'POSITIVE', 'score': 0.9598049521446228},\n",
       " {'label': 'NEGATIVE', 'score': 0.9994558691978455}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "classifier = pipeline(\"sentiment-analysis\")\n",
    "classifier(\n",
    "    [\n",
    "    \"I've been waiting for a HuggingFace course my whole life.\",\n",
    "    \"I hate this so much!\",\n",
    "    ]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0658671-9b82-4d1e-a213-d96541043486",
   "metadata": {},
   "source": [
    "## Broken down step by step"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ca4fbf1-2480-4099-a6d0-832eb55307b5",
   "metadata": {},
   "source": [
    "step 1: tokenize input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "6cf36c04-399f-43de-9d32-ea51e4677595",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[  101,  1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,\n",
      "          2607,  2026,  2878,  2166,  1012,   102],\n",
      "        [  101,  1045,  5223,  2023,  2061,  2172,   999,   102,     0,     0,\n",
      "             0,     0,     0,     0,     0,     0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]])}\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "\n",
    "raw_inputs = [\n",
    "    \"I've been waiting for a HuggingFace course my whole life.\",\n",
    "    \"I hate this so much!\",\n",
    "]\n",
    "inputs = tokenizer(raw_inputs, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "print(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6404d49-b7ff-4eef-a389-cbb22082941a",
   "metadata": {},
   "source": [
    "step 2: pass tokens to model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98803574-b40d-4ef3-8504-423bab3b19f7",
   "metadata": {},
   "source": [
    "2a: get \"hidden state\" output of headless transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "0ccb3067-55c2-41a4-b96a-1d6e8988c724",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased-finetuned-sst-2-english were not used when initializing DistilBertModel: ['pre_classifier.weight', 'classifier.bias', 'classifier.weight', 'pre_classifier.bias']\n",
      "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 16, 768])\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModel\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "model = AutoModel.from_pretrained(checkpoint)\n",
    "\n",
    "outputs = model(**inputs)\n",
    "print(outputs.last_hidden_state.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ab10ce5-bd0f-4922-96b3-c71e4245b274",
   "metadata": {},
   "source": [
    "2b: get output passed through model with a head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "549be192-a9ff-4a09-be7c-a937012ef9b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 2])\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "outputs = model(**inputs)\n",
    "print(outputs.logits.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd153a17-3dca-4a8d-a017-1d8595ebde34",
   "metadata": {},
   "source": [
    "step 3: postprocess the output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "25ea44fa-fbbe-420c-867d-24aa91113f64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[4.0195e-02, 9.5980e-01],\n",
      "        [9.9946e-01, 5.4418e-04]], grad_fn=<SoftmaxBackward0>)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0: 'NEGATIVE', 1: 'POSITIVE'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\n",
    "print(predictions)\n",
    "model.config.id2label"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9fdfdec-a577-49bf-a2da-2433e26a2441",
   "metadata": {},
   "source": [
    "# -> Models ->"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7499af4c-df5a-493b-8e82-76d881208f99",
   "metadata": {},
   "source": [
    "## Creating / saving transformers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33ed6c22-f28f-417d-adc3-76af31af60e4",
   "metadata": {},
   "source": [
    "Loading a transformer with random weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "3ec098d4-5f95-4c6a-818e-206c3fe7decd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertConfig {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.25.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertConfig, BertModel\n",
    "\n",
    "# Building the config\n",
    "config = BertConfig()\n",
    "\n",
    "# Building the model from the config\n",
    "model = BertModel(config)\n",
    "print(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "976492e7-b032-4ac8-8bb8-fc4ddc8d510b",
   "metadata": {},
   "source": [
    "load a model with pretrained weights\n",
    "\n",
    "(always use the same checkpoint as your tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "4f5d697e-7184-4a02-991e-d9c20b26f533",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: 100%|██████████████████████████████| 570/570 [00:00<00:00, 365kB/s]\n",
      "Downloading: 100%|███████████████████████████| 436M/436M [00:39<00:00, 10.9MB/s]\n",
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertConfig, BertModel\n",
    "\n",
    "model = BertModel.from_pretrained(\"bert-base-cased\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9919fa2f-a78d-4d5e-87d7-cfb66a662218",
   "metadata": {},
   "source": [
    "save a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "93a34216-dc94-465e-be62-07699d235864",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config.json       pytorch_model.bin\n",
      "{\n",
      "  \"_name_or_path\": \"bert-base-cased\",\n",
      "  \"architectures\": [\n",
      "    \"BertModel\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.25.1\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "model.save_pretrained(\"directory_on_my_computer\")\n",
    "!ls directory_on_my_computer\n",
    "!cat directory_on_my_computer/config.json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ed66a7a-f6a5-4295-ab94-47c159cf9aa1",
   "metadata": {},
   "source": [
    "## Using a Transformer model for inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "43fc1864-e0b3-4a82-ac31-ff73122fae98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input shape: torch.Size([3, 4])\n",
      "output hidden state shape: torch.Size([3, 4, 768])\n",
      "pooler output shape: torch.Size([3, 768])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "sequences = [\"Hello!\", \"Cool.\", \"Nice!\"]\n",
    "encoded_sequences = [\n",
    "    [101, 7592, 999, 102],\n",
    "    [101, 4658, 1012, 102],\n",
    "    [101, 3835, 999, 102],\n",
    "]\n",
    "\n",
    "model_inputs = torch.tensor(encoded_sequences)\n",
    "output = model(model_inputs)\n",
    "\n",
    "print(\"input shape:\", model_inputs.shape)\n",
    "print(\"output hidden state shape:\", output[\"last_hidden_state\"].shape)\n",
    "print(\"pooler output shape:\", output[\"pooler_output\"].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef273588-a16f-4a4b-af7a-1f621146b983",
   "metadata": {},
   "source": [
    "# -> Tokenizers ->"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44b6618f-3c73-45cd-82e7-6741ed035e1a",
   "metadata": {},
   "source": [
    "Basic tokenizer steps: 1. tokenize (usually subword-based), 2. convert to ids, 3. add special tokens (e.g. start/stop)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79b8377d-242a-46a6-943d-1adce3d26bab",
   "metadata": {},
   "source": [
    "Loading and using tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "9668a259-6a0c-4a4e-9ab4-767e7b1d56bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 7993, 170, 13809, 23763, 2443, 1110, 3014, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-cased\")\n",
    "\n",
    "# OR\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
    "\n",
    "tokenizer(\"Using a Transformer network is simple\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8d6cf86-dc55-42d8-b67f-7a97b2fc6b28",
   "metadata": {},
   "source": [
    "saving tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b0e90497-acb7-4437-9727-e705aca2897b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('directory_on_my_computer/tokenizer_config.json',\n",
       " 'directory_on_my_computer/special_tokens_map.json',\n",
       " 'directory_on_my_computer/vocab.txt',\n",
       " 'directory_on_my_computer/added_tokens.json',\n",
       " 'directory_on_my_computer/tokenizer.json')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(\"directory_on_my_computer\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5056266d-74e4-4025-af8d-08bec935d72b",
   "metadata": {},
   "source": [
    "using individual tokenizer functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "125a8f97-607d-458f-94bb-72de4b6f8e15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Using', 'a', 'Trans', '##former', 'network', 'is', 'simple']\n",
      "[7993, 170, 13809, 23763, 2443, 1110, 3014]\n",
      "Using a transformer network is simple\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
    "\n",
    "sequence = \"Using a Transformer network is simple\"\n",
    "tokens = tokenizer.tokenize(sequence)\n",
    "\n",
    "print(tokens)\n",
    "\n",
    "['Using', 'a', 'transform', '##er', 'network', 'is', 'simple']\n",
    "\n",
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "print(ids)\n",
    "\n",
    "decoded_string = tokenizer.decode([7993, 170, 11303, 1200, 2443, 1110, 3014])\n",
    "print(decoded_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64c2de60-3067-41ab-b749-738a36e061ae",
   "metadata": {},
   "source": [
    "# -> Handling multiple sequences ->"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb13a47a-4d21-472f-86e6-fd6bea882c38",
   "metadata": {},
   "source": [
    "## Batching inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56cea7cb-e1c9-4cac-ac3f-27e752a8845c",
   "metadata": {},
   "source": [
    "instantiate tokenizer, model, and text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "08d69fdc-f297-4d44-bccd-a99ea93bb236",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "\n",
    "sequence = \"I've been waiting for a HuggingFace course my whole life.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbbfbbaa-0306-435d-826a-d9ef9042eca5",
   "metadata": {},
   "source": [
    "this model input works b/c it takes care of adding the batching dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "2650f193-0269-46fc-a89a-014f91362089",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[  101,  1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,\n",
      "          2607,  2026,  2878,  2166,  1012,   102]])\n",
      "Logits: tensor([[-2.7276,  2.8789]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "tokenized_inputs = tokenizer(sequence, return_tensors=\"pt\")\n",
    "model_inputs = tokenized_inputs[\"input_ids\"]\n",
    "print(model_inputs)\n",
    "output = model(input_ids)\n",
    "print(\"Logits:\", output.logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc17d177-f9ca-4e75-8f6e-8d260aaffe9e",
   "metadata": {},
   "source": [
    "this doesn't"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "cc6ec836-366b-485e-8f9b-77456cfdc0bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,  2607,\n",
      "         2026,  2878,  2166,  1012])\n"
     ]
    }
   ],
   "source": [
    "tokens = tokenizer.tokenize(sequence)\n",
    "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "input_ids = torch.tensor(ids)\n",
    "print(input_ids)\n",
    "# The line below would fail.\n",
    "# model(input_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76f17f29-a1fc-4d2d-ac1d-88289d895886",
   "metadata": {},
   "source": [
    "so we need to add the batch dimension, and voila!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "f1376ac8-27a8-4912-ad6e-d8c416194f3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logits: tensor([[-2.7276,  2.8789]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "input_ids = torch.tensor([ids])\n",
    "output = model(input_ids)\n",
    "print(\"Logits:\", output.logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01c1dedb-55bf-400b-bd4e-fc2cb3aadc71",
   "metadata": {},
   "source": [
    "## Padding and masking inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "61f8e53e-d14e-42e1-8578-fdb173596f30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "padding token id: 0\n"
     ]
    }
   ],
   "source": [
    "print(\"padding token id:\", tokenizer.pad_token_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b21864-13b8-49db-9034-b4716e89985a",
   "metadata": {},
   "source": [
    "padding without masking results in different predictions for the same sentence/sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "48208aa8-718c-45d7-afe1-0ae817d274b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.5694, -1.3895]], grad_fn=<AddmmBackward0>)\n",
      "tensor([[ 0.5803, -0.4125]], grad_fn=<AddmmBackward0>)\n",
      "tensor([[ 1.5694, -1.3895],\n",
      "        [ 1.3374, -1.2163]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "\n",
    "sequence1_ids = [[200, 200, 200]]\n",
    "sequence2_ids = [[200, 200]]\n",
    "batched_ids = [\n",
    "    [200, 200, 200],\n",
    "    [200, 200, tokenizer.pad_token_id],\n",
    "]\n",
    "\n",
    "print(model(torch.tensor(sequence1_ids)).logits)\n",
    "print(model(torch.tensor(sequence2_ids)).logits)\n",
    "print(model(torch.tensor(batched_ids)).logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf77345a-d92d-49d3-91ad-fac2c9698d8e",
   "metadata": {},
   "source": [
    "including an attention mask ensures equivalence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "e575c0b0-748a-4e26-bd1b-8e6cfe73be43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.5694, -1.3895],\n",
      "        [ 0.5803, -0.4125]], grad_fn=<AddmmBackward0>)\n"
     ]
    }
   ],
   "source": [
    "attention_mask = [\n",
    "    [1, 1, 1],\n",
    "    [1, 1, 0],\n",
    "]\n",
    "\n",
    "outputs = model(torch.tensor(batched_ids), attention_mask=torch.tensor(attention_mask))\n",
    "print(outputs.logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "917581ce-f921-4435-9406-2e3807d3391e",
   "metadata": {},
   "source": [
    "# -> Putting it all together ->"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d4cefd5-5806-4533-b7a3-8af9385dabc3",
   "metadata": {
    "tags": []
   },
   "source": [
    "can specify padding, truncation, and tensor type in tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "0ed0a2b0-9bf5-47d1-b826-b19be80c8090",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[  101,  1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,\n",
      "          2607,  2026,  2878,  2166,  1012,   102],\n",
      "        [  101,  2061,  2031,  1045,   999,   102,     0,     0,     0,     0,\n",
      "             0,     0,     0,     0,     0,     0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])}\n",
      "[\"[CLS] i've been waiting for a huggingface course my whole life. [SEP]\", '[CLS] so have i! [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]']\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
    "\n",
    "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
    "\n",
    "# Returns PyTorch tensors; use \"tf\" for tensorflow or \"np\" for numpy\n",
    "model_inputs = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
    "print(model_inputs)\n",
    "print([tokenizer.decode(seq) for seq in model_inputs[\"input_ids\"]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3964c931-ede1-4981-937c-ee8efb966bfa",
   "metadata": {},
   "source": [
    "put through model and postprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "98475716-87c6-4e48-8fb8-ee98dc843909",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'NEGATIVE': 0.04019509255886078, 'POSITIVE': 0.9598048329353333}, {'NEGATIVE': 0.0005353438318707049, 'POSITIVE': 0.9994646906852722}]\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
    "\n",
    "output = model(**model_inputs)\n",
    "predictions = torch.nn.functional.softmax(output.logits, dim=-1)\n",
    "\n",
    "# outputs human readable predictions using model.config.id2label\n",
    "sorted_inds = np.argsort(predictions.detach().numpy(), axis=-1)\n",
    "sorted_pred_dicts = []\n",
    "for c, inds in enumerate(sorted_inds):\n",
    "    sorted_pred_dict = {model.config.id2label[ind]: float(predictions[c][ind]) for ind in inds}\n",
    "    sorted_pred_dicts.append(sorted_pred_dict)\n",
    "    \n",
    "print(sorted_pred_dicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93829e88-484d-4f09-b8be-c2eabc5686aa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "huggingface",
   "language": "python",
   "name": "huggingface"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}