Youtube_Course_Sentence_Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Youtube: Course Sentence Transformers.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "<h2>Sentence Transformers</h2>"
      ],
      "metadata": {
        "id": "SjNAmtyUT33o"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Resources:**\n",
        "\n",
        "*   https://www.sbert.net/index.html\n",
        "*   https://www.sbert.net/docs/pretrained_models.html\n",
        "\n"
      ],
      "metadata": {
        "id": "PC08IHg2jNe1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Use cases:**\n",
        "\n",
        "\n",
        "\n",
        "*   Sentence Embedding\n",
        "*   Sentence Similarity\n",
        "*   Semantic Search\n",
        "*   Clustering\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "aJ9U2MTSQCGe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U sentence-transformers"
      ],
      "metadata": {
        "id": "iBxOfXwbeurS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Generate Embeding**"
      ],
      "metadata": {
        "id": "8Fe3mt_jCTL0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer,util\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')"
      ],
      "metadata": {
        "id": "QXuwDrEsTl-7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "03fm2wdZejPc"
      },
      "outputs": [],
      "source": [
        "sentences = ['This framework generates embeddings for each input sentence',\n",
        "    'Sentences are passed as a list of string.']\n",
        "\n",
        "\n",
        "embeddings = model.encode(sentences)\n",
        "\n",
        "\n",
        "for sentence, embedding in zip(sentences, embeddings):\n",
        "    print(\"Sentence:\", sentence)\n",
        "    print(\"Embedding:\", embedding)\n",
        "    print(\"\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Cosine-Similarity**"
      ],
      "metadata": {
        "id": "YDc8_VCrCQGt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "emb1 = model.encode(\"I am eating Apple\")\n",
        "emb2 = model.encode(\"I like fruits\")\n",
        "cos_sim = util.cos_sim(emb1, emb2)\n",
        "print(\"Cosine-Similarity:\", cos_sim)"
      ],
      "metadata": {
        "id": "ZGz3bDLhewrZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a14d7c3c-638a-4975-e70c-370d8a61ca03"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cosine-Similarity: tensor([[0.5398]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Compute cosine similarity between all pairs**"
      ],
      "metadata": {
        "id": "Ml5sIVHOCY6u"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Compute cosine similarity between all pairs\n",
        "\n",
        "sentences = ['A man is eating food.',\n",
        "          'A man is eating a piece of bread.',\n",
        "          'The girl is carrying a baby.',\n",
        "          'A man is riding a horse.',\n",
        "          'A woman is playing violin.',\n",
        "          'Two men pushed carts through the woods.',\n",
        "          'A man is riding a white horse on an enclosed ground.',\n",
        "          'A monkey is playing drums.',\n",
        "          'Someone in a gorilla costume is playing a set of drums.'\n",
        "          ]\n",
        "\n",
        "#Encode all sentences\n",
        "embeddings = model.encode(sentences)\n",
        "\n",
        "#Compute cosine similarity between all pairs\n",
        "cos_sim = util.cos_sim(embeddings, embeddings)\n",
        "\n",
        "cos_sim\n",
        "\n"
      ],
      "metadata": {
        "id": "9lyr6Tjge9UY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "01b57e4d-0c3d-435e-8b4f-25eda073c88e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 1.0000,  0.7553, -0.1050,  0.2474, -0.0704, -0.0333,  0.1707,  0.0476,\n",
              "          0.0630],\n",
              "        [ 0.7553,  1.0000, -0.0610,  0.1442, -0.0809, -0.0216,  0.1157,  0.0362,\n",
              "          0.0216],\n",
              "        [-0.1050, -0.0610,  1.0000, -0.1088,  0.0217, -0.0413, -0.0928,  0.0231,\n",
              "          0.0247],\n",
              "        [ 0.2474,  0.1442, -0.1088,  1.0000, -0.0348,  0.0362,  0.7369,  0.0821,\n",
              "          0.1389],\n",
              "        [-0.0704, -0.0809,  0.0217, -0.0348,  1.0000, -0.1654, -0.0592,  0.1961,\n",
              "          0.2564],\n",
              "        [-0.0333, -0.0216, -0.0413,  0.0362, -0.1654,  1.0000,  0.0769, -0.0380,\n",
              "         -0.0895],\n",
              "        [ 0.1707,  0.1157, -0.0928,  0.7369, -0.0592,  0.0769,  1.0000,  0.0495,\n",
              "          0.1191],\n",
              "        [ 0.0476,  0.0362,  0.0231,  0.0821,  0.1961, -0.0380,  0.0495,  1.0000,\n",
              "          0.6433],\n",
              "        [ 0.0630,  0.0216,  0.0247,  0.1389,  0.2564, -0.0895,  0.1191,  0.6433,\n",
              "          1.0000]])"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Add all pairs to a list with their cosine similarity score\n",
        "all_sentence_combinations = []\n",
        "for i in range(len(cos_sim)-1):\n",
        "    for j in range(i+1, len(cos_sim)):\n",
        "        all_sentence_combinations.append((cos_sim[i][j], i, j))\n",
        "all_sentence_combinations        "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "06H7O06b91_q",
        "outputId": "ecf163db-0c42-4ebe-9a3f-9f5b6f900272"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(tensor(0.7553), 0, 1),\n",
              " (tensor(-0.1050), 0, 2),\n",
              " (tensor(0.2474), 0, 3),\n",
              " (tensor(-0.0704), 0, 4),\n",
              " (tensor(-0.0333), 0, 5),\n",
              " (tensor(0.1707), 0, 6),\n",
              " (tensor(0.0476), 0, 7),\n",
              " (tensor(0.0630), 0, 8),\n",
              " (tensor(-0.0610), 1, 2),\n",
              " (tensor(0.1442), 1, 3),\n",
              " (tensor(-0.0809), 1, 4),\n",
              " (tensor(-0.0216), 1, 5),\n",
              " (tensor(0.1157), 1, 6),\n",
              " (tensor(0.0362), 1, 7),\n",
              " (tensor(0.0216), 1, 8),\n",
              " (tensor(-0.1088), 2, 3),\n",
              " (tensor(0.0217), 2, 4),\n",
              " (tensor(-0.0413), 2, 5),\n",
              " (tensor(-0.0928), 2, 6),\n",
              " (tensor(0.0231), 2, 7),\n",
              " (tensor(0.0247), 2, 8),\n",
              " (tensor(-0.0348), 3, 4),\n",
              " (tensor(0.0362), 3, 5),\n",
              " (tensor(0.7369), 3, 6),\n",
              " (tensor(0.0821), 3, 7),\n",
              " (tensor(0.1389), 3, 8),\n",
              " (tensor(-0.1654), 4, 5),\n",
              " (tensor(-0.0592), 4, 6),\n",
              " (tensor(0.1961), 4, 7),\n",
              " (tensor(0.2564), 4, 8),\n",
              " (tensor(0.0769), 5, 6),\n",
              " (tensor(-0.0380), 5, 7),\n",
              " (tensor(-0.0895), 5, 8),\n",
              " (tensor(0.0495), 6, 7),\n",
              " (tensor(0.1191), 6, 8),\n",
              " (tensor(0.6433), 7, 8)]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Sort list by the highest cosine similarity score\n",
        "all_sentence_combinations = sorted(all_sentence_combinations, key=lambda x: x[0], reverse=True)\n",
        "\n",
        "print(\"Top-5 most similar pairs:\")\n",
        "for score, i, j in all_sentence_combinations[0:5]:\n",
        "    print(\"{} \\t {} \\t {:.4f}\".format(sentences[i], sentences[j], cos_sim[i][j]))"
      ],
      "metadata": {
        "id": "qhggYpNbfK4J",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b4d486f0-4435-4513-8b01-f99f59946370"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Top-5 most similar pairs:\n",
            "A man is eating food. \t A man is eating a piece of bread. \t 0.7553\n",
            "A man is riding a horse. \t A man is riding a white horse on an enclosed ground. \t 0.7369\n",
            "A monkey is playing drums. \t Someone in a gorilla costume is playing a set of drums. \t 0.6433\n",
            "A woman is playing violin. \t Someone in a gorilla costume is playing a set of drums. \t 0.2564\n",
            "A man is eating food. \t A man is riding a horse. \t 0.2474\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Semantic search**"
      ],
      "metadata": {
        "id": "E0-VolFWK-Rq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sentence_transformers import SentenceTransformer, util\n",
        "model = SentenceTransformer('clips/mfaq')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wpX-L96mTthR",
        "outputId": "e4ff82e3-aac0-4853-df35-17df034ffcaf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/configuration_utils.py:369: UserWarning: Passing `gradient_checkpointing` to a config initialization is deprecated and will be removed in v5 Transformers. Using `model.gradient_checkpointing_enable()` instead, or if you are using the `Trainer` API, pass `gradient_checkpointing=True` in your `TrainingArguments`.\n",
            "  \"Passing `gradient_checkpointing` to a config initialization is deprecated and will be removed in v5 \"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "question = \"<Q>How many models can I host on HuggingFace?\"\n",
        "answer_1 = \"<A>All plans come with unlimited private models and datasets.\"\n",
        "answer_2 = \"<A>AutoNLP is an automatic way to train and deploy state-of-the-art NLP models, seamlessly integrated with the Hugging Face ecosystem.\"\n",
        "answer_3 = \"<A>Based on how much training data and model variants are created, we send you a compute cost and payment link - as low as $10 per job.\"\n",
        "\n",
        "query_embedding = model.encode(question)\n",
        "corpus_embeddings = model.encode([answer_1, answer_2, answer_3])\n",
        "\n",
        "print(util.semantic_search(query_embedding, corpus_embeddings))"
      ],
      "metadata": {
        "id": "z1gio-Pci9RF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5c45c533-ade1-469c-8d16-75248660302a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[{'corpus_id': 0, 'score': 0.5646325945854187}, {'corpus_id': 2, 'score': 0.5142340660095215}, {'corpus_id': 1, 'score': 0.4730038046836853}]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline"
      ],
      "metadata": {
        "id": "6FG1WXOudgYq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qa_model = pipeline(\"question-answering\")\n",
        "question = \"How many models can I host on HuggingFace?\"\n",
        "context = \"All plans come with unlimited private models and datasets.\"\n",
        "qa_model(question = question, context = context)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Qa18QF-ddMa",
        "outputId": "440e262a-1672-4f9b-e85b-c98bd1f96344"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to distilbert-base-cased-distilled-squad and revision 626af31 (https://huggingface.co/distilbert-base-cased-distilled-squad).\n",
            "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'unlimited', 'end': 29, 'score': 0.701717734336853, 'start': 20}"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(util.semantic_search(query_embedding, corpus_embeddings))"
      ],
      "metadata": {
        "id": "_J6IY-4oC4hF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Clustering**"
      ],
      "metadata": {
        "id": "G9tF0hu6K0TD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.cluster import KMeans\n",
        "import numpy as np\n",
        "\n",
        "embedder = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Corpus with example sentences\n",
        "corpus = ['A man is eating food.',\n",
        "          'A man is eating a piece of bread.',\n",
        "          'Horse is eating grass.',\n",
        "          'A man is eating pasta.',\n",
        "          'A Woman is eating Biryani.',\n",
        "          'The girl is carrying a baby.',\n",
        "          'The baby is carried by the woman',\n",
        "          'A man is riding a horse.',\n",
        "          'A man is riding a white horse on an enclosed ground.',\n",
        "          'A monkey is playing drums.',\n",
        "          'Someone in a gorilla costume is playing a set of drums.',\n",
        "          'A cheetah is running behind its prey.',\n",
        "          'A cheetah chases prey on across a field.',\n",
        "          'The cheetah is chasing a man who is riding the horse.',\n",
        "          'man and women with their baby are watching cheetah in zoo'\n",
        "          ]\n",
        "corpus_embeddings = embedder.encode(corpus)\n",
        "\n",
        "# Normalize the embeddings to unit length\n",
        "corpus_embeddings = corpus_embeddings /  np.linalg.norm(corpus_embeddings, axis=1, keepdims=True)"
      ],
      "metadata": {
        "id": "BC9xCE63G0UF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corpus_embeddings[0]"
      ],
      "metadata": {
        "id": "ri3xLpcXerJW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# source: https://stackoverflow.com/questions/55619176/how-to-cluster-similar-sentences-using-bert\n",
        "\n",
        "clustering_model = KMeans(n_clusters=4)\n",
        "clustering_model.fit(corpus_embeddings)\n",
        "cluster_assignment = clustering_model.labels_\n",
        "print(cluster_assignment)"
      ],
      "metadata": {
        "id": "f-pCHO7EkDKj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9728213e-3be7-4735-b468-9708aa55dcb0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 1 3 1 1 0 0 3 3 2 2 3 3 3 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "clustered_sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ih-1L7bJJq0k",
        "outputId": "d0bdf4f9-eae8-40a8-d98b-5458b1d26a36"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: ['The girl is carrying a baby.',\n",
              "  'The baby is carried by the woman',\n",
              "  'A monkey is playing drums.',\n",
              "  'Someone in a gorilla costume is playing a set of drums.',\n",
              "  'man and women with their baby are watching cheetah in zoo'],\n",
              " 1: ['Horse is eating grass.',\n",
              "  'A man is riding a horse.',\n",
              "  'A man is riding a white horse on an enclosed ground.',\n",
              "  'A cheetah is running behind its prey.',\n",
              "  'A cheetah chases prey on across a field.',\n",
              "  'The cheetah is chasing a man who is riding the horse.'],\n",
              " 2: ['A man is eating food.',\n",
              "  'A man is eating a piece of bread.',\n",
              "  'A man is eating pasta.',\n",
              "  'A Woman is eating Biryani.']}"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clustered_sentences = {}\n",
        "for sentence_id, cluster_id in enumerate(cluster_assignment):\n",
        "    if cluster_id not in clustered_sentences:\n",
        "        clustered_sentences[cluster_id] = []\n",
        "\n",
        "    clustered_sentences[cluster_id].append(corpus[sentence_id])\n",
        "clustered_sentences"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O2eINiS3Sivc",
        "outputId": "90bd9c78-4a64-4ade-a824-225f199c454e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: ['The girl is carrying a baby.',\n",
              "  'The baby is carried by the woman',\n",
              "  'man and women with their baby are watching cheetah in zoo'],\n",
              " 1: ['A man is eating food.',\n",
              "  'A man is eating a piece of bread.',\n",
              "  'A man is eating pasta.',\n",
              "  'A Woman is eating Biryani.'],\n",
              " 2: ['A monkey is playing drums.',\n",
              "  'Someone in a gorilla costume is playing a set of drums.'],\n",
              " 3: ['Horse is eating grass.',\n",
              "  'A man is riding a horse.',\n",
              "  'A man is riding a white horse on an enclosed ground.',\n",
              "  'A cheetah is running behind its prey.',\n",
              "  'A cheetah chases prey on across a field.',\n",
              "  'The cheetah is chasing a man who is riding the horse.']}"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "O5GW-zqcfsPl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}