material/deep_learning/webcam_pose_extraction.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2YPiAFkkeDqT"
      },
      "outputs": [],
      "source": [
        "from IPython.display import display, Javascript\n",
        "from google.colab.output import eval_js\n",
        "from base64 import b64decode\n",
        "from IPython.display import Image\n",
        "from IPython.display import clear_output\n",
        "import warnings\n",
        "import os \n",
        "from glob import glob \n",
        "import pandas as pd \n",
        "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
        "warnings.filterwarnings(\"ignore\", category=DeprecationWarning) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z7goEMaMvCOy"
      },
      "source": [
        "# 프로젝트 다운로드"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UdZlGFCFDG4t",
        "outputId": "dafbc388-6fec-4ab6-a627-b1d63cdddcb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf Open-Pose-Keras"
      ],
      "metadata": {
        "id": "u4w_NyatDJ3B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pmEyIi2CrVxK",
        "outputId": "dc6bc22f-4c09-45fb-ce48-8da140acdb02"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'Open-Pose-Keras'...\n",
            "remote: Enumerating objects: 415, done.\u001b[K\n",
            "remote: Counting objects: 100% (18/18), done.\u001b[K\n",
            "remote: Compressing objects: 100% (13/13), done.\u001b[K\n",
            "remote: Total 415 (delta 9), reused 13 (delta 5), pack-reused 397\u001b[K\n",
            "Receiving objects: 100% (415/415), 29.70 MiB | 16.88 MiB/s, done.\n",
            "Resolving deltas: 100% (234/234), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/dhrim/Open-Pose-Keras"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CI1n8CZurYgR",
        "outputId": "b033f9ae-ed17-43d8-cf49-51781c3578c1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/Open-Pose-Keras\n"
          ]
        }
      ],
      "source": [
        "%cd Open-Pose-Keras"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QKqcb95xvJJC"
      },
      "source": [
        "# 필요한 라이브러리 설치"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_0q3KcAqrhQp",
        "outputId": "769abf1e-e4bf-426e-d7f6-4334b2937e10"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: configobj in /usr/local/lib/python3.7/dist-packages (5.0.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from configobj) (1.15.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install configobj"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yG-nT63k-Hgl"
      },
      "source": [
        "# 모델 로딩"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CiN9XIptCHqy",
        "outputId": "8f1afe9e-3505-4b2c-89f6-d3707bb5a66e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1d9HGnnLhq6IhC5ZSVpBf7bSrpjNj5kPB\n",
            "To: /content/Open-Pose-Keras/open_pose_model.h5\n",
            "100%|██████████| 210M/210M [00:00<00:00, 258MB/s]\n"
          ]
        }
      ],
      "source": [
        "from open_pose_wrapper import OpenPoseWrapper\n",
        "model = OpenPoseWrapper()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ju-QnRi5Bxkl"
      },
      "source": [
        "# 포즈 추출 실행"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8pSrvVMVtz5f"
      },
      "source": [
        "copy from https://www.youtube.com/watch?v=YjWh7QvVH60"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3PpFo2TkPtcS"
      },
      "outputs": [],
      "source": [
        "# import dependencies\n",
        "from IPython.display import display, Javascript, Image\n",
        "from google.colab.output import eval_js\n",
        "from base64 import b64decode, b64encode\n",
        "import cv2\n",
        "import numpy as np\n",
        "import PIL\n",
        "import io\n",
        "import html\n",
        "import time"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## util"
      ],
      "metadata": {
        "id": "UPCVI1V_9vkc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DiKiXLJwQLWC"
      },
      "outputs": [],
      "source": [
        "# function to convert the JavaScript object into an OpenCV image\n",
        "def js_to_image(js_reply):\n",
        "  \"\"\"\n",
        "  Params:\n",
        "          js_reply: JavaScript object containing image from webcam\n",
        "  Returns:\n",
        "          img: OpenCV BGR image\n",
        "  \"\"\"\n",
        "  # decode base64 image\n",
        "  image_bytes = b64decode(js_reply.split(',')[1])\n",
        "  # convert bytes to numpy array\n",
        "  jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)\n",
        "  # decode numpy array into OpenCV BGR image\n",
        "  img = cv2.imdecode(jpg_as_np, flags=1)\n",
        "\n",
        "  return img\n",
        "\n",
        "# function to convert OpenCV Rectangle bounding box image into base64 byte string to be overlayed on video stream\n",
        "def bbox_to_bytes(bbox_array):\n",
        "  \"\"\"\n",
        "  Params:\n",
        "          bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.\n",
        "  Returns:\n",
        "        bytes: Base64 image byte string\n",
        "  \"\"\"\n",
        "  # convert array into PIL image\n",
        "  bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGB')\n",
        "  iobuf = io.BytesIO()\n",
        "  # format bbox into png for return\n",
        "  bbox_PIL.save(iobuf, format='png')\n",
        "  # format return string\n",
        "  bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))\n",
        "\n",
        "  return bbox_bytes"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## java script"
      ],
      "metadata": {
        "id": "fmmKayYw9s7Q"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UBg46_OJPw56"
      },
      "outputs": [],
      "source": [
        "# JavaScript to properly create our live video stream using our webcam as input\n",
        "def video_stream():\n",
        "  js = Javascript('''\n",
        "    var video;\n",
        "    var div = null;\n",
        "    var stream;\n",
        "    var captureCanvas;\n",
        "    var imgElement;\n",
        "    var labelElement;\n",
        "    \n",
        "    var pendingResolve = null;\n",
        "    var shutdown = false;\n",
        "    \n",
        "    function removeDom() {\n",
        "       stream.getVideoTracks()[0].stop();\n",
        "       video.remove();\n",
        "       div.remove();\n",
        "       video = null;\n",
        "       div = null;\n",
        "       stream = null;\n",
        "       imgElement = null;\n",
        "       captureCanvas = null;\n",
        "       labelElement = null;\n",
        "    }\n",
        "    \n",
        "    function onAnimationFrame() {\n",
        "      if (!shutdown) {\n",
        "        window.requestAnimationFrame(onAnimationFrame);\n",
        "      }\n",
        "      if (pendingResolve) {\n",
        "        var result = \"\";\n",
        "        if (!shutdown) {\n",
        "          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n",
        "          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n",
        "        }\n",
        "        var lp = pendingResolve;\n",
        "        pendingResolve = null;\n",
        "        lp(result);\n",
        "      }\n",
        "    }\n",
        "    \n",
        "    async function createDom() {\n",
        "      if (div !== null) {\n",
        "        return stream;\n",
        "      }\n",
        "\n",
        "      div = document.createElement('div');\n",
        "      div.style.border = '2px solid black';\n",
        "      div.style.padding = '3px';\n",
        "      div.style.width = '100%';\n",
        "      div.style.maxWidth = '600px';\n",
        "      document.body.appendChild(div);\n",
        "      \n",
        "      const modelOut = document.createElement('div');\n",
        "      modelOut.innerHTML = \"<span>Status:</span>\";\n",
        "      labelElement = document.createElement('span');\n",
        "      labelElement.innerText = 'No data';\n",
        "      labelElement.style.fontWeight = 'bold';\n",
        "      modelOut.appendChild(labelElement);\n",
        "      div.appendChild(modelOut);\n",
        "           \n",
        "      video = document.createElement('video');\n",
        "      video.style.display = 'block';\n",
        "      video.width = div.clientWidth - 6;\n",
        "      video.setAttribute('playsinline', '');\n",
        "      video.onclick = () => { shutdown = true; };\n",
        "      stream = await navigator.mediaDevices.getUserMedia(\n",
        "          {video: { facingMode: \"environment\"}});\n",
        "      div.appendChild(video);\n",
        "\n",
        "      imgElement = document.createElement('img');\n",
        "      imgElement.style.position = 'absolute';\n",
        "      imgElement.style.zIndex = 1;\n",
        "      imgElement.onclick = () => { shutdown = true; };\n",
        "      div.appendChild(imgElement);\n",
        "      \n",
        "      const instruction = document.createElement('div');\n",
        "      instruction.innerHTML = \n",
        "          '<span style=\"color: red; font-weight: bold;\">' +\n",
        "          'When finished, click here or on the video to stop this demo</span>';\n",
        "      div.appendChild(instruction);\n",
        "      instruction.onclick = () => { shutdown = true; };\n",
        "      \n",
        "      video.srcObject = stream;\n",
        "      await video.play();\n",
        "\n",
        "      captureCanvas = document.createElement('canvas');\n",
        "      captureCanvas.width = 640; //video.videoWidth;\n",
        "      captureCanvas.height = 480; //video.videoHeight;\n",
        "      window.requestAnimationFrame(onAnimationFrame);\n",
        "      \n",
        "      return stream;\n",
        "    }\n",
        "    async function stream_frame(label, imgData) {\n",
        "      if (shutdown) {\n",
        "        removeDom();\n",
        "        shutdown = false;\n",
        "        return '';\n",
        "      }\n",
        "\n",
        "      var preCreate = Date.now();\n",
        "      stream = await createDom();\n",
        "      \n",
        "      var preShow = Date.now();\n",
        "      if (label != \"\") {\n",
        "        labelElement.innerHTML = label;\n",
        "      }\n",
        "            \n",
        "      if (imgData != \"\") {\n",
        "        var videoRect = video.getClientRects()[0];\n",
        "        imgElement.style.top = videoRect.top + \"px\";\n",
        "        imgElement.style.left = videoRect.left + \"px\";\n",
        "        imgElement.style.width = videoRect.width + \"px\";\n",
        "        imgElement.style.height = videoRect.height + \"px\";\n",
        "        imgElement.src = imgData;\n",
        "      }\n",
        "      \n",
        "      var preCapture = Date.now();\n",
        "      var result = await new Promise(function(resolve, reject) {\n",
        "        pendingResolve = resolve;\n",
        "      });\n",
        "      shutdown = false;\n",
        "      \n",
        "      return {'create': preShow - preCreate, \n",
        "              'show': preCapture - preShow, \n",
        "              'capture': Date.now() - preCapture,\n",
        "              'img': result};\n",
        "    }\n",
        "    ''')\n",
        "\n",
        "  display(js)\n",
        "  \n",
        "def video_frame(label, overay_img):\n",
        "  data = eval_js('stream_frame(\"{}\", \"{}\")'.format(label, overay_img))\n",
        "  return data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZeRnDWeycHij"
      },
      "outputs": [],
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Pose Extractor 생성"
      ],
      "metadata": {
        "id": "69mgQD3s9lcn"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0LJ8t70tcG84"
      },
      "outputs": [],
      "source": [
        "import cv2\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kKcpBHiUQgut"
      },
      "outputs": [],
      "source": [
        "class PoseExtractor:\n",
        "\n",
        "    def extract(self, image):\n",
        "        return model.extract(image)\n",
        "\n",
        "    def get_extracted_image(self, image, all_peaks, subset, candidate):\n",
        "        return model.get_skeleton_image(image, all_peaks, subset, candidate)\n",
        "\n",
        "pose_extractor = PoseExtractor()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 실행 루프"
      ],
      "metadata": {
        "id": "3-Z19We29jky"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "yNVYtGEXPq2H",
        "outputId": "a56cbfff-dd3f-4439-da5b-a4671a3906e7"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    var video;\n",
              "    var div = null;\n",
              "    var stream;\n",
              "    var captureCanvas;\n",
              "    var imgElement;\n",
              "    var labelElement;\n",
              "    \n",
              "    var pendingResolve = null;\n",
              "    var shutdown = false;\n",
              "    \n",
              "    function removeDom() {\n",
              "       stream.getVideoTracks()[0].stop();\n",
              "       video.remove();\n",
              "       div.remove();\n",
              "       video = null;\n",
              "       div = null;\n",
              "       stream = null;\n",
              "       imgElement = null;\n",
              "       captureCanvas = null;\n",
              "       labelElement = null;\n",
              "    }\n",
              "    \n",
              "    function onAnimationFrame() {\n",
              "      if (!shutdown) {\n",
              "        window.requestAnimationFrame(onAnimationFrame);\n",
              "      }\n",
              "      if (pendingResolve) {\n",
              "        var result = \"\";\n",
              "        if (!shutdown) {\n",
              "          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n",
              "          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n",
              "        }\n",
              "        var lp = pendingResolve;\n",
              "        pendingResolve = null;\n",
              "        lp(result);\n",
              "      }\n",
              "    }\n",
              "    \n",
              "    async function createDom() {\n",
              "      if (div !== null) {\n",
              "        return stream;\n",
              "      }\n",
              "\n",
              "      div = document.createElement('div');\n",
              "      div.style.border = '2px solid black';\n",
              "      div.style.padding = '3px';\n",
              "      div.style.width = '100%';\n",
              "      div.style.maxWidth = '600px';\n",
              "      document.body.appendChild(div);\n",
              "      \n",
              "      const modelOut = document.createElement('div');\n",
              "      modelOut.innerHTML = \"<span>Status:</span>\";\n",
              "      labelElement = document.createElement('span');\n",
              "      labelElement.innerText = 'No data';\n",
              "      labelElement.style.fontWeight = 'bold';\n",
              "      modelOut.appendChild(labelElement);\n",
              "      div.appendChild(modelOut);\n",
              "           \n",
              "      video = document.createElement('video');\n",
              "      video.style.display = 'block';\n",
              "      video.width = div.clientWidth - 6;\n",
              "      video.setAttribute('playsinline', '');\n",
              "      video.onclick = () => { shutdown = true; };\n",
              "      stream = await navigator.mediaDevices.getUserMedia(\n",
              "          {video: { facingMode: \"environment\"}});\n",
              "      div.appendChild(video);\n",
              "\n",
              "      imgElement = document.createElement('img');\n",
              "      imgElement.style.position = 'absolute';\n",
              "      imgElement.style.zIndex = 1;\n",
              "      imgElement.onclick = () => { shutdown = true; };\n",
              "      div.appendChild(imgElement);\n",
              "      \n",
              "      const instruction = document.createElement('div');\n",
              "      instruction.innerHTML = \n",
              "          '<span style=\"color: red; font-weight: bold;\">' +\n",
              "          'When finished, click here or on the video to stop this demo</span>';\n",
              "      div.appendChild(instruction);\n",
              "      instruction.onclick = () => { shutdown = true; };\n",
              "      \n",
              "      video.srcObject = stream;\n",
              "      await video.play();\n",
              "\n",
              "      captureCanvas = document.createElement('canvas');\n",
              "      captureCanvas.width = 640; //video.videoWidth;\n",
              "      captureCanvas.height = 480; //video.videoHeight;\n",
              "      window.requestAnimationFrame(onAnimationFrame);\n",
              "      \n",
              "      return stream;\n",
              "    }\n",
              "    async function stream_frame(label, imgData) {\n",
              "      if (shutdown) {\n",
              "        removeDom();\n",
              "        shutdown = false;\n",
              "        return '';\n",
              "      }\n",
              "\n",
              "      var preCreate = Date.now();\n",
              "      stream = await createDom();\n",
              "      \n",
              "      var preShow = Date.now();\n",
              "      if (label != \"\") {\n",
              "        labelElement.innerHTML = label;\n",
              "      }\n",
              "            \n",
              "      if (imgData != \"\") {\n",
              "        var videoRect = video.getClientRects()[0];\n",
              "        imgElement.style.top = videoRect.top + \"px\";\n",
              "        imgElement.style.left = videoRect.left + \"px\";\n",
              "        imgElement.style.width = videoRect.width + \"px\";\n",
              "        imgElement.style.height = videoRect.height + \"px\";\n",
              "        imgElement.src = imgData;\n",
              "      }\n",
              "      \n",
              "      var preCapture = Date.now();\n",
              "      var result = await new Promise(function(resolve, reject) {\n",
              "        pendingResolve = resolve;\n",
              "      });\n",
              "      shutdown = false;\n",
              "      \n",
              "      return {'create': preShow - preCreate, \n",
              "              'show': preCapture - preShow, \n",
              "              'capture': Date.now() - preCapture,\n",
              "              'img': result};\n",
              "    }\n",
              "    "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "# start streaming video from webcam\n",
        "video_stream()\n",
        "\n",
        "# label for video\n",
        "label_html = 'Capturing...'\n",
        "\n",
        "# initialze bounding box to empty\n",
        "overray_bytes = ''\n",
        "\n",
        "while True:\n",
        "    js_reply = video_frame(label_html, overray_bytes)\n",
        "    if not js_reply:\n",
        "        break\n",
        "    # convert JS response to OpenCV Image\n",
        "    img = js_to_image(js_reply[\"img\"])\n",
        "\n",
        "    all_peaks, subset, candidate = pose_extractor.extract(img)\n",
        "    overray_img = pose_extractor.get_extracted_image(img, all_peaks, subset, candidate)\n",
        "\n",
        "    # convert overlay of bbox into bytes\n",
        "    overray_bytes = bbox_to_bytes(overray_img)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h_2BtA44eGmC"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "webcam_pose_extraction.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}