laplacian_pyramid_blending.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from skimage.io import imread\n",
    "from skimage.transform import rotate\n",
    "from numba import jit # conversion to machine code\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions\n",
    "### a function to show multiple images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a function to show multiple images\n",
    "def show_images(**args):\n",
    "    for i in args.keys():\n",
    "        img=args[i]\n",
    "        plt.figure(figsize=(10,10))\n",
    "        d=len(img.shape)\n",
    "        if d==2:\n",
    "            plt.imshow(img,cmap='gray')\n",
    "        else:\n",
    "            plt.imshow(img)\n",
    "        # plt.axis('off')\n",
    "        plt.title(f'{i}')\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate Gaussian kernel, padding, and convolution\n",
    "ref:  \n",
    "exercise 2 @ https://homepages.inf.ed.ac.uk/rbf/HIPR2/gsmooth.htm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gaussian_kernel(sigma, size=0):\n",
    "    if size == 0: # sizes are determined so that the kernel edges correspond to 5% of the peak value. \n",
    "        size = int(1 + 2 * np.ceil(sigma * np.sqrt(np.log(400))))\n",
    "    mid = size // 2\n",
    "    kernel = np.zeros((size, 1))\n",
    "    dkernel = np.zeros((size, 1))\n",
    "    for i in range(size):\n",
    "        kernel[i] = np.exp(-((i - mid) / sigma) ** 2 / 2)\n",
    "        dkernel[i] = -(i - mid) / (np.sqrt(2 * np.pi) * sigma ** 3) * kernel[i]\n",
    "    kernel = kernel / kernel.sum()\n",
    "    return kernel, dkernel\n",
    "\n",
    "\n",
    "def padding_nearest(img, kernel):\n",
    "    h, w = img.shape[:2]\n",
    "    dim = len(img.shape)\n",
    "    hk, wk = kernel.shape[:2]\n",
    "    mid_hk, mid_wk = hk // 2, wk // 2\n",
    "    if dim == 3:\n",
    "        padded = np.zeros((h + hk - 1, w + wk - 1, 3))\n",
    "    else:\n",
    "        padded = np.zeros((h + hk - 1, w + wk - 1))\n",
    "    padded[:mid_hk, mid_wk:w + mid_wk] = img[0:1, :]  # top\n",
    "    padded[h + mid_hk:, mid_wk:w + mid_wk] = img[-1:, :]  # bottom\n",
    "    padded[mid_hk:h + mid_hk, :mid_wk] = img[:, 0:1]  # left\n",
    "    padded[mid_hk:h + mid_hk, w + mid_wk:] = img[:, -1:]  # right\n",
    "    padded[:mid_hk, :mid_wk] = img[0, 0]  # top-left\n",
    "    padded[:mid_hk, w + mid_wk:] = img[0, -1]  # top-right\n",
    "    padded[h + mid_hk:, :mid_wk] = img[-1, 0]  # bottom-left\n",
    "    padded[h + mid_hk:, w + mid_wk:] = img[-1, -1]  # bottom-right\n",
    "    padded[mid_hk:h + mid_hk, mid_wk:w + mid_wk] = img\n",
    "    return padded\n",
    "\n",
    "\n",
    "def convolution(img, kernel):\n",
    "    dim = len(img.shape)\n",
    "    padded = padding_nearest(img, kernel)\n",
    "    if dim == 3:\n",
    "        kernel = np.repeat(kernel[:, :, np.newaxis], 3, axis=2)\n",
    "    return convolution_numba(img, kernel, padded)\n",
    "\n",
    "\n",
    "@jit(nopython=True, cache=True)\n",
    "def convolution_numba(img, kernel, padded):\n",
    "    h, w = img.shape[:2]\n",
    "    hk, wk = kernel.shape[:2]\n",
    "    output = np.zeros(img.shape)\n",
    "    kernel = np.fliplr(np.flipud(kernel))  # flip upside-down and left-right\n",
    "    for row in range(h):\n",
    "        for col in range(w):\n",
    "            output[row, col] = (padded[row:row + hk, col:col + wk] * kernel).sum(0).sum(0)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### reduce, expand, generate Gaussian pyramid, Laplacian pyramid, and collapse of the Laplacian pyramid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce(img,sigma):\n",
    "    kernel=get_gaussian_kernel(sigma=sigma)[0]\n",
    "    return convolution(convolution(img,kernel)[::2,:],kernel.transpose())[:,::2]\n",
    "\n",
    "\n",
    "def expand(img,img_up,sigma):\n",
    "    kernel=2*get_gaussian_kernel(sigma=sigma)[0]\n",
    "    output=np.zeros_like(img_up)\n",
    "    output[::2,::2]=img\n",
    "    return convolution(convolution(output,kernel),kernel.transpose())\n",
    "\n",
    "\n",
    "def pyramid_gaussian(img,sigma,max_layer):\n",
    "    pyramid=[img]\n",
    "    for i in range(max_layer-1):\n",
    "        downsampled=reduce(img,sigma)\n",
    "        pyramid.append(downsampled)\n",
    "        img=downsampled\n",
    "    return np.array(pyramid,dtype=object)\n",
    "\n",
    "\n",
    "def pyramid_laplacian(img,sigma,max_layer):\n",
    "    pyramid=[]\n",
    "    for i in range(max_layer-1):\n",
    "        downsampled=reduce(img,sigma)\n",
    "        upsampled=expand(downsampled,img,sigma)\n",
    "        pyramid.append(img-upsampled)\n",
    "        img=downsampled\n",
    "    pyramid.append(img)\n",
    "    return np.array(pyramid,dtype=object)\n",
    "\n",
    "\n",
    "def collapse(pyramid,sigma):\n",
    "    img=pyramid[-1]\n",
    "    for i in reversed(range(len(pyramid)-1)):\n",
    "        img=expand(img,pyramid[i],sigma)+pyramid[i]\n",
    "    return img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Blend apple and orange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "sigma=1\n",
    "max_layer=6\n",
    "\n",
    "apple=imread('images/apple.png')/255\n",
    "orange=imread('images/orange.png')/255\n",
    "H,W,D=apple.shape\n",
    "\n",
    "laplacian_pyramid_apple=pyramid_laplacian(apple,sigma,max_layer)\n",
    "laplacian_pyramid_orange=pyramid_laplacian(orange,sigma,max_layer)\n",
    "\n",
    "mask=np.zeros_like(apple,dtype=np.double)\n",
    "mask[:,:W//2]=1\n",
    "\n",
    "gaussian_pyramid_mask=pyramid_gaussian(mask,sigma,max_layer)\n",
    "\n",
    "merged=collapse(laplacian_pyramid_apple*gaussian_pyramid_mask+laplacian_pyramid_orange*(1-gaussian_pyramid_mask),sigma)\n",
    "merged[merged<0]=0\n",
    "merged[merged>1]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_images(pyramid_blend=merged,no_blend=apple*mask+orange*(1-mask))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## face swap between a cheetah and a cat\n",
    "### read images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# face 1 is the cheetah face\n",
    "face1=imread('images/face1.png')/255\n",
    "H,W,D=face1.shape\n",
    "\n",
    "# face 2 is the cat face, aligned using rotation and translation.\n",
    "cat=rotate(imread('images/face2.png')/255,-4)\n",
    "face2=np.zeros_like(cat)\n",
    "face2[:H-50,:W-20]=cat[50:,20:]\n",
    "\n",
    "show_images(cheetah=face1, cat=face2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### face masks\n",
    "ref:  \n",
    "https://stackoverflow.com/questions/8647024/how-to-apply-a-disc-shaped-mask-to-a-numpy-array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cy, cx = 350,335 # ellipse center\n",
    "a=240 # major axis\n",
    "b=220 # minor axis\n",
    "\n",
    "y,x = np.ogrid[-cy:H-cy, -cx:W-cx]\n",
    "criteria = b*b*x*x + a*a*y*y <= a*a*b*b\n",
    "\n",
    "mask = np.zeros((H, W,3),dtype=np.double)\n",
    "mask[criteria] = 1\n",
    "mask[~criteria] = 0.5\n",
    "show_images(mask_on_cheetah=face1*mask,mask_on_cat=face2*mask)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cheetah face on cat head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "sigma=1\n",
    "max_layer=10\n",
    "# create laplacian pyramids for cheetah and cat\n",
    "laplacian_pyramid_cheetah=pyramid_laplacian(face1,sigma,max_layer)\n",
    "laplacian_pyramid_cat=pyramid_laplacian(face2,sigma,max_layer)\n",
    "\n",
    "# create mask\n",
    "cy, cx = 350,335 # ellipse center\n",
    "a=240 # major axis\n",
    "b=220 # minor axis\n",
    "\n",
    "y,x = np.ogrid[-cy:H-cy, -cx:W-cx]\n",
    "criteria = b*b*x*x + a*a*y*y <= a*a*b*b\n",
    "\n",
    "mask = np.zeros((H, W,3),dtype=np.double)\n",
    "mask[criteria] = 1\n",
    "\n",
    "# create gaussian pyramids of the mask\n",
    "gaussian_pyramid_mask=pyramid_gaussian(mask,sigma,max_layer)\n",
    "\n",
    "# merge laplacian pyramids with the mask, then collapse for the final merged image\n",
    "merged=collapse(laplacian_pyramid_cheetah*gaussian_pyramid_mask+laplacian_pyramid_cat*(1-gaussian_pyramid_mask),sigma)\n",
    "merged[merged<0]=0\n",
    "merged[merged>1]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_images(cheetah_face_on_cat_head=merged,without_blending=face1*mask+face2*(1-mask))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cat face on cheetah head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "mask = np.ones((H, W,3),dtype=np.double)\n",
    "mask[criteria] = 0\n",
    "\n",
    "gaussian_pyramid_mask=pyramid_gaussian(mask,sigma,max_layer)\n",
    "\n",
    "merged=collapse(laplacian_pyramid_cheetah*gaussian_pyramid_mask+laplacian_pyramid_cat*(1-gaussian_pyramid_mask),sigma)\n",
    "merged[merged<0]=0\n",
    "merged[merged>1]=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_images(cat_face_on_cheetah_head=merged,without_blending=face1*mask+face2*(1-mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}