Introduction

2 Notebooks ago, I published a Notebook about the technic of Seam Carving. In this Notebook, we will use this method on some images and a pre-trained metwork to check if there is benefits in the prediction for non squared images.

In [4]:
import sys
import numba
import numpy as np
from imageio import imread, imwrite
from scipy.ndimage.filters import convolve
import matplotlib.pyplot as plt
from skimage import transform,io
import tensorflow as tf
from tensorflow.keras.applications.inception_v3 import decode_predictions

from scipy.stats import entropy
from numpy.linalg import norm

from PIL import Image
import cv2

Preparation

Let's load 3 images of high ratio, scale them normally and with Seam Carving. This will be used for the prediction. The rest is very similar to what was done on the previous Notebook about Class Activation Map

In [10]:
def calc_energy(img):
    filter_du = np.array([
        [1.0, 2.0, 1.0],
        [0.0, 0.0, 0.0],
        [-1.0, -2.0, -1.0],
    ])
    # This converts it from a 2D filter to a 3D filter, replicating the same
    # filter for each channel: R, G, B
    filter_du = np.stack([filter_du] * 3, axis=2)

    filter_dv = np.array([
        [1.0, 0.0, -1.0],
        [2.0, 0.0, -2.0],
        [1.0, 0.0, -1.0],
    ])
    # This converts it from a 2D filter to a 3D filter, replicating the same
    # filter for each channel: R, G, B
    filter_dv = np.stack([filter_dv] * 3, axis=2)

    img = img.astype('float32')
    convolved = np.absolute(convolve(img, filter_du)) + np.absolute(convolve(img, filter_dv))

    # We sum the energies in the red, green, and blue channels
    energy_map = convolved.sum(axis=2)

    return energy_map

@numba.jit
def minimum_seam(img):
    r, c, _ = img.shape
    energy_map = calc_energy(img)

    M = energy_map.copy()
    backtrack = np.zeros_like(M, dtype=np.int)

    for i in range(1, r):
        for j in range(0, c):
            # Handle the left edge of the image, to ensure we don't index -1
            if j == 0:
                idx = np.argmin(M[i - 1, j:j + 2])
                backtrack[i, j] = idx + j
                min_energy = M[i - 1, idx + j]
            else:
                idx = np.argmin(M[i - 1, j - 1:j + 2])
                backtrack[i, j] = idx + j - 1
                min_energy = M[i - 1, idx + j - 1]

            M[i, j] += min_energy

    return M, backtrack

@numba.jit
def carve_column(img):
    r, c, _ = img.shape

    M, backtrack = minimum_seam(img)

    # Create a (r, c) matrix filled with the value True
    # We'll be removing all pixels from the image which
    # have False later
    mask = np.ones((r, c), dtype=np.bool)

    # Find the position of the smallest element in the
    # last row of M
    j = np.argmin(M[-1])
    
    # fill the table energy with the energy fo the best path
    if 'energy' in globals():
        global energy
        energy.append(np.min(M[-1]))
    
    for i in reversed(range(r)):
        # Mark the pixels for deletion
        mask[i, j] = False
        j = backtrack[i, j]

    # Since the image has 3 channels, we convert our
    # mask to 3D
    mask = np.stack([mask] * 3, axis=2)

    # Delete all the pixels marked False in the mask,
    # and resize it to the new image dimensions
    img = img[mask].reshape((r, c - 1, 3))

    return img


def crop_c(img, scale_c):
    r, c, _ = img.shape
    if isinstance(scale_c, float):
        new_c = int(scale_c * c)
    else:
        new_c = c-scale_c  # give a range(scale_c)

    for i in range(c - new_c):
        img = carve_column(img)

    return img

def crop_r(img, scale_r):
    img = np.rot90(img, 1, (0, 1))
    img = crop_c(img, scale_r)
    img = np.rot90(img, 3, (0, 1))
    return img
In [ ]:
test_image1 = imread("img1_original.jpg")
test_image2 = imread("img2_original.jpg")
test_image3 = imread("img3_original.jpg")
test_image4 = crop_c(test_image1, 660-281)
test_image5 = crop_c(test_image2, 870-310)
test_image6 = crop_c(test_image3, 1440-597)
In [48]:
test_image1 = transform.resize(test_image1, (299, 299))
test_image2 = transform.resize(test_image2, (299, 299))
test_image3 = transform.resize(test_image3, (299, 299))
test_image4 = transform.resize(test_image4, (299, 299))
test_image5 = transform.resize(test_image5, (299, 299))
test_image6 = transform.resize(test_image6, (299, 299))
C:\python36\envs\machine_learning\lib\site-packages\skimage\transform\_warps.py:84: UserWarning: The default mode, 'constant', will be changed to 'reflect' in skimage 0.15.
  warn("The default mode, 'constant', will be changed to 'reflect' in "

Let's plot both pre-processed images.

In [49]:
fig, axes = plt.subplots(2, 3, figsize=(20,12))
axes[0, 0].imshow(test_image1)
axes[0, 1].imshow(test_image2)
axes[0, 2].imshow(test_image3)
axes[1, 0].imshow(test_image4)
axes[1, 1].imshow(test_image5)
axes[1, 2].imshow(test_image6)
plt.show()