diffvg/apps/seam_carving.py

"""Retargets an .svg using image-domain seam carving to shrink it."""
import os
import pydiffvg
import argparse
import torch as th
import scipy.ndimage.filters as filters
import numba
import numpy as np
import skimage.io


def energy(im):
    """Compute image energy.

    Args:
        im(np.ndarray) with shape [h, w, 3]: input image.

    Returns:
        (np.ndarray) with shape [h, w]: energy map.
    """
    f_dx = np.array([
        [-1, 0, 1 ],
        [-2, 0, 2 ],
        [-1, 0, 1 ],
    ])
    f_dy = f_dx.T
    dx = filters.convolve(im.mean(2), f_dx)
    dy = filters.convolve(im.mean(2), f_dy)

    return np.abs(dx) + np.abs(dy)


@numba.jit(nopython=True)
def min_seam(e):
    """Finds the seam with minimal cost in an energy map.

    Args:
        e(np.ndarray) with shape [h, w]: energy map.
    
    Returns:
        min_e(np.ndarray) with shape [h, w]: for all (y,x) min_e[y, x]
            is the cost of the minimal seam from 0 to y (top to bottom).
            The minimal seam can be found by looking at the last row of min_e.
            This is computed by dynamic programming.
        argmin_e(np.ndarray) with shape [h, w]: for all (y,x) argmin_e[y, x]
            contains the x coordinate corresponding to this seam in the
            previous row (y-1). We use this for backtracking.
    """
    # initialize to local energy
    min_e = e.copy()
    argmin_e = np.zeros_like(e, dtype=np.int64)

    h, w =  e.shape

    # propagate vertically
    for y in range(1, h):
        for x in range(w):
            if x == 0:
                idx = np.argmin(e[y-1, x:x+2])
                argmin_e[y, x] = idx + x
                mini = e[y-1, x + idx]
            elif x == w-1:
                idx = np.argmin(e[y-1, x-1:x+1])
                argmin_e[y, x] = idx + x - 1
                mini = e[y-1, x + idx - 1]
            else:
                idx = np.argmin(e[y-1, x-1:x+2])
                argmin_e[y, x] = idx + x - 1
                mini = e[y-1, x + idx - 1]

            min_e[y, x] = min_e[y, x] + mini

    return min_e, argmin_e


def carve_seam(im):
    """Carves a vertical seam in an image, reducing it's horizontal size by 1.

    Args:
        im(np.ndarray) with shape [h, w, 3]: input image.

    Returns:
        (np.ndarray) with shape [h, w-1, 1]: the image with one seam removed.
    """

    e = energy(im)
    min_e, argmin_e = min_seam(e)
    h, w =  im.shape[:2]

    # boolean flags for the pixels to preserve
    to_keep = np.ones((h, w), dtype=np.bool)

    # get lowest energy (from last row)
    x = np.argmin(min_e[-1])
    print("carving seam", x, "with energy", min_e[-1, x])

    # backtract to identify the seam
    for y in range(h-1, -1, -1):
        # remove seam pixel
        to_keep[y, x] = False
        x = argmin_e[y, x]

    # replicate mask over color channels
    to_keep = np.stack(3*[to_keep], axis=2)
    new_im = im[to_keep].reshape((h, w-1, 3))
    return new_im


def render(canvas_width, canvas_height, shapes, shape_groups, samples=2):
    _render = pydiffvg.RenderFunction.apply
    scene_args = pydiffvg.RenderFunction.serialize_scene(\
        canvas_width, canvas_height, shapes, shape_groups)

    img = _render(canvas_width, # width
                 canvas_height, # height
                 samples,   # num_samples_x
                 samples,   # num_samples_y
                 0,   # seed
                 None,
                 *scene_args)
    return img


def vector_rescale(shapes, scale_x=1.00, scale_y=1.00):
    new_shapes = []
    for path in shapes:
        path.points[..., 0] *= scale_x
        path.points[..., 1] *= scale_y


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--svg", default=os.path.join("imgs", "hokusai.svg"))
    parser.add_argument("--optim_steps", default=10, type=int)
    parser.add_argument("--lr", default=1e-1, type=int)
    args = parser.parse_args()

    name = os.path.splitext(os.path.basename(args.svg))[0]
    root = os.path.join("results", "seam_carving", name)
    svg_root = os.path.join(root, "svg")
    os.makedirs(root, exist_ok=True)
    os.makedirs(os.path.join(root, "svg"), exist_ok=True)

    pydiffvg.set_use_gpu(False)
    # pydiffvg.set_device(th.device('cuda'))

    # Load SVG
    print("loading svg %s" % args.svg)
    canvas_width, canvas_height, shapes, shape_groups = \
        pydiffvg.svg_to_scene(args.svg)
    print("done loading")

    max_size = 512
    scale_factor = max_size / max(canvas_width, canvas_height)
    print("rescaling from %dx%d with scale %f" % (canvas_width, canvas_height, scale_factor))
    canvas_width = int(canvas_width*scale_factor)
    canvas_height = int(canvas_height*scale_factor)
    print("new shape %dx%d" % (canvas_width, canvas_height))
    vector_rescale(shapes, scale_x=scale_factor, scale_y=scale_factor)

    # Shrink image by 33 %
    # num_seams_to_remove = 2
    num_seams_to_remove = canvas_width // 3
    new_canvas_width  = canvas_width - num_seams_to_remove
    scaling =  new_canvas_width * 1.0 / canvas_width

    # Naive scaling baseline
    print("rendering naive rescaling...")
    vector_rescale(shapes, scale_x=scaling)
    resized = render(new_canvas_width, canvas_height, shapes, shape_groups)
    pydiffvg.imwrite(resized.cpu(), os.path.join(root, 'uniform_scaling.png'), gamma=2.2)
    pydiffvg.save_svg(os.path.join(svg_root, 'uniform_scaling.svg') , canvas_width,
                      canvas_height, shapes, shape_groups, use_gamma=False)
    vector_rescale(shapes, scale_x=1.0/scaling)  # bring back original coordinates
    print("saved naiving scaling")

    # Save initial state
    print("rendering initial state...")
    im = render(canvas_width, canvas_height, shapes, shape_groups)
    pydiffvg.imwrite(im.cpu(), os.path.join(root, 'init.png'), gamma=2.2)
    pydiffvg.save_svg(os.path.join(svg_root, 'init.svg'), canvas_width,
                      canvas_height, shapes, shape_groups, use_gamma=False)
    print("saved initial state")

    # Optimize
    # color_optim = th.optim.Adam(color_vars, lr=0.01)

    retargeted = im[..., :3].cpu().numpy()
    previous_width = canvas_width
    print("carving seams")
    for seam_idx in range(num_seams_to_remove):
        print('\nseam', seam_idx+1, 'of', num_seams_to_remove)

        # Remove a seam
        retargeted = carve_seam(retargeted)

        current_width = canvas_width - seam_idx - 1
        scale_factor = current_width * 1.0 / previous_width
        previous_width = current_width

        padded = np.zeros((canvas_height, canvas_width, 4))
        padded[:, :-seam_idx-1, :3] = retargeted
        padded[:, :-seam_idx-1, -1] = 1.0  # alpha
        padded = th.from_numpy(padded).to(im.device)

        # Remap points to the smaller canvas and
        # collect variables to optimize
        points_vars = []
        # width_vars = []
        mini, maxi = canvas_width, 0
        for path in shapes:
            path.points.requires_grad = False
            x = path.points[..., 0]
            y = path.points[..., 1]
            # rescale

            x = x * scale_factor

            # clip to canvas
            path.points[..., 0] = th.clamp(x, 0, current_width)
            path.points[..., 1] = th.clamp(y, 0, canvas_height)

            path.points.requires_grad = True
            points_vars.append(path.points)
            path.stroke_width.requires_grad = True
            # width_vars.append(path.stroke_width)

            mini = min(mini, path.points.min().item())
            maxi = max(maxi, path.points.max().item())
        print("points", mini, maxi, "scale", scale_factor)

        # recreate an optimizer so we don't carry over the previous update
        # (momentum)?
        geom_optim = th.optim.Adam(points_vars, lr=args.lr)

        for step in range(args.optim_steps):
            geom_optim.zero_grad()

            img = render(canvas_width, canvas_height, shapes, shape_groups,
                         samples=2)

            pydiffvg.imwrite(
                img.cpu(), 
                os.path.join(root, "seam_%03d_iter_%02d.png" % (seam_idx, step)), gamma=2.2)

            # NO alpha
            loss = (img - padded)[..., :3].pow(2).mean()
            # loss = (img - padded).pow(2).mean()
            print('render loss:', loss.item())

            # Backpropagate the gradients.
            loss.backward()

            # Take a gradient descent step.
            geom_optim.step()
        pydiffvg.save_svg(os.path.join(svg_root, "seam%03d.svg" % seam_idx),
                          canvas_width-seam_idx, canvas_height, shapes,
                          shape_groups, use_gamma=False)

        for path in shapes:
            mini = min(mini, path.points.min().item())
            maxi = max(maxi, path.points.max().item())
        print("points", mini, maxi)

    img = render(canvas_width, canvas_height, shapes, shape_groups)
    img = img[:, :-num_seams_to_remove]

    pydiffvg.imwrite(img.cpu(), os.path.join(root, 'final.png'),
                     gamma=2.2)
    pydiffvg.imwrite(retargeted, os.path.join(root, 'ref.png'),
                     gamma=2.2)

    pydiffvg.save_svg(os.path.join(svg_root, 'final.svg'),
                      canvas_width-seam_idx, canvas_height, shapes,
                      shape_groups, use_gamma=False)

    # Convert the intermediate renderings to a video.
    from subprocess import call
    call(["ffmpeg", "-framerate", "24", "-i", os.path.join(root, "seam_%03d_iter_00.png"), "-vb", "20M",
         os.path.join(root, "out.mp4")])


if __name__ == "__main__":
    main()