1 late day used. Note one "bells and whistles" using the differentiable augmentation library in part 1

# CMU 16-726 Learning-Based Image Synthesis / Spring 2021, Assignment 3
# Conversion to notebook and edits by Trenton Tabor for course above
# The code base is based on the great work from CSC 321, U Toronto
# https://www.cs.toronto.edu/~rgrosse/courses/csc321_2018/assignments/a4-code.zip

import glob
import os
import PIL.Image as Image
import torch
import torch.optim as optim
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms
import torch.nn as nn
from torch.nn import init
import torch.nn.functional as F
from torch.utils.tensorboard import SummaryWriter
import numpy as np
import matplotlib.pyplot as mpl
import skimage as sk
import skimage.io as skio

import vanilla_gan as vg
import cycle_gan as cg
import models as models
import utils
from DiffAugment_pytorch import DiffAugment

# dangerous!
import warnings
warnings.simplefilter('ignore')

class Object(object):
    pass

Implement Data Augmentation

class CustomDataSet(Dataset):
    """Load images under folders"""
    def __init__(self, main_dir, ext='*.png', transform=None):
        self.main_dir = main_dir
        self.transform = transform
        all_imgs = glob.glob(os.path.join(main_dir, ext))
        self.total_imgs = all_imgs
        print(os.path.join(main_dir, ext))
        print(len(self))

    def __len__(self):
        return len(self.total_imgs)

    def __getitem__(self, idx):
        img_loc = self.total_imgs[idx]
        image = Image.open(img_loc).convert("RGB")
        tensor_image = self.transform(image)
        return tensor_image, 0.

def get_data_loader(data_path, opts):
    """Creates data loaders.
    """
    basic_transform = transforms.Compose([
        transforms.Resize(opts.image_size, Image.BICUBIC),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    ])

    if opts.data_aug == 'basic':
        transform = basic_transform
    elif opts.data_aug == 'deluxe':
        # todo: add your code here: below are some ideas for your reference
        # load_size = int(1.1 * opts.image_size)
        # osize = [load_size, load_size]
        # transforms.Resize(osize, Image.BICUBIC)
        # transforms.RandomCrop(opts.image_size)
        # transforms.RandomHorizontalFlip()
        print('deluxe activated')
        transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.RandomResizedCrop(opts.image_size, interpolation=Image.BICUBIC,
                                         scale=(0.8,1.0), # I didn't want to crop a lot, I don't want to predict cropped cats
                                         ratio=(0.7,1.4)), # additional option for aspect ratio, kept this small
            transforms.RandomHorizontalFlip(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
        ])

    dataset = CustomDataSet(os.path.join('data/', data_path), opts.ext, transform)
    print(dataset)
    dloader = DataLoader(dataset=dataset, batch_size=opts.batch_size, shuffle=True, num_workers=opts.num_workers)

    return dloader

Implement the Discriminator

From pytorch docs ( https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html)

layers 1-4

h = ⌊(𝑛ℎ+2×𝑝ℎ−dh×(𝑘ℎ-1)-1)/𝑠ℎ+1⌋

32 = ⌊(64+2×𝑝ℎ−1×(4-1)-1)/2+1⌋

32 = ⌊(64+2×𝑝ℎ−3-1)/2+1⌋

32 = ⌊(60+2×𝑝ℎ)/2+1⌋

31 = (60+2×𝑝ℎ)/2

62 = (60+2×𝑝ℎ)

2 = 2×𝑝ℎ

1 = 𝑝ℎ

layer 5

h = ⌊(𝑛ℎ+2×𝑝ℎ−dh×(𝑘ℎ-1)-1)/𝑠ℎ+1⌋

1 = ⌊(4+2×𝑝ℎ−1×(4-1)-1)/2+1⌋

1 = ⌊(4+2×𝑝ℎ−3-1)/2+1⌋

1 = ⌊(0+2×𝑝ℎ)/2+1⌋

0 = (0+2×𝑝ℎ)/2

0 = (0+2×𝑝ℎ)

0 = 𝑝ℎ

class DCDiscriminator(nn.Module):
    """Defines the architecture of the discriminator network.
       Note: Both discriminators D_X and D_Y have the same architecture in this assignment.
    """
    def __init__(self, conv_dim=64, norm='batch'):
        super(DCDiscriminator, self).__init__()

        ###########################################
        ##   FILL THIS IN: CREATE ARCHITECTURE   ##
        ###########################################

        self.conv1 = models.conv(3, conv_dim, 4, stride=2, padding=1, norm='instance')
        self.conv2 = models.conv(conv_dim, conv_dim*2, 4, stride=2, padding=1, norm='instance')
        self.conv3 = models.conv(conv_dim*2, conv_dim*4, 4, stride=2, padding=1, norm='instance')
        self.conv4 = models.conv(conv_dim*4, conv_dim*8, 4, stride=2, padding=1, norm='instance')
        self.conv5 = models.conv(conv_dim*8, 1, 4, stride=2, padding=0, norm='')

    def forward(self, x):
        out = F.relu(self.conv1(x))

        ###########################################
        ##   FILL THIS IN: FORWARD PASS   ##
        ###########################################
        out = F.relu(self.conv2(out))
        out = F.relu(self.conv3(out))
        out = F.relu(self.conv4(out))

        out = self.conv5(out).squeeze()
        return out

Implement Generator

From https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html Hout=(Hin−1)×stride−2×padding+dilation×(kernel_size−1)+output_padding+1

layer 1

4 = (1-1)×2-2×padding+1×3+0+1

4 = 0-2×padding+1×3+0+1

4 = 2×padding+4

0 = 2×padding

0 = padding

layers 2-5

8 = (4-1)×2-2×padding+1×3+0+1

8 = 6-2×padding+1×3+0+1

8 = 6-2×padding+3+1

-2 = -2×padding

1 = padding

class DCGenerator(nn.Module):
    def __init__(self, noise_size, conv_dim):
        super(DCGenerator, self).__init__()

        ###########################################
        ##   FILL THIS IN: CREATE ARCHITECTURE   ##
        ###########################################

        self.deconv1 = models.deconv(noise_size, conv_dim*8, 4, stride=2, padding=0, norm='instance')
        self.deconv2 = models.deconv(conv_dim*8, conv_dim*4, 4, stride=2, padding=1, norm='instance')
        self.deconv3 = models.deconv(conv_dim*4, conv_dim*2, 4, stride=2, padding=1, norm='instance')
        self.deconv4 = models.deconv(conv_dim*2, conv_dim,   4, stride=2, padding=1, norm='instance')
        self.deconv5 = models.deconv(conv_dim,          3,   4, stride=2, padding=1, norm='')

    def forward(self, z):
        """Generates an image given a sample of random noise.

            Input
            -----
                z: BS x noise_size x 1 x 1   -->  16x100x1x1

            Output
            ------
                out: BS x channels x image_width x image_height  -->  16x3x32x32
        """


        ###########################################
        ##   FILL THIS IN: FORWARD PASS   ##
        ###########################################
        out = F.relu(self.deconv1(z))
        out = F.relu(self.deconv2(out))
        out = F.relu(self.deconv3(out))
        out = F.relu(self.deconv4(out))
        out = F.tanh(self.deconv5(out))
        return out

def create_model(opts):
    """Builds the generators and discriminators.
    """
    G = DCGenerator(noise_size=opts.noise_size, conv_dim=opts.conv_dim)
    D = DCDiscriminator(conv_dim=opts.conv_dim)

    vg.print_models(G, D)

    if torch.cuda.is_available():
        G.cuda()
        D.cuda()
        print('Models moved to GPU.')

    return G, D

def training_loop(train_dataloader, opts):
    """Runs the training loop.
        * Saves checkpoints every opts.checkpoint_every iterations
        * Saves generated samples every opts.sample_every iterations
    """

    # Create generators and discriminators
    G, D = create_model(opts)

    # Create optimizers for the generators and discriminators
    g_optimizer = optim.Adam(G.parameters(), opts.lr, [opts.beta1, opts.beta2])
    d_optimizer = optim.Adam(D.parameters(), opts.lr, [opts.beta1, opts.beta2])

    # Generate fixed noise for sampling from the generator
    fixed_noise = vg.sample_noise(opts.batch_size,opts.noise_size)  # batch_size x noise_size x 1 x 1

    iteration = 1

    total_train_iters = opts.num_epochs * len(train_dataloader)
    
    output = Object()
    
    output.iteration_viz = np.zeros([total_train_iters//opts.sample_every,1])
    output.vizPath = []
    output.iteration = np.zeros([total_train_iters//opts.log_step,1])
    output.D_real_loss = np.zeros([total_train_iters//opts.log_step,1])
    output.D_fake_loss = np.zeros([total_train_iters//opts.log_step,1])
    output.G_loss = np.zeros([total_train_iters//opts.log_step,1])

    for epoch in range(opts.num_epochs):

        for batch in train_dataloader:

            real_images, labels = batch
            real_images, labels = utils.to_var(real_images), utils.to_var(labels).long().squeeze()

            ################################################
            ###         TRAIN THE DISCRIMINATOR         ####
            ################################################

            d_optimizer.zero_grad()

            # FILL THIS IN
            # 1. Compute the discriminator loss on real images
            if (opts.diffaug):
                D_real_loss = 1/(2*opts.batch_size) * sum((D(DiffAugment(real_images,opts.diffaug))-1)**2)
            else: 
                D_real_loss = 1/(2*opts.batch_size) * sum((D(real_images)-1)**2)

            # 2. Sample noise
            noise = vg.sample_noise(opts.batch_size,opts.noise_size)

            # 3. Generate fake images from the noise
            fake_images = G(noise)

            # 4. Compute the discriminator loss on the fake images
            if (opts.diffaug):
                D_fake_loss = 1/(2*opts.batch_size) * sum((D(DiffAugment(fake_images,opts.diffaug)))**2)
            else:
                D_fake_loss = 1/(2*opts.batch_size) * sum((D(fake_images))**2)
            
            D_total_loss = D_fake_loss + D_real_loss
            if iteration % 2 == 0:
                D_total_loss.backward()
                d_optimizer.step()

            ###########################################
            ###          TRAIN THE GENERATOR        ###
            ###########################################

            g_optimizer.zero_grad()

            # FILL THIS IN
            # 1. Sample noise
            noise = vg.sample_noise(opts.batch_size,opts.noise_size)

            # 2. Generate fake images from the noise
            fake_images = G(noise)

            # 3. Compute the generator loss
            if (opts.diffaug):
                G_loss = 1/(opts.batch_size) * sum((D(DiffAugment(fake_images,opts.diffaug))-1)**2)
            else:
                G_loss = 1/(opts.batch_size) * sum((D(fake_images)-1)**2)
                                                          
            G_loss.backward()
            g_optimizer.step()


            # Print the log info
            if iteration % opts.log_step == 0:
                #print('Iteration [{:4d}/{:4d}] | D_real_loss: {:6.4f} | D_fake_loss: {:6.4f} | G_loss: {:6.4f}'.format(
                #       iteration, total_train_iters, D_real_loss.item(), D_fake_loss.item(), G_loss.item()))
                output.iteration[iteration//opts.log_step-1] = iteration
                output.D_real_loss[iteration//opts.log_step-1] = float(D_real_loss.item())
                output.D_fake_loss[iteration//opts.log_step-1] = float(D_fake_loss.item())
                output.G_loss[iteration//opts.log_step-1] = float(G_loss.item())
            # todo: add fake loss, real loss, G loss to tensorboard

            # Save the generated samples
            if iteration % opts.sample_every == 0:
                path = vg.save_samples(G, fixed_noise, iteration, opts)
                vg.save_images(real_images, iteration, opts, 'real')
                output.vizPath.append(path)
                output.iteration_viz[iteration//opts.sample_every-1] = iteration
                

            # Save the model parameters
            if iteration % opts.checkpoint_every == 0:
                vg.checkpoint(iteration, G, D, opts)

            iteration += 1
            
    return output

def main_cg(opts):
    """Loads the data, creates checkpoint and sample directories, and starts the training loop.
    """

    # Create a dataloader for the training images
    dataloader = get_data_loader(opts.data, opts)

    # Create checkpoint and sample directories
    utils.create_dir(opts.checkpoint_dir)
    utils.create_dir(opts.sample_dir)

    trainingOut = training_loop(dataloader, opts)
    
    mpl.rcParams['figure.figsize'] = [8, 6]
    mpl.plot(trainingOut.iteration, trainingOut.D_real_loss, 'b', label='D Real Loss')
    mpl.plot(trainingOut.iteration, trainingOut.D_fake_loss, 'r', label='D Fake Loss')
    mpl.plot(trainingOut.iteration, trainingOut.G_loss, 'g', label='G Loss')
    mpl.legend()
    mpl.show()
    
    for ind in range(0,len(trainingOut.vizPath),max(len(trainingOut.vizPath)//6,1)):
        print("result at iteration " + str(trainingOut.iteration_viz[ind]))
        im = skio.imread(trainingOut.vizPath[ind])
        skio.imshow(im)
        skio.show()

opts = Object()

# Model hyper-parameters
opts.image_size = 64 # 'The side length N to convert images to NxN.'
opts.conv_dim = 32
opts.noise_size = 100

# Training hyper-parameters
opts.num_epochs = 400 # default 40
opts.batch_size = 16 # 'The number of images in a batch.'
opts.num_workers = 0 # 'The number of threads to use for the DataLoader.'
opts.lr = 0.0003 # 'The learning rate (default 0.0003'
opts.beta1 = 0.5
opts.beta2 = 0.999

# Data sources
opts.data = 'cat/grumpifyBprocessed' # 'Choose the type of emojis to generate.'
opts.ext = '*.png' # 'Choose the type of emojis to generate.'

# Directories and checkpoint/sample iterations
opts.checkpoint_dir = './checkpoints_vanilla'
opts.sample_dir = './vanilla'
opts.log_step = 10
opts.sample_every = 1500
opts.checkpoint_every = 400

Basic Augmentation

opts.data_aug = 'basic' # 'data augmentation diff / basic / deluxe'
opts.diffaug = ''

leafdir = '%s_%s_%s' % (os.path.basename(opts.data), opts.data_aug,opts.diffaug)
opts.sample_dir = os.path.join('output/', opts.sample_dir,
                               leafdir)
if os.path.exists(opts.sample_dir):
    cmd = 'rm %s/*' % opts.sample_dir
    os.system(cmd)
logger = SummaryWriter(opts.sample_dir)
print(opts)

trainingout = main_cg(opts)

<__main__.Object object at 0x7f1b54556650>
data/cat/grumpifyBprocessed/*.png
204
<__main__.CustomDataSet object at 0x7f1b54692390>
                    G                  
---------------------------------------
DCGenerator(
  (deconv1): Sequential(
    (0): ConvTranspose2d(100, 256, kernel_size=(4, 4), stride=(2, 2), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv2): Sequential(
    (0): ConvTranspose2d(256, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv3): Sequential(
    (0): ConvTranspose2d(128, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv4): Sequential(
    (0): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv5): Sequential(
    (0): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
  )
)
---------------------------------------
                    D                  
---------------------------------------
DCDiscriminator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv3): Sequential(
    (0): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv4): Sequential(
    (0): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv5): Sequential(
    (0): Conv2d(256, 1, kernel_size=(4, 4), stride=(2, 2), bias=False)
  )
)
---------------------------------------
Models moved to GPU.

Lossy conversion from float32 to uint8. Range [-0.9884359836578369, 0.9945548176765442]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0, 1.0]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9996617436408997, 0.9996916651725769]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0, 1.0]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9992828369140625, 0.9998173117637634]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0, 1.0]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.997160792350769, 0.9979491829872131]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0, 1.0]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9841479659080505, 0.987975001335144]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0, 1.0]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9957241415977478, 0.9995067119598389]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0, 1.0]. Convert image to uint8 prior to saving to suppress this warning.

result at iteration [800.]

result at iteration [1600.]

result at iteration [2400.]

result at iteration [3200.]

result at iteration [4000.]

result at iteration [4800.]

Deluxe Augmentation

opts.data_aug = 'deluxe' # 'data augmentation diff / basic / deluxe'
opts.diffaug = ''

leafdir = '%s_%s_%s' % (os.path.basename(opts.data), opts.data_aug,opts.diffaug)
opts.sample_dir = os.path.join('output/', opts.sample_dir,
                               leafdir)
if os.path.exists(opts.sample_dir):
    cmd = 'rm %s/*' % opts.sample_dir
    os.system(cmd)
logger = SummaryWriter(opts.sample_dir)
print(opts)

trainingout = main_cg(opts)

<__main__.Object object at 0x7f1b54556650>
deluxe activated
data/cat/grumpifyBprocessed/*.png
204
<__main__.CustomDataSet object at 0x7f1b544e8750>
                    G                  
---------------------------------------
DCGenerator(
  (deconv1): Sequential(
    (0): ConvTranspose2d(100, 256, kernel_size=(4, 4), stride=(2, 2), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv2): Sequential(
    (0): ConvTranspose2d(256, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv3): Sequential(
    (0): ConvTranspose2d(128, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv4): Sequential(
    (0): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv5): Sequential(
    (0): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
  )
)
---------------------------------------
                    D                  
---------------------------------------
DCDiscriminator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv3): Sequential(
    (0): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv4): Sequential(
    (0): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv5): Sequential(
    (0): Conv2d(256, 1, kernel_size=(4, 4), stride=(2, 2), bias=False)
  )
)
---------------------------------------
Models moved to GPU.

Lossy conversion from float32 to uint8. Range [-0.9816900491714478, 0.975562572479248]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.1358473300933838, 1.135138988494873]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9993252158164978, 0.9997537136077881]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.1102163791656494, 1.1017520427703857]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9992446899414062, 0.998241126537323]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.1243468523025513, 1.1064558029174805]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9956967830657959, 0.9952065348625183]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.068445086479187, 1.0737276077270508]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9973799586296082, 0.9965782761573792]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0942877531051636, 1.1060471534729004]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9962162971496582, 0.9993811845779419]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0538065433502197, 1.0620863437652588]. Convert image to uint8 prior to saving to suppress this warning.

result at iteration [800.]

result at iteration [1600.]

result at iteration [2400.]

result at iteration [3200.]

result at iteration [4000.]

result at iteration [4800.]

Deluxe + Diff Data Aug

opts.data_aug = 'deluxe' # 'data augmentation diff / basic / deluxe'
opts.diffaug = 'color,translation,cutout'

leafdir = '%s_%s_%s' % (os.path.basename(opts.data), opts.data_aug,opts.diffaug)
opts.sample_dir = os.path.join('output/', opts.sample_dir,
                               leafdir)
if os.path.exists(opts.sample_dir):
    cmd = 'rm %s/*' % opts.sample_dir
    os.system(cmd)
logger = SummaryWriter(opts.sample_dir)
print(opts)

trainingout = main_cg(opts)

<__main__.Object object at 0x7f1b54556650>
deluxe activated
data/cat/grumpifyBprocessed/*.png
204
<__main__.CustomDataSet object at 0x7f1b54484a90>
                    G                  
---------------------------------------
DCGenerator(
  (deconv1): Sequential(
    (0): ConvTranspose2d(100, 256, kernel_size=(4, 4), stride=(2, 2), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv2): Sequential(
    (0): ConvTranspose2d(256, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv3): Sequential(
    (0): ConvTranspose2d(128, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv4): Sequential(
    (0): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv5): Sequential(
    (0): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
  )
)
---------------------------------------
                    D                  
---------------------------------------
DCDiscriminator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv3): Sequential(
    (0): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv4): Sequential(
    (0): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv5): Sequential(
    (0): Conv2d(256, 1, kernel_size=(4, 4), stride=(2, 2), bias=False)
  )
)
---------------------------------------
Models moved to GPU.

Lossy conversion from float32 to uint8. Range [-0.9999356865882874, 0.9962500333786011]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.075476050376892, 1.1091561317443848]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9994246959686279, 0.9991104006767273]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.1388424634933472, 1.1023967266082764]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9993036985397339, 0.9996132850646973]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0935463905334473, 1.084238052368164]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.99967360496521, 0.9972966313362122]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0676584243774414, 1.094282627105713]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9993047118186951, 0.9993261098861694]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.1561263799667358, 1.072535753250122]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-0.9977079629898071, 0.9947453141212463]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float32 to uint8. Range [-1.0989060401916504, 1.1292095184326172]. Convert image to uint8 prior to saving to suppress this warning.

result at iteration [800.]

result at iteration [1600.]

result at iteration [2400.]

result at iteration [3200.]

result at iteration [4000.]

result at iteration [4800.]

DCGAN Comments

1. What should the training curve look like?

The loss on the discriminator should start high, from random weights. It should then get lower, as it learns to discriminate. Eventually, it should increase or stabilize as the generator gets better. The loss on the generator should start high, from random weights. It will be unstable, especially at first. Eventually, it may stabilize and enter a stalemate with the discriminator. With differentiable augmentation, we hope that both losses will be more stable

1. Do these curves exhibit this behavior?

It is difficult to tell, do to the instability. However, the differentiable data aug curve is much more stable (It also seems to take longer to reach a minimum)

1. What do you notice about the samples?

Due to the instability, samples without differentiable data augmentation might be really bad, even after training has converged. Initial samples seem to focus on getting colors into the correct place. Afterwards, details start to appear. For instance, the nose is basically missing on early iterations

Cycle GAN

class CycleGenerator(nn.Module):
    """Defines the architecture of the generator network.
       Note: Both generators G_XtoY and G_YtoX have the same architecture in this assignment.
    """
    def __init__(self, conv_dim=64, init_zero_weights=False, norm='batch'):
        super(CycleGenerator, self).__init__()

        ###########################################
        ##   FILL THIS IN: CREATE ARCHITECTURE   ##
        ###########################################

        # 1. Define the encoder part of the generator (that extracts features from the input image)
        self.conv1 = models.conv(3, conv_dim, 4, stride=2, padding=1, norm='instance')
        self.conv2 = models.conv(conv_dim, conv_dim // 2, 4, stride=2, padding=1, norm='instance')

        # 2. Define the transformation part of the generator
        self.resnet_block1 = models.ResnetBlock(conv_dim//2, 'instance')
        self.resnet_block2 = models.ResnetBlock(conv_dim//2, 'instance')
        self.resnet_block3 = models.ResnetBlock(conv_dim//2, 'instance')

        # 3. Define the decoder part of the generator (that builds up the output image from features)
        self.deconv1 = models.deconv(conv_dim // 2, conv_dim,   4, stride=2, padding=1, norm='instance')
        self.deconv2 = models.deconv(conv_dim,          3,   4, stride=2, padding=1, norm='')

    def forward(self, x):
        """Generates an image conditioned on an input image.

            Input
            -----
                x: BS x 3 x 32 x 32

            Output
            ------
                out: BS x 3 x 32 x 32
        """

        out = F.relu(self.conv1(x))
        out = F.relu(self.conv2(out))

        out = F.relu(self.resnet_block1(out))
        out = F.relu(self.resnet_block2(out))
        out = F.relu(self.resnet_block3(out))

        out = F.relu(self.deconv1(out))
        out = F.tanh(self.deconv2(out))

        return out

def create_model(opts):
    """Builds the generators and discriminators.
    """
    model_dict = {'cycle': CycleGenerator}
    G_XtoY = model_dict[opts.gen](conv_dim=opts.g_conv_dim, init_zero_weights=opts.init_zero_weights, norm=opts.norm)
    G_YtoX = model_dict[opts.gen](conv_dim=opts.g_conv_dim, init_zero_weights=opts.init_zero_weights, norm=opts.norm)

    model_dict = {'dc': DCDiscriminator}
    D_X = model_dict[opts.disc](conv_dim=opts.d_conv_dim, norm=opts.norm)
    D_Y = model_dict[opts.disc](conv_dim=opts.d_conv_dim, norm=opts.norm)
    cg.print_models(G_XtoY, G_YtoX, D_X, D_Y)

    # todo: B&W add your own initialization here
    # 

    if torch.cuda.is_available():
        G_XtoY.cuda()
        G_YtoX.cuda()
        D_X.cuda()
        D_Y.cuda()
        print('Models moved to GPU.')

    return G_XtoY, G_YtoX, D_X, D_Y

def training_loop_cg(dataloader_X, dataloader_Y, opts):
    """Runs the training loop.
        * Saves checkpoint every opts.checkpoint_every iterations
        * Saves generated samples every opts.sample_every iterations
    """

    output = Object()
    
    # Create generators and discriminators
    G_XtoY, G_YtoX, D_X, D_Y = create_model(opts)

    g_params = list(G_XtoY.parameters()) + list(G_YtoX.parameters())  # Get generator parameters
    d_params = list(D_X.parameters()) + list(D_Y.parameters())  # Get discriminator parameters

    # Create optimizers for the generators and discriminators
    g_optimizer = optim.Adam(g_params, opts.lr, [opts.beta1, opts.beta2])
    d_optimizer = optim.Adam(d_params, opts.lr, [opts.beta1, opts.beta2])

    iter_X = iter(dataloader_X)
    iter_Y = iter(dataloader_Y)


    # Get some fixed data from domains X and Y for sampling. These are images that are held
    # constant throughout training, that allow us to inspect the model's performance.
    fixed_X = utils.to_var(iter_X.next()[0])
    fixed_Y = utils.to_var(iter_Y.next()[0])

    iter_per_epoch = min(len(iter_X), len(iter_Y))
    
    output.iteration_viz = np.zeros([opts.train_iters//opts.sample_every,1])
    output.vizPath = []
    
    output.iteration = np.zeros([opts.train_iters//opts.log_step,1])
    output.D_real_loss = np.zeros([opts.train_iters//opts.log_step,1])
    output.D_X_loss = np.zeros([opts.train_iters//opts.log_step,1])
    output.D_Y_loss = np.zeros([opts.train_iters//opts.log_step,1])
    output.G_loss = np.zeros([opts.train_iters//opts.log_step,1])

    for iteration in range(1, opts.train_iters+1):

        # Reset data_iter for each epoch
        if iteration % iter_per_epoch == 0:
            iter_X = iter(dataloader_X)
            iter_Y = iter(dataloader_Y)

        images_X, labels_X = iter_X.next()
        images_X, labels_X = utils.to_var(images_X), utils.to_var(labels_X).long().squeeze()

        images_Y, labels_Y = iter_Y.next()
        images_Y, labels_Y = utils.to_var(images_Y), utils.to_var(labels_Y).long().squeeze()


        # ============================================
        #            TRAIN THE DISCRIMINATORS
        # ============================================

        #########################################
        ##             FILL THIS IN            ##
        #########################################

        # Train with real images
        d_optimizer.zero_grad()

        # 1. Compute the discriminator losses on real images
        D_X_loss = 1/(images_X.shape[0]) * sum((D_X(images_X)-1)**2)
        D_Y_loss = 1/(images_Y.shape[0]) * sum((D_Y(images_Y)-1)**2)

        d_real_loss = D_X_loss + D_Y_loss
        d_real_loss.backward()
        d_optimizer.step()
        logger.add_scalar('D/XY/real', D_X_loss, iteration)
        logger.add_scalar('D/YX/real', D_Y_loss, iteration)
        # Train with fake images
        d_optimizer.zero_grad()

        # 2. Generate fake images that look like domain X based on real images in domain Y
        fake_X = G_YtoX(images_Y)

        # 3. Compute the loss for D_X
        D_X_loss = 1/(fake_X.shape[0]) * sum((D_X(fake_X))**2)

        # 4. Generate fake images that look like domain Y based on real images in domain X
        fake_Y = G_XtoY(images_X)

        # 5. Compute the loss for D_Y
        D_Y_loss = 1/(fake_Y.shape[0]) * sum((D_Y(fake_Y))**2)

        d_fake_loss = D_X_loss + D_Y_loss
        if iteration % 2 == 0:
            d_fake_loss.backward()
            d_optimizer.step()
        logger.add_scalar('D/XY/fake', D_X_loss, iteration)
        logger.add_scalar('D/YX/fake', D_Y_loss, iteration)


        # =========================================
        #            TRAIN THE GENERATORS
        # =========================================


        #########################################
        ##    FILL THIS IN: Y--X-->Y CYCLE     ##
        #########################################
        g_optimizer.zero_grad()

        # 1. Generate fake images that look like domain X based on real images in domain Y
        fake_X = G_YtoX(images_Y)

        # 2. Compute the generator loss based on domain X
        g_loss = 1/(fake_X.shape[0]) * sum((D_X(fake_X)-1)**2)
        logger.add_scalar('G/XY/fake', g_loss, iteration)

        if opts.use_cycle_consistency_loss:
            reconstructed_Y = G_XtoY(fake_X)
            # 3. Compute the cycle consistency loss (the reconstruction loss)
            cycle_consistency_loss = 1/torch.numel(images_Y) * torch.sum( torch.abs(images_Y-reconstructed_Y))
                        
            g_loss += opts.lambda_cycle * cycle_consistency_loss
            logger.add_scalar('G/XY/cycle', opts.lambda_cycle * cycle_consistency_loss, iteration)

        g_loss.backward()
        g_optimizer.step()



        #########################################
        ##    FILL THIS IN: X--Y-->X CYCLE     ##
        #########################################

        g_optimizer.zero_grad()

        # 1. Generate fake images that look like domain Y based on real images in domain X
        fake_Y = G_XtoY(images_X)

        # 2. Compute the generator loss based on domain Y
        g_loss = 1/(fake_Y.shape[0]) * sum((D_Y(fake_Y)-1)**2)
        logger.add_scalar('G/YX/fake', g_loss, iteration)

        if opts.use_cycle_consistency_loss:
            reconstructed_X = G_YtoX(fake_Y)
            # 3. Compute the cycle consistency loss (the reconstruction loss)
            cycle_consistency_loss = 1/torch.numel(images_X) * torch.sum(torch.abs(images_X-reconstructed_X))
                        
            g_loss += opts.lambda_cycle * cycle_consistency_loss
            logger.add_scalar('G/YX/cycle', cycle_consistency_loss, iteration)

        g_loss.backward()
        g_optimizer.step()

        # Print the log info
        if iteration % opts.log_step == 0:
            #print('Iteration [{:5d}/{:5d}] | d_real_loss: {:6.4f} | d_Y_loss: {:6.4f} | d_X_loss: {:6.4f} | '
            #      'd_fake_loss: {:6.4f} | g_loss: {:6.4f}'.format(
            #        iteration, opts.train_iters, d_real_loss.item(), D_Y_loss.item(),
            #        D_X_loss.item(), d_fake_loss.item(), g_loss.item()))
            output.iteration[iteration//opts.log_step-1] = iteration
            output.D_real_loss[iteration//opts.log_step-1] = float(d_real_loss.item())
            output.D_X_loss[iteration//opts.log_step-1] = float(D_X_loss.item())
            output.D_Y_loss[iteration//opts.log_step-1] = float(D_Y_loss.item())
            output.G_loss[iteration//opts.log_step-1] = float(g_loss.item())

        # Save the generated samples
        if iteration % opts.sample_every == 0:
            paths = cg.save_samples(iteration, fixed_Y, fixed_X, G_YtoX, G_XtoY, opts)
            output.vizPath.append(paths)
            output.iteration_viz[iteration//opts.sample_every-1] = iteration

        # Save the model parameters
        if iteration % opts.checkpoint_every == 0:
            cg.checkpoint(iteration, G_XtoY, G_YtoX, D_X, D_Y, opts)
            
    
    mpl.rcParams['figure.figsize'] = [5, 3]
    mpl.plot(output.iteration, output.D_real_loss, 'b', label='D Real Loss')
    mpl.plot(output.iteration, output.D_X_loss+output.D_Y_loss, 'r', label='D Fake Loss')
    mpl.plot(output.iteration, output.D_X_loss, 'c', label='D X Loss')
    mpl.plot(output.iteration, output.D_Y_loss, 'm', label='D Y Loss')
    mpl.plot(output.iteration, output.G_loss, 'g', label='G Loss')
    mpl.legend()
    mpl.show()
    
    return output

opts = Object()

# Model hyper-parameters
opts.image_size = 64
opts.disc = 'dc'
opts.gen = 'cycle'
opts.g_conv_dim = 32
opts.d_conv_dim = 32
opts.norm = 'instance'
opts.init_zero_weights = False
opts.init_type = 'naive'

# Training hyper-parameters
opts.train_iters = 10000
opts.batch_size = 16
opts.num_workers = 0
opts.lr = 0.0003
opts.beta1 = 0.5
opts.beta2 = 0.999
opts.lambda_cycle = 10

# Data sources
opts.X = 'cat/grumpifyAprocessed'
opts.Y = 'cat/grumpifyBprocessed'
opts.ext = '*.png'
opts.data_aug = 'deluxe'

# Saving directories and checkpoint/sample iterations
opts.log_step = 100
opts.sample_every = 2000
opts.checkpoint_every = 1000

opts.gpu = '0'

Training without cycle consistency

opts.use_cycle_consistency_loss = False

opts.sample_dir = 'cyclegan'+str(opts.use_cycle_consistency_loss)
opts.checkpoint_dir = 'checkpoints_cyclegan'+str(opts.use_cycle_consistency_loss)

logger = SummaryWriter(opts.sample_dir)

# Create  dataloaders for images from the two domains X and Y
dataloader_X= get_data_loader(opts.X, opts=opts)
dataloader_Y = get_data_loader(opts.Y, opts=opts)

# Create checkpoint and sample directories
utils.create_dir(opts.checkpoint_dir)
utils.create_dir(opts.sample_dir)

# Start training
trainingOut = training_loop_cg(dataloader_X, dataloader_Y, opts)

deluxe activated
data/cat/grumpifyAprocessed/*.png
75
<__main__.CustomDataSet object at 0x7f1b54104190>
deluxe activated
data/cat/grumpifyBprocessed/*.png
204
<__main__.CustomDataSet object at 0x7f1b54307850>
                 G_XtoY                
---------------------------------------
CycleGenerator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 16, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (resnet_block1): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block2): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block3): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (deconv1): Sequential(
    (0): ConvTranspose2d(16, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv2): Sequential(
    (0): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
  )
)
---------------------------------------
                 G_YtoX                
---------------------------------------
CycleGenerator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 16, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (resnet_block1): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block2): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block3): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (deconv1): Sequential(
    (0): ConvTranspose2d(16, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv2): Sequential(
    (0): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
  )
)
---------------------------------------
                  D_X                  
---------------------------------------
DCDiscriminator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv3): Sequential(
    (0): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv4): Sequential(
    (0): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv5): Sequential(
    (0): Conv2d(256, 1, kernel_size=(4, 4), stride=(2, 2), bias=False)
  )
)
---------------------------------------
                  D_Y                  
---------------------------------------
DCDiscriminator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv3): Sequential(
    (0): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv4): Sequential(
    (0): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv5): Sequential(
    (0): Conv2d(256, 1, kernel_size=(4, 4), stride=(2, 2), bias=False)
  )
)
---------------------------------------
Models moved to GPU.

Lossy conversion from float64 to uint8. Range [-1.0585345029830933, 1.0955185890197754]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0684411525726318, 1.0659780502319336]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0585345029830933, 1.0955185890197754]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0684411525726318, 1.0659780502319336]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0585345029830933, 1.0955185890197754]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0684411525726318, 1.0659780502319336]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0585345029830933, 1.0955185890197754]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0684411525726318, 1.0659780502319336]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0585345029830933, 1.0955185890197754]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.0684411525726318, 1.0659780502319336]. Convert image to uint8 prior to saving to suppress this warning.

ind = (len(trainingOut.vizPath)//6)
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [2000.]

result YX at iteration [2000.]

ind = (len(trainingOut.vizPath)*2//6)
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [4000.]

result YX at iteration [4000.]

ind = (len(trainingOut.vizPath)*3//6)
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [6000.]

result YX at iteration [6000.]

ind = (len(trainingOut.vizPath) * 4//6)
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [8000.]

result YX at iteration [8000.]

ind = (len(trainingOut.vizPath) * 5//6)
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [10000.]

result YX at iteration [10000.]

Training with cycle consistency

opts.use_cycle_consistency_loss = True

opts.sample_dir = 'cyclegan'+str(opts.use_cycle_consistency_loss)
opts.checkpoint_dir = 'checkpoints_cyclegan'+str(opts.use_cycle_consistency_loss)

logger = SummaryWriter(opts.sample_dir)

# Create  dataloaders for images from the two domains X and Y
dataloader_X= get_data_loader(opts.X, opts=opts)
dataloader_Y = get_data_loader(opts.Y, opts=opts)

# Create checkpoint and sample directories
utils.create_dir(opts.checkpoint_dir)
utils.create_dir(opts.sample_dir)

# Start training
trainingOut = training_loop_cg(dataloader_X, dataloader_Y, opts)

deluxe activated
data/cat/grumpifyAprocessed/*.png
75
<__main__.CustomDataSet object at 0x7f1b54411d90>
deluxe activated
data/cat/grumpifyBprocessed/*.png
204
<__main__.CustomDataSet object at 0x7f1b54765150>
                 G_XtoY                
---------------------------------------
CycleGenerator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 16, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (resnet_block1): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block2): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block3): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (deconv1): Sequential(
    (0): ConvTranspose2d(16, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv2): Sequential(
    (0): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
  )
)
---------------------------------------
                 G_YtoX                
---------------------------------------
CycleGenerator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 16, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (resnet_block1): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block2): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (resnet_block3): ResnetBlock(
    (conv_layer): Sequential(
      (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (1): InstanceNorm2d(16, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
    )
  )
  (deconv1): Sequential(
    (0): ConvTranspose2d(16, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (deconv2): Sequential(
    (0): ConvTranspose2d(32, 3, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
  )
)
---------------------------------------
                  D_X                  
---------------------------------------
DCDiscriminator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv3): Sequential(
    (0): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv4): Sequential(
    (0): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv5): Sequential(
    (0): Conv2d(256, 1, kernel_size=(4, 4), stride=(2, 2), bias=False)
  )
)
---------------------------------------
                  D_Y                  
---------------------------------------
DCDiscriminator(
  (conv1): Sequential(
    (0): Conv2d(3, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(32, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv2): Sequential(
    (0): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(64, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv3): Sequential(
    (0): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(128, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv4): Sequential(
    (0): Conv2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (1): InstanceNorm2d(256, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)
  )
  (conv5): Sequential(
    (0): Conv2d(256, 1, kernel_size=(4, 4), stride=(2, 2), bias=False)
  )
)
---------------------------------------
Models moved to GPU.

Lossy conversion from float64 to uint8. Range [-1.1062146425247192, 1.103689193725586]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1305162906646729, 1.123417854309082]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1062146425247192, 1.103689193725586]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1305162906646729, 1.123417854309082]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1062146425247192, 1.103689193725586]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1305162906646729, 1.123417854309082]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1062146425247192, 1.103689193725586]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1305162906646729, 1.123417854309082]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1062146425247192, 1.103689193725586]. Convert image to uint8 prior to saving to suppress this warning.
Lossy conversion from float64 to uint8. Range [-1.1305162906646729, 1.123417854309082]. Convert image to uint8 prior to saving to suppress this warning.

ind = (len(trainingOut.vizPath)//6) * 1
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [2000.]

result YX at iteration [2000.]

ind = (len(trainingOut.vizPath)* 2//6) 
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [4000.]

result YX at iteration [4000.]

ind = (len(trainingOut.vizPath)* 3//6) 
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [6000.]

result YX at iteration [6000.]

ind = (len(trainingOut.vizPath)* 4//6) 
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [8000.]

result YX at iteration [8000.]

ind = (len(trainingOut.vizPath* 5)//6) 
print("result XY at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][0])
skio.imshow(im)
skio.show()

print("result YX at iteration " + str(trainingOut.iteration_viz[ind]))
im = skio.imread(trainingOut.vizPath[ind][1])
skio.imshow(im)
skio.show()

result XY at iteration [10000.]

result YX at iteration [10000.]

1. Do you see a difference?

The predicted russian blue looks a lot better with cycle consistency. The predicted grumpy cat seems to be more interesting/less constant with cycle consistency.

1. Why? For the first observation, the russian blue looks better because cycle consistency allows us to get more information from grumpy cat images in the training of the grumpy to russian mapping. This helps since we have so many fewer examples total. For the second observation, the cycle consistency requires us to be able to represent the greater pose variety in the russian blue images with the created grumpy cat images. This results in generated grumpys with matching pose variety.