Implementing same convolution in Pytorch

Question

I was trying to implement same convolution. It seems (surprisingly) that such a standard layer isn't available out of the box in Pytorch. I hope this helps make sure a layer available. I was needing help generalizing it to use ALL hyper parameters. Right now it only uses kernel size (i.e. variable stride, dialation or whatever else that might affect things isn't part of this. Anyone know how to generalize it with all hyper params of a Convolution (in Conv Nets)?

Code:

def get_cnn(C, out_filters=[14,13,12,3], kernels=[(9,7),(7,7),(5,3),(3,3)]):
    N = len(out_filters)
    print()
    print(f'C = {C}')
    model_layers = OrderedDict()
    in_channels = C
    x = torch.randn(1,C,10,10)
    out = x
    print(f'initial_size = {x.size()}')
    for i in range(1,N):
        print(f'--- building layer = {i}')
        ## create conv layer
        kernel_size = kernels[i]
        out_channels = out_filters[i]
        # to make sure its a same convolution (ignoring stride)
        print(f'in_channels = {in_channels}')
        print(f'out_channels = {out_channels}')
        padding = ((kernels[i][0]-1)//2,(kernels[i][1]-1)//2)
        conv = torch.nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding)
        #conv = torch.nn.Conv2d(in_channels, out_channels, kernel_size)
        model_layers[f'Conv{i}'] = conv
        ## compute dummy data
        out = conv(out)
        print(f'x.size() = {x.size()}')
        print(f'out.size() = {out.size()}')
        ## add activation
        model_layers[f'ReLU{i}'] = torch.nn.ReLU()
        ## so that next layer works
        in_channels = out_channels
    ## make sequential model
    mdl = torch.nn.Sequential(model_layers)
    y = mdl(x)
    return mdl

---

References:

• Does padding ensure that the sizes of input and output of a CNN match? : https://stats.stackexchange.com/questions/419902/does-padding-ensure-that-the-sizes-of-input-and-output-of-a-cnn-match

• arithmetic of convolutions: https://arxiv.org/abs/1603.07285

• https://discuss.pytorch.org/t/same-convolution-in-pytorch/19937

• https://discuss.pytorch.org/t/how-do-i-make-sure-the-output-of-a-cnn-never-decrease-in-size-using-padding/52103

---

Full running code:

import torch

from collections import OrderedDict

import random

def do_conv():
    x = torch.randn(1,3,5,6)
    conv = torch.nn.Conv2d(in_channels=3, out_channels=5, kernel_size=(3,3))
    y = conv(x)
    print(y)

def get_cnn(C, out_filters=[14,13,12,3], kernels=[(9,7),(7,7),(5,3),(3,3)]):
    N = len(out_filters)
    print()
    print(f'C = {C}')
    model_layers = OrderedDict()
    in_channels = C
    x = torch.randn(1,C,10,10)
    out = x
    print(f'initial_size = {x.size()}')
    for i in range(1,N):
        print(f'--- building layer = {i}')
        ## create conv layer
        kernel_size = kernels[i]
        out_channels = out_filters[i]
        # to make sure its a same convolution (ignoring stride)
        print(f'in_channels = {in_channels}')
        print(f'out_channels = {out_channels}')
        padding = ((kernels[i][0]-1)//2,(kernels[i][1]-1)//2)
        conv = torch.nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding)
        #conv = torch.nn.Conv2d(in_channels, out_channels, kernel_size)
        model_layers[f'Conv{i}'] = conv
        ## compute dummy data
        out = conv(out)
        print(f'x.size() = {x.size()}')
        print(f'out.size() = {out.size()}')
        ## add activation
        model_layers[f'ReLU{i}'] = torch.nn.ReLU()
        ## so that next layer works
        in_channels = out_channels
    ## make sequential model
    mdl = torch.nn.Sequential(model_layers)
    y = mdl(x)
    return mdl

def get_hardcoded_variable_size_input():
    C = 3
    variable_size_input = [torch.randn(1,C,15,15), torch.randn(1,C,12,16), torch.randn(1,C,15,11)]
    return variable_size_input

def get_variable_size_input(N=3):
    variable_size_input = []
    for i in range(N):
        C,H,W = 3, random.randint(11,20), random.randint(14,19)
        x = torch.randn(C,H,W)
        variable_size_input.append(x)
    return variable_size_input

def any_input_output_equals_input():
    '''
    test that no matter what input, the output of the network is same as the input
    '''
    X_list = get_hardcoded_variable_size_input()
    for i in range(len(X_list)):
        print(f'data point i = {i}')
        ## get data
        x = X_list[i]
        print(x.size())
        ## create cnn
        _,C,_,_ = x.size()
        cnn = get_cnn(C)
        # pass data
        y = cnn(x)
        ## make sure input and output have same size
        assert x.size() == y.size(), f'Error: {x.size()} != {y.size()}'
    ##

if __name__ == '__main__':
    #do_conv()
    print()
    print('start main')
    any_input_output_equals_input()
    print('Done \a')

Answer 1

Maybe something like this could work?

import torch
import torch.nn
class PaddedConv2d(nn.Module):
    """
    A simple class to perform convolutions with padding so that input and
    output size is the same 
    """

    def __init__(self, conv2d, pad2d_type):
        """
        Parameters
        ---

        * conv2d : torch.nn.Conv2d
            a convolutional layer used in this PaddedConv2d

        * pad2d_type : type
            a padding layer from torch.nn. I don't want the
            instance itself since it needs the padding size to be created;
            instead, I want the callable which returns an instance of 
            a padding layer, which will be created on the fly during the 
            "forward" pass.
        """

        super().__init__(self)
        self.conv2d = conv2d
        self.pad2d_type = pad2d_type

    def forward(self, in):
        """
        Parameters
        ---
        * in : torch.Tensor
             the input tensor to be padded and then convolved. Shape (batch_size, channels, rows, cols)
        """
        # computing padding size:
        pad_h = torch.ceil((self.conv2d.kernel[0] - in.shape[2] * (1 - self.conv2d.stride[0]) - self.conv2d.stride[0]) / 2)
        pad_w = torch.ceil((self.conv2d.kernel[1] - in.shape[3] * (1 - self.conv2d.stride[1]) - self.conv2d.stride[1]) / 2)
        padder = self.pad2d_type((pad_w, pad_w, pad_h, pad_h))

        return self.conv2d(padder(in))