Convolutional Neural Network CNN with TensorFlow tutorial

Welcome to part thirteen of the Deep Learning with Neural Networks and TensorFlow tutorials. In this tutorial, we're going to cover how to write a basic convolutional neural network within TensorFlow with Python.

To begin, just like before, we're going to grab the code we used in our basic multilayer perceptron model in TensorFlow tutorial.

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot = True)

n_nodes_hl1 = 500
n_nodes_hl2 = 500
n_nodes_hl3 = 500

n_classes = 10
batch_size = 100

x = tf.placeholder('float', [None, 784])
y = tf.placeholder('float')

def neural_network_model(data):
    hidden_1_layer = {'weights':tf.Variable(tf.random_normal([784, n_nodes_hl1])),
                      'biases':tf.Variable(tf.random_normal([n_nodes_hl1]))}

    hidden_2_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),
                      'biases':tf.Variable(tf.random_normal([n_nodes_hl2]))}

    hidden_3_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),
                      'biases':tf.Variable(tf.random_normal([n_nodes_hl3]))}

    output_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])),
                    'biases':tf.Variable(tf.random_normal([n_classes])),}


    l1 = tf.add(tf.matmul(data,hidden_1_layer['weights']), hidden_1_layer['biases'])
    l1 = tf.nn.relu(l1)

    l2 = tf.add(tf.matmul(l1,hidden_2_layer['weights']), hidden_2_layer['biases'])
    l2 = tf.nn.relu(l2)

    l3 = tf.add(tf.matmul(l2,hidden_3_layer['weights']), hidden_3_layer['biases'])
    l3 = tf.nn.relu(l3)

    output = tf.matmul(l3,output_layer['weights']) + output_layer['biases']

    return output

def train_neural_network(x):
    prediction = neural_network_model(x)
    cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(prediction,y) )
    optimizer = tf.train.AdamOptimizer().minimize(cost)
    
    hm_epochs = 10
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())

        for epoch in range(hm_epochs):
            epoch_loss = 0
            for _ in range(int(mnist.train.num_examples/batch_size)):
                epoch_x, epoch_y = mnist.train.next_batch(batch_size)
                _, c = sess.run([optimizer, cost], feed_dict={x: epoch_x, y: epoch_y})
                epoch_loss += c

            print('Epoch', epoch, 'completed out of',hm_epochs,'loss:',epoch_loss)

        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))

        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
        print('Accuracy:',accuracy.eval({x:mnist.test.images, y:mnist.test.labels}))

train_neural_network(x)

Much of our code structure is different, but I've tried to keep the variable/parameter names that matter the same as the ones in the TensorFlow CNN Tutorial. If you're not yet comfortable with building your own neural network models, try going through the official CNN TensorFlow tutorial, comparing the fundamentals of both codesets to see what you can vary, and what you can't, when creating models.

To begin, we're mostly simply removing some starting variables, and then we're going to modify batch size to be 128:

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)

batch_size = 128
n_classes = 10 # MNIST total classes (0-9 digits)

# tf Graph input
x = tf.placeholder(tf.float32, [None, 784])
y = tf.placeholder(tf.float32, [None, n_classes])

Next, we're going to define a couple super simple functions that will help us with our convolutions and pooling:

def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def maxpool2d(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

The functions here are the exact same as the ones from the offical TensorFlow CNN tutorial. I don't see much point in having 1 liner functions defined, but it is conceivable that you have more complex operations happening in these steps. The strides parameter dictates the movement of the window. In this case, we just move 1 pixel at a time for the conv2d function, and 2 at a time for the maxpool2d function. The ksize parameter is the size of the pooling window. In our case, we're choosing a 2x2 pooling window for pooling.

Now, we will begin building the Convolutional Neural Network model:

def convolutional_neural_network(x):#, keep_rate):
    weights = {
        # 5 x 5 convolution, 1 input image, 32 outputs
        'W_conv1': tf.Variable(tf.random_normal([5, 5, 1, 32])),
        # 5x5 conv, 32 inputs, 64 outputs 
        'W_conv2': tf.Variable(tf.random_normal([5, 5, 32, 64])),
        # fully connected, 7*7*64 inputs, 1024 outputs
        'W_fc': tf.Variable(tf.random_normal([7*7*64, 1024])),
        # 1024 inputs, 10 outputs (class prediction)
        'out': tf.Variable(tf.random_normal([1024, n_classes]))
    }

    biases = {
        'b_conv1': tf.Variable(tf.random_normal([32])),
        'b_conv2': tf.Variable(tf.random_normal([64])),
        'b_fc': tf.Variable(tf.random_normal([1024])),
        'out': tf.Variable(tf.random_normal([n_classes]))
    }

For the sizes, I have commented what's happening. We're taking 5x5 convolutions on the initial image, and producing 32 outputs. Next, we take 5x5 convolutions of the 32 inputs and make 64 outputs. From here, we're left with 7x7 sized images, and 64 of them, and then we're outputting to 1024 nodes in the fully connected layer. Then, the output layer is 1024 layers, to 10, which are the final 10 possible outputs for the actual label itself (0-9).

Now we begin to apply this to our input:

    # Reshape input to a 4D tensor 
    x = tf.reshape(x, shape=[-1, 28, 28, 1])
    # Convolution Layer, using our function
    conv1 = tf.nn.relu(conv2d(x, weights['W_conv1']) + biases['b_conv1'])
    # Max Pooling (down-sampling)
    conv1 = maxpool2d(conv1)
    # Convolution Layer
    conv2 = tf.nn.relu(conv2d(conv1, weights['W_conv2']) + biases['b_conv2'])
    # Max Pooling (down-sampling)
    conv2 = maxpool2d(conv2)

Now we create the fully-connected layer:

    # Fully connected layer
    # Reshape conv2 output to fit fully connected layer
    fc = tf.reshape(conv2, [-1, 7*7*64])
    fc = tf.nn.relu(tf.matmul(fc, weights['W_fc']) + biases['b_fc'])

Finally:

    output = tf.matmul(fc, weights['out']) + biases['out']
    return output

We're done with the model. Now all we need to do is modify the prediction line in the train_neural_network function:

    prediction = convolutional_neural_network(x)

We can run this:

Epoch 0 completed out of 10 loss: 1986337.79602
Epoch 1 completed out of 10 loss: 411925.395897
Epoch 2 completed out of 10 loss: 240980.746948
Epoch 3 completed out of 10 loss: 165433.599199
Epoch 4 completed out of 10 loss: 116812.418831
Epoch 5 completed out of 10 loss: 80588.521759
Epoch 6 completed out of 10 loss: 65083.381073
Epoch 7 completed out of 10 loss: 33403.1349106
Epoch 8 completed out of 10 loss: 36376.6163025
Epoch 9 completed out of 10 loss: 31059.2574168
Accuracy: 0.9725

Not bad, but our Recurrent Neural Network did better. The main reason here is that we're not likely working with enough data. I have found RNNs to perform better than CNNs with less data.

Another concept that is fairly common with neural networks, especially convolutional neural networks, is "dropout." The idea of it is to mimic dead neurons in your own brain. The actual impact of it is that it appears to actually decrease the chance of over-weighted, or otherwise biasing, neurons in the artificial neural network.

We can implement dropout on the fully connected layer by doing the following. First, let's add a couple new variables at the top:

keep_rate = 0.8
keep_prob = tf.placeholder(tf.float32)

We're going to keep 80% of our neurons per training iteration. To implement this, it's as simple as adding a line in the convolutional_neural_network function:

    fc = tf.reshape(conv2,[-1, 7*7*64])
    fc = tf.nn.relu(tf.matmul(fc, weights['W_fc'])+biases['b_fc'])
    fc = tf.nn.dropout(fc, keep_rate)

Again, results are not really any better. If we use a much larger dataset, we'd find that we had better results.

Epoch 0 completed out of 10 loss: 1597806.96085
Epoch 1 completed out of 10 loss: 347677.287781
Epoch 2 completed out of 10 loss: 203316.559639
Epoch 3 completed out of 10 loss: 135096.661589
Epoch 4 completed out of 10 loss: 91069.0741692
Epoch 5 completed out of 10 loss: 62255.3888874
Epoch 6 completed out of 10 loss: 44628.810773
Epoch 7 completed out of 10 loss: 34804.6252755
Epoch 8 completed out of 10 loss: 28218.4389267
Epoch 9 completed out of 10 loss: 24621.0535733
Accuracy: 0.9764

Full code, including dropout:

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot = True)

n_classes = 10
batch_size = 128

x = tf.placeholder('float', [None, 784])
y = tf.placeholder('float')

keep_rate = 0.8
keep_prob = tf.placeholder(tf.float32)

def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')

def maxpool2d(x):
    #                        size of window         movement of window
    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')



def convolutional_neural_network(x):
    weights = {'W_conv1':tf.Variable(tf.random_normal([5,5,1,32])),
               'W_conv2':tf.Variable(tf.random_normal([5,5,32,64])),
               'W_fc':tf.Variable(tf.random_normal([7*7*64,1024])),
               'out':tf.Variable(tf.random_normal([1024, n_classes]))}

    biases = {'b_conv1':tf.Variable(tf.random_normal([32])),
               'b_conv2':tf.Variable(tf.random_normal([64])),
               'b_fc':tf.Variable(tf.random_normal([1024])),
               'out':tf.Variable(tf.random_normal([n_classes]))}

    x = tf.reshape(x, shape=[-1, 28, 28, 1])

    conv1 = tf.nn.relu(conv2d(x, weights['W_conv1']) + biases['b_conv1'])
    conv1 = maxpool2d(conv1)
    
    conv2 = tf.nn.relu(conv2d(conv1, weights['W_conv2']) + biases['b_conv2'])
    conv2 = maxpool2d(conv2)

    fc = tf.reshape(conv2,[-1, 7*7*64])
    fc = tf.nn.relu(tf.matmul(fc, weights['W_fc'])+biases['b_fc'])
    fc = tf.nn.dropout(fc, keep_rate)

    output = tf.matmul(fc, weights['out'])+biases['out']

    return output

def train_neural_network(x):
    prediction = convolutional_neural_network(x)
    cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(prediction,y) )
    optimizer = tf.train.AdamOptimizer().minimize(cost)
    
    hm_epochs = 10
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())

        for epoch in range(hm_epochs):
            epoch_loss = 0
            for _ in range(int(mnist.train.num_examples/batch_size)):
                epoch_x, epoch_y = mnist.train.next_batch(batch_size)
                _, c = sess.run([optimizer, cost], feed_dict={x: epoch_x, y: epoch_y})
                epoch_loss += c

            print('Epoch', epoch, 'completed out of',hm_epochs,'loss:',epoch_loss)

        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))

        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
        print('Accuracy:',accuracy.eval({x:mnist.test.images, y:mnist.test.labels}))

train_neural_network(x)

The next tutorial: TFLearn - High Level Abstraction Layer for TensorFlow Tutorial