인공지능 구현에 대한 글입니다.

글의 순서는 아래와 같습니다.

================================================

1.# lab-11-1-mnist_cnn

    ==> http://cs.stanford.edu/people/karpathy/convnetjs/demo/cifar10.html : cnn 시물레이션

    ==> 2단 CNN => 정확도 98.83%

    ==> CNN은 수행시간이 많이 소요됨, 코드 테스트시에는 TRAINING EPOCHS를 줄여 실행할 필요가 있음.

         물론 구글, 아마존 크라우드 서비스 활용도 가능함( 경우에 따라 비용발생)

         그래픽 카드 (GPU NVIDIA )를 적용하는 방법도 있음.

2. # lab-11-2-mnist_deep_cnn 

   ==> 3단 CNN : 정확도 99.38 %

3. # lab-11-3-mnist_cnn_class

   ==> python의 clss를 이용해서 코드의 반복을 줄이고, 코드를 단순화함

4. #lab-11-4-mnist_cnn_layers

   ==> tf.layers 패키지 사용하면 코드가 더욱 단순화 될수 있음

   ==> https://www.tensorflow.org/api_docs/python/tf/layers

5. # lab-11-5-mnist_cnn_ensemble_layers

  ==> 각각 예측시키고, 후에 병합을 해서 예측치를 출력시킴

  ==> 정확도 : 99.52%

6. 코드탐구(추가)

  ==>lab-11-X-mnist_cnn_low_memory

7. 참고자료

=================================================

[  lab-11-1-mnist_cnn ]

# lab-11-1-mnist_cnn

"""http://cs.stanford.edu/people/karpathy/convnetjs/demo/cifar10.html : cnn 시물레이션

   2단 CNN => 정확도 98.83%

"""

import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# Lab 11 MNIST and Convolutional Neural Network

import tensorflow as tf

import random

# import matplotlib.pyplot as plt

from tensorflow.examples.tutorials.mnist import input_data

tf.set_random_seed(777)  # reproducibility

mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

# Check out https://www.tensorflow.org/get_started/mnist/beginners for

# more information about the mnist dataset

# hyper parameters

learning_rate = 0.001

training_epochs = 15

batch_size = 100

# input place holders

X = tf.placeholder(tf.float32, [None, 784])

X_img = tf.reshape(X, [-1, 28, 28, 1])   # img 28x28x1 (black/white)

Y = tf.placeholder(tf.float32, [None, 10])

# L1 ImgIn shape=(?, 28, 28, 1)

W1 = tf.Variable(tf.random_normal([3, 3, 1, 32], stddev=0.01))

#    Conv     -> (?, 28, 28, 32)

#    Pool     -> (?, 14, 14, 32)

L1 = tf.nn.conv2d(X_img, W1, strides=[1, 1, 1, 1], padding='SAME')

L1 = tf.nn.relu(L1)

L1 = tf.nn.max_pool(L1, ksize=[1, 2, 2, 1],

                    strides=[1, 2, 2, 1], padding='SAME')

'''

Tensor("Conv2D:0", shape=(?, 28, 28, 32), dtype=float32)

Tensor("Relu:0", shape=(?, 28, 28, 32), dtype=float32)

Tensor("MaxPool:0", shape=(?, 14, 14, 32), dtype=float32)

'''

# L2 ImgIn shape=(?, 14, 14, 32)

W2 = tf.Variable(tf.random_normal([3, 3, 32, 64], stddev=0.01))

#    Conv      ->(?, 14, 14, 64)

#    Pool      ->(?, 7, 7, 64)

L2 = tf.nn.conv2d(L1, W2, strides=[1, 1, 1, 1], padding='SAME')

L2 = tf.nn.relu(L2)

L2 = tf.nn.max_pool(L2, ksize=[1, 2, 2, 1],

                    strides=[1, 2, 2, 1], padding='SAME')

L2_flat = tf.reshape(L2, [-1, 7 * 7 * 64])

'''

Tensor("Conv2D_1:0", shape=(?, 14, 14, 64), dtype=float32)

Tensor("Relu_1:0", shape=(?, 14, 14, 64), dtype=float32)

Tensor("MaxPool_1:0", shape=(?, 7, 7, 64), dtype=float32)

Tensor("Reshape_1:0", shape=(?, 3136), dtype=float32)

'''

# Final FC 7x7x64 inputs -> 10 outputs

W3 = tf.get_variable("W3", shape=[7 * 7 * 64, 10],

                     initializer=tf.contrib.layers.xavier_initializer())

b = tf.Variable(tf.random_normal([10]))

logits = tf.matmul(L2_flat, W3) + b

# define cost/loss & optimizer

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(

    logits=logits, labels=Y))

optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# initialize

sess = tf.Session()

sess.run(tf.global_variables_initializer())

# train my model

print('Learning started. It takes sometime.')

for epoch in range(training_epochs):

    avg_cost = 0

    total_batch = int(mnist.train.num_examples / batch_size)

    for i in range(total_batch):

        batch_xs, batch_ys = mnist.train.next_batch(batch_size)

        feed_dict = {X: batch_xs, Y: batch_ys}

        c, _ = sess.run([cost, optimizer], feed_dict=feed_dict)

        avg_cost += c / total_batch

    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))

print('Learning Finished!')

# Test model and check accuracy

correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))

accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

print('Accuracy:', sess.run(accuracy, feed_dict={

      X: mnist.test.images, Y: mnist.test.labels}))

# Get one and predict

r = random.randint(0, mnist.test.num_examples - 1)

print("Label: ", sess.run(tf.argmax(mnist.test.labels[r:r + 1], 1)))

print("Prediction: ", sess.run(

    tf.argmax(logits, 1), feed_dict={X: mnist.test.images[r:r + 1]}))

# plt.imshow(mnist.test.images[r:r + 1].

#           reshape(28, 28), cmap='Greys', interpolation='nearest')

# plt.show()

'''

Epoch: 0001 cost = 0.340291267

Epoch: 0002 cost = 0.090731326

Epoch: 0003 cost = 0.064477619

Epoch: 0004 cost = 0.050683064

Epoch: 0005 cost = 0.041864835

Epoch: 0006 cost = 0.035760704

Epoch: 0007 cost = 0.030572132

Epoch: 0008 cost = 0.026207981

Epoch: 0009 cost = 0.022622454

Epoch: 0010 cost = 0.019055919

Epoch: 0011 cost = 0.017758641

Epoch: 0012 cost = 0.014156652

Epoch: 0013 cost = 0.012397016

Epoch: 0014 cost = 0.010693789

Epoch: 0015 cost = 0.009469977

Learning Finished!

Accuracy: 0.9885

'''

[ # lab-11-2-mnist_deep_cnn ]

# lab-11-2-mnist_deep_cnn

"""

3단 CNN : 정확도 99.38 %

"""

import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# Lab 11 MNIST and Deep learning CNN

import tensorflow as tf

import random

# import matplotlib.pyplot as plt

from tensorflow.examples.tutorials.mnist import input_data

tf.set_random_seed(777)  # reproducibility

mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

# Check out https://www.tensorflow.org/get_started/mnist/beginners for

# more information about the mnist dataset

# hyper parameters

learning_rate = 0.001

training_epochs = 15

batch_size = 100

# dropout (keep_prob) rate  0.7~0.5 on training, but should be 1 for testing

keep_prob = tf.placeholder(tf.float32)

# input place holders

X = tf.placeholder(tf.float32, [None, 784])

X_img = tf.reshape(X, [-1, 28, 28, 1])   # img 28x28x1 (black/white)

Y = tf.placeholder(tf.float32, [None, 10])

# L1 ImgIn shape=(?, 28, 28, 1)

W1 = tf.Variable(tf.random_normal([3, 3, 1, 32], stddev=0.01))

#    Conv     -> (?, 28, 28, 32)

#    Pool     -> (?, 14, 14, 32)

L1 = tf.nn.conv2d(X_img, W1, strides=[1, 1, 1, 1], padding='SAME')

L1 = tf.nn.relu(L1)

L1 = tf.nn.max_pool(L1, ksize=[1, 2, 2, 1],

                    strides=[1, 2, 2, 1], padding='SAME')

L1 = tf.nn.dropout(L1, keep_prob=keep_prob)

'''

Tensor("Conv2D:0", shape=(?, 28, 28, 32), dtype=float32)

Tensor("Relu:0", shape=(?, 28, 28, 32), dtype=float32)

Tensor("MaxPool:0", shape=(?, 14, 14, 32), dtype=float32)

Tensor("dropout/mul:0", shape=(?, 14, 14, 32), dtype=float32)

'''

# L2 ImgIn shape=(?, 14, 14, 32)

W2 = tf.Variable(tf.random_normal([3, 3, 32, 64], stddev=0.01))

#    Conv      ->(?, 14, 14, 64)

#    Pool      ->(?, 7, 7, 64)

L2 = tf.nn.conv2d(L1, W2, strides=[1, 1, 1, 1], padding='SAME')

L2 = tf.nn.relu(L2)

L2 = tf.nn.max_pool(L2, ksize=[1, 2, 2, 1],

                    strides=[1, 2, 2, 1], padding='SAME')

L2 = tf.nn.dropout(L2, keep_prob=keep_prob)

'''

Tensor("Conv2D_1:0", shape=(?, 14, 14, 64), dtype=float32)

Tensor("Relu_1:0", shape=(?, 14, 14, 64), dtype=float32)

Tensor("MaxPool_1:0", shape=(?, 7, 7, 64), dtype=float32)

Tensor("dropout_1/mul:0", shape=(?, 7, 7, 64), dtype=float32)

'''

# L3 ImgIn shape=(?, 7, 7, 64)

W3 = tf.Variable(tf.random_normal([3, 3, 64, 128], stddev=0.01))

#    Conv      ->(?, 7, 7, 128)

#    Pool      ->(?, 4, 4, 128)

#    Reshape   ->(?, 4 * 4 * 128) # Flatten them for FC

L3 = tf.nn.conv2d(L2, W3, strides=[1, 1, 1, 1], padding='SAME')

L3 = tf.nn.relu(L3)

L3 = tf.nn.max_pool(L3, ksize=[1, 2, 2, 1], strides=[

                    1, 2, 2, 1], padding='SAME')

L3 = tf.nn.dropout(L3, keep_prob=keep_prob)

L3_flat = tf.reshape(L3, [-1, 128 * 4 * 4])

'''

Tensor("Conv2D_2:0", shape=(?, 7, 7, 128), dtype=float32)

Tensor("Relu_2:0", shape=(?, 7, 7, 128), dtype=float32)

Tensor("MaxPool_2:0", shape=(?, 4, 4, 128), dtype=float32)

Tensor("dropout_2/mul:0", shape=(?, 4, 4, 128), dtype=float32)

Tensor("Reshape_1:0", shape=(?, 2048), dtype=float32)

'''

# L4 FC 4x4x128 inputs -> 625 outputs

W4 = tf.get_variable("W4", shape=[128 * 4 * 4, 625],

                     initializer=tf.contrib.layers.xavier_initializer())

b4 = tf.Variable(tf.random_normal([625]))

L4 = tf.nn.relu(tf.matmul(L3_flat, W4) + b4)

L4 = tf.nn.dropout(L4, keep_prob=keep_prob)

'''

Tensor("Relu_3:0", shape=(?, 625), dtype=float32)

Tensor("dropout_3/mul:0", shape=(?, 625), dtype=float32)

'''

# L5 Final FC 625 inputs -> 10 outputs

W5 = tf.get_variable("W5", shape=[625, 10],

                     initializer=tf.contrib.layers.xavier_initializer())

b5 = tf.Variable(tf.random_normal([10]))

logits = tf.matmul(L4, W5) + b5

'''

Tensor("add_1:0", shape=(?, 10), dtype=float32)

'''

# define cost/loss & optimizer

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(

    logits=logits, labels=Y))

optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# initialize

sess = tf.Session()

sess.run(tf.global_variables_initializer())

# train my model

print('Learning started. It takes sometime.')

for epoch in range(training_epochs):

    avg_cost = 0

    total_batch = int(mnist.train.num_examples / batch_size)

    for i in range(total_batch):

        batch_xs, batch_ys = mnist.train.next_batch(batch_size)

        feed_dict = {X: batch_xs, Y: batch_ys, keep_prob: 0.7}

        c, _ = sess.run([cost, optimizer], feed_dict=feed_dict)

        avg_cost += c / total_batch

    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))

print('Learning Finished!')

# Test model and check accuracy

# if you have a OOM error, please refer to lab-11-X-mnist_deep_cnn_low_memory.py

correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))

accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

print('Accuracy:', sess.run(accuracy, feed_dict={

      X: mnist.test.images, Y: mnist.test.labels, keep_prob: 1}))

# Get one and predict

r = random.randint(0, mnist.test.num_examples - 1)

print("Label: ", sess.run(tf.argmax(mnist.test.labels[r:r + 1], 1)))

print("Prediction: ", sess.run(

    tf.argmax(logits, 1), feed_dict={X: mnist.test.images[r:r + 1], keep_prob: 1}))

# plt.imshow(mnist.test.images[r:r + 1].

#           reshape(28, 28), cmap='Greys', interpolation='nearest')

# plt.show()

'''

Learning stared. It takes sometime.

Epoch: 0001 cost = 0.385748474

Epoch: 0002 cost = 0.092017397

Epoch: 0003 cost = 0.065854684

Epoch: 0004 cost = 0.055604566

Epoch: 0005 cost = 0.045996377

Epoch: 0006 cost = 0.040913645

Epoch: 0007 cost = 0.036924479

Epoch: 0008 cost = 0.032808939

Epoch: 0009 cost = 0.031791007

Epoch: 0010 cost = 0.030224456

Epoch: 0011 cost = 0.026849916

Epoch: 0012 cost = 0.026826763

Epoch: 0013 cost = 0.027188021

Epoch: 0014 cost = 0.023604777

Epoch: 0015 cost = 0.024607201

Learning Finished!

Accuracy: 0.9938

[ # lab-11-3-mnist_cnn_class ]

# lab-11-3-mnist_cnn_class

""" python의 clss를 이용해서 코드의 반복을 줄이고, 코드를 단순화함"""

import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# Lab 11 MNIST and Deep learning CNN

import tensorflow as tf

# import matplotlib.pyplot as plt

from tensorflow.examples.tutorials.mnist import input_data

tf.set_random_seed(777)  # reproducibility

mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

# Check out https://www.tensorflow.org/get_started/mnist/beginners for

# more information about the mnist dataset

# hyper parameters

learning_rate = 0.001

training_epochs = 15

batch_size = 100

class Model:

    def __init__(self, sess, name):

        self.sess = sess

        self.name = name

        self._build_net()

    def _build_net(self):

        with tf.variable_scope(self.name):

            # dropout (keep_prob) rate  0.7~0.5 on training, but should be 1

            # for testing

            self.keep_prob = tf.placeholder(tf.float32)

            # input place holders

            self.X = tf.placeholder(tf.float32, [None, 784])

            # img 28x28x1 (black/white)

            X_img = tf.reshape(self.X, [-1, 28, 28, 1])

            self.Y = tf.placeholder(tf.float32, [None, 10])

            # L1 ImgIn shape=(?, 28, 28, 1)

            W1 = tf.Variable(tf.random_normal([3, 3, 1, 32], stddev=0.01))

            #    Conv     -> (?, 28, 28, 32)

            #    Pool     -> (?, 14, 14, 32)

            L1 = tf.nn.conv2d(X_img, W1, strides=[1, 1, 1, 1], padding='SAME')

            L1 = tf.nn.relu(L1)

            L1 = tf.nn.max_pool(L1, ksize=[1, 2, 2, 1],

                                strides=[1, 2, 2, 1], padding='SAME')

            L1 = tf.nn.dropout(L1, keep_prob=self.keep_prob)

            '''

            Tensor("Conv2D:0", shape=(?, 28, 28, 32), dtype=float32)

            Tensor("Relu:0", shape=(?, 28, 28, 32), dtype=float32)

            Tensor("MaxPool:0", shape=(?, 14, 14, 32), dtype=float32)

            Tensor("dropout/mul:0", shape=(?, 14, 14, 32), dtype=float32)

            '''

            # L2 ImgIn shape=(?, 14, 14, 32)

            W2 = tf.Variable(tf.random_normal([3, 3, 32, 64], stddev=0.01))

            #    Conv      ->(?, 14, 14, 64)

            #    Pool      ->(?, 7, 7, 64)

            L2 = tf.nn.conv2d(L1, W2, strides=[1, 1, 1, 1], padding='SAME')

            L2 = tf.nn.relu(L2)

            L2 = tf.nn.max_pool(L2, ksize=[1, 2, 2, 1],

                                strides=[1, 2, 2, 1], padding='SAME')

            L2 = tf.nn.dropout(L2, keep_prob=self.keep_prob)

            '''

            Tensor("Conv2D_1:0", shape=(?, 14, 14, 64), dtype=float32)

            Tensor("Relu_1:0", shape=(?, 14, 14, 64), dtype=float32)

            Tensor("MaxPool_1:0", shape=(?, 7, 7, 64), dtype=float32)

            Tensor("dropout_1/mul:0", shape=(?, 7, 7, 64), dtype=float32)

            '''

            # L3 ImgIn shape=(?, 7, 7, 64)

            W3 = tf.Variable(tf.random_normal([3, 3, 64, 128], stddev=0.01))

            #    Conv      ->(?, 7, 7, 128)

            #    Pool      ->(?, 4, 4, 128)

            #    Reshape   ->(?, 4 * 4 * 128) # Flatten them for FC

            L3 = tf.nn.conv2d(L2, W3, strides=[1, 1, 1, 1], padding='SAME')

            L3 = tf.nn.relu(L3)

            L3 = tf.nn.max_pool(L3, ksize=[1, 2, 2, 1], strides=[

                                1, 2, 2, 1], padding='SAME')

            L3 = tf.nn.dropout(L3, keep_prob=self.keep_prob)

            L3_flat = tf.reshape(L3, [-1, 128 * 4 * 4])

            '''

            Tensor("Conv2D_2:0", shape=(?, 7, 7, 128), dtype=float32)

            Tensor("Relu_2:0", shape=(?, 7, 7, 128), dtype=float32)

            Tensor("MaxPool_2:0", shape=(?, 4, 4, 128), dtype=float32)

            Tensor("dropout_2/mul:0", shape=(?, 4, 4, 128), dtype=float32)

            Tensor("Reshape_1:0", shape=(?, 2048), dtype=float32)

            '''

            # L4 FC 4x4x128 inputs -> 625 outputs

            W4 = tf.get_variable("W4", shape=[128 * 4 * 4, 625],

                                 initializer=tf.contrib.layers.xavier_initializer())

            b4 = tf.Variable(tf.random_normal([625]))

            L4 = tf.nn.relu(tf.matmul(L3_flat, W4) + b4)

            L4 = tf.nn.dropout(L4, keep_prob=self.keep_prob)

            '''

            Tensor("Relu_3:0", shape=(?, 625), dtype=float32)

            Tensor("dropout_3/mul:0", shape=(?, 625), dtype=float32)

            '''

            # L5 Final FC 625 inputs -> 10 outputs

            W5 = tf.get_variable("W5", shape=[625, 10],

                                 initializer=tf.contrib.layers.xavier_initializer())

            b5 = tf.Variable(tf.random_normal([10]))

            self.logits = tf.matmul(L4, W5) + b5

            '''

            Tensor("add_1:0", shape=(?, 10), dtype=float32)

            '''

        # define cost/loss & optimizer

        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(

            logits=self.logits, labels=self.Y))

        self.optimizer = tf.train.AdamOptimizer(

            learning_rate=learning_rate).minimize(self.cost)

        correct_prediction = tf.equal(

            tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))

        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    def predict(self, x_test, keep_prop=1.0):

        return self.sess.run(self.logits, feed_dict={self.X: x_test, self.keep_prob: keep_prop})

    def get_accuracy(self, x_test, y_test, keep_prop=1.0):

        return self.sess.run(self.accuracy, feed_dict={self.X: x_test, self.Y: y_test, self.keep_prob: keep_prop})

    def train(self, x_data, y_data, keep_prop=0.7):

        return self.sess.run([self.cost, self.optimizer], feed_dict={

            self.X: x_data, self.Y: y_data, self.keep_prob: keep_prop})

# initialize

sess = tf.Session()

m1 = Model(sess, "m1")

sess.run(tf.global_variables_initializer())

print('Learning Started!')

# train my model

for epoch in range(training_epochs):

    avg_cost = 0

    total_batch = int(mnist.train.num_examples / batch_size)

    for i in range(total_batch):

        batch_xs, batch_ys = mnist.train.next_batch(batch_size)

        c, _ = m1.train(batch_xs, batch_ys)

        avg_cost += c / total_batch

    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))

print('Learning Finished!')

# Test model and check accuracy

print('Accuracy:', m1.get_accuracy(mnist.test.images, mnist.test.labels))

[#lab-11-4-mnist_cnn_layers]

#lab-11-4-mnist_cnn_layers

"""

tf.layers 패키지 사용하면 코드가 더욱 단순화 될수 있음

 https://www.tensorflow.org/api_docs/python/tf/layers

CNN은 수행시간이 많이 소요됨, 코드 테스트시에는 TRAINING EPOCHS를 줄여 실행할 필요가 있음.

물론 구글, 아마존 크라우드 서비스 활용도 가능함( 경우에 따라 비용발생)

그래픽 카드 (GPU NVIDIA )를 적용하는 방법도 있음.

"""

import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# Lab 11 MNIST and Deep learning CNN

import tensorflow as tf

# import matplotlib.pyplot as plt

from tensorflow.examples.tutorials.mnist import input_data

tf.set_random_seed(777)  # reproducibility

mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

# Check out https://www.tensorflow.org/get_started/mnist/beginners for

# more information about the mnist dataset

# hyper parameters

learning_rate = 0.001

training_epochs = 15

batch_size = 100

class Model:

    def __init__(self, sess, name):

        self.sess = sess

        self.name = name

        self._build_net()

    def _build_net(self):

        with tf.variable_scope(self.name):

            # dropout (keep_prob) rate  0.7~0.5 on training, but should be 1

            # for testing

            self.training = tf.placeholder(tf.bool)

            # input place holders

            self.X = tf.placeholder(tf.float32, [None, 784])

            # img 28x28x1 (black/white), Input Layer

            X_img = tf.reshape(self.X, [-1, 28, 28, 1])

            self.Y = tf.placeholder(tf.float32, [None, 10])

            # Convolutional Layer #1

            conv1 = tf.layers.conv2d(inputs=X_img, filters=32, kernel_size=[3, 3],

                                     padding="SAME", activation=tf.nn.relu)

            # Pooling Layer #1

            pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2],

                                            padding="SAME", strides=2)

            dropout1 = tf.layers.dropout(inputs=pool1,

                                         rate=0.7, training=self.training)

            # Convolutional Layer #2 and Pooling Layer #2

            conv2 = tf.layers.conv2d(inputs=dropout1, filters=64, kernel_size=[3, 3],

                                     padding="SAME", activation=tf.nn.relu)

            pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2],

                                            padding="SAME", strides=2)

            dropout2 = tf.layers.dropout(inputs=pool2,

                                         rate=0.7, training=self.training)

            # Convolutional Layer #2 and Pooling Layer #2

            conv3 = tf.layers.conv2d(inputs=dropout2, filters=128, kernel_size=[3, 3],

                                     padding="same", activation=tf.nn.relu)

            pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=[2, 2],

                                            padding="same", strides=2)

            dropout3 = tf.layers.dropout(inputs=pool3,

                                         rate=0.7, training=self.training)

            # Dense Layer with Relu

            flat = tf.reshape(dropout3, [-1, 128 * 4 * 4])

            dense4 = tf.layers.dense(inputs=flat,

                                     units=625, activation=tf.nn.relu)

            dropout4 = tf.layers.dropout(inputs=dense4,

                                         rate=0.5, training=self.training)

            # Logits (no activation) Layer: L5 Final FC 625 inputs -> 10 outputs

            self.logits = tf.layers.dense(inputs=dropout4, units=10)

        # define cost/loss & optimizer

        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(

            logits=self.logits, labels=self.Y))

        self.optimizer = tf.train.AdamOptimizer(

            learning_rate=learning_rate).minimize(self.cost)

        correct_prediction = tf.equal(

            tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))

        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    def predict(self, x_test, training=False):

        return self.sess.run(self.logits,

                             feed_dict={self.X: x_test, self.training: training})

    def get_accuracy(self, x_test, y_test, training=False):

        return self.sess.run(self.accuracy,

                             feed_dict={self.X: x_test,

                                        self.Y: y_test, self.training: training})

    def train(self, x_data, y_data, training=True):

        return self.sess.run([self.cost, self.optimizer], feed_dict={

            self.X: x_data, self.Y: y_data, self.training: training})

# initialize

sess = tf.Session()

m1 = Model(sess, "m1")

sess.run(tf.global_variables_initializer())

print('Learning Started!')

# train my model

for epoch in range(training_epochs):

    avg_cost = 0

    total_batch = int(mnist.train.num_examples / batch_size)

    for i in range(total_batch):

        batch_xs, batch_ys = mnist.train.next_batch(batch_size)

        c, _ = m1.train(batch_xs, batch_ys)

        avg_cost += c / total_batch

    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))

print('Learning Finished!')

# Test model and check accuracy

print('Accuracy:', m1.get_accuracy(mnist.test.images, mnist.test.labels))

[ # lab-11-5-mnist_cnn_ensemble_layers ]

# lab-11-5-mnist_cnn_ensemble_layers

" 각각 예측시키고, 후에 병합을 해서 예측치를 출력시킴"

import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# Lab 11 MNIST and Deep learning CNN

# https://www.tensorflow.org/tutorials/layers

import tensorflow as tf

import numpy as np

from tensorflow.examples.tutorials.mnist import input_data

tf.set_random_seed(777)  # reproducibility

mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

# Check out https://www.tensorflow.org/get_started/mnist/beginners for

# more information about the mnist dataset

# hyper parameters

learning_rate = 0.001

training_epochs = 20

batch_size = 100

class Model:

    def __init__(self, sess, name):

        self.sess = sess

        self.name = name

        self._build_net()

    def _build_net(self):

        with tf.variable_scope(self.name):

            # dropout (keep_prob) rate  0.7~0.5 on training, but should be 1

            # for testing

            self.training = tf.placeholder(tf.bool)

            # input place holders

            self.X = tf.placeholder(tf.float32, [None, 784])

            # img 28x28x1 (black/white), Input Layer

            X_img = tf.reshape(self.X, [-1, 28, 28, 1])

            self.Y = tf.placeholder(tf.float32, [None, 10])

            # Convolutional Layer #1

            conv1 = tf.layers.conv2d(inputs=X_img, filters=32, kernel_size=[3, 3],

                                     padding="SAME", activation=tf.nn.relu)

            # Pooling Layer #1

            pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2],

                                            padding="SAME", strides=2)

            dropout1 = tf.layers.dropout(inputs=pool1,

                                         rate=0.7, training=self.training)

            # Convolutional Layer #2 and Pooling Layer #2

            conv2 = tf.layers.conv2d(inputs=dropout1, filters=64, kernel_size=[3, 3],

                                     padding="SAME", activation=tf.nn.relu)

            pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2],

                                            padding="SAME", strides=2)

            dropout2 = tf.layers.dropout(inputs=pool2,

                                         rate=0.7, training=self.training)

            # Convolutional Layer #3 and Pooling Layer #3

            conv3 = tf.layers.conv2d(inputs=dropout2, filters=128, kernel_size=[3, 3],

                                     padding="SAME", activation=tf.nn.relu)

            pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=[2, 2],

                                            padding="SAME", strides=2)

            dropout3 = tf.layers.dropout(inputs=pool3,

                                         rate=0.7, training=self.training)

            # Dense Layer with Relu

            flat = tf.reshape(dropout3, [-1, 128 * 4 * 4])

            dense4 = tf.layers.dense(inputs=flat,

                                     units=625, activation=tf.nn.relu)

            dropout4 = tf.layers.dropout(inputs=dense4,

                                         rate=0.5, training=self.training)

            # Logits (no activation) Layer: L5 Final FC 625 inputs -> 10 outputs

            self.logits = tf.layers.dense(inputs=dropout4, units=10)

        # define cost/loss & optimizer

        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(

            logits=self.logits, labels=self.Y))

        self.optimizer = tf.train.AdamOptimizer(

            learning_rate=learning_rate).minimize(self.cost)

        correct_prediction = tf.equal(

            tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))

        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    def predict(self, x_test, training=False):

        return self.sess.run(self.logits,

                             feed_dict={self.X: x_test, self.training: training})

    def get_accuracy(self, x_test, y_test, training=False):

        return self.sess.run(self.accuracy,

                             feed_dict={self.X: x_test,

                                        self.Y: y_test, self.training: training})

    def train(self, x_data, y_data, training=True):

        return self.sess.run([self.cost, self.optimizer], feed_dict={

            self.X: x_data, self.Y: y_data, self.training: training})

# initialize

sess = tf.Session()

models = []

num_models = 2

for m in range(num_models):

    models.append(Model(sess, "model" + str(m)))

sess.run(tf.global_variables_initializer())

print('Learning Started!')

# train my model

for epoch in range(training_epochs):

    avg_cost_list = np.zeros(len(models))

    total_batch = int(mnist.train.num_examples / batch_size)

    for i in range(total_batch):

        batch_xs, batch_ys = mnist.train.next_batch(batch_size)

        # train each model

        for m_idx, m in enumerate(models):

            c, _ = m.train(batch_xs, batch_ys)

            avg_cost_list[m_idx] += c / total_batch

    print('Epoch:', '%04d' % (epoch + 1), 'cost =', avg_cost_list)

print('Learning Finished!')

# Test model and check accuracy

test_size = len(mnist.test.labels)

predictions = np.zeros(test_size * 10).reshape(test_size, 10)

for m_idx, m in enumerate(models):

    print(m_idx, 'Accuracy:', m.get_accuracy(

        mnist.test.images, mnist.test.labels))

    p = m.predict(mnist.test.images)

    predictions += p

ensemble_correct_prediction = tf.equal(

    tf.argmax(predictions, 1), tf.argmax(mnist.test.labels, 1))

ensemble_accuracy = tf.reduce_mean(

    tf.cast(ensemble_correct_prediction, tf.float32))

print('Ensemble accuracy:', sess.run(ensemble_accuracy))

'''

0 Accuracy: 0.9933

1 Accuracy: 0.9946

2 Accuracy: 0.9934

3 Accuracy: 0.9935

4 Accuracy: 0.9935

5 Accuracy: 0.9949

6 Accuracy: 0.9941

Ensemble accuracy: 0.9952

'''

[ 참고자료 ]

  https://www.inflearn.com/course/기본적인-머신러닝-딥러닝-강좌

  https://github.com/hunkim/deeplearningzerotoall

  https://www.tensorflow.org/api_docs/python/tf/layers