棧式自編碼：

一．基本原理

AE的原理是先通過一個encode層對輸入進行編碼，這個編碼就是特徵，然後利用encode乘第2層引數（也可以是encode層的引數的轉置乘特徵並加偏執），重構（解碼）輸入，然後用重構的輸入和實際輸入的損失訓練引數。

對於我的應用來說，我要做的首先是抽取特徵。AE的encode的過程很簡單，是這樣的：

SAE是這樣的：

訓練過程中，兩個SAE分別訓練，第一個SAE訓練完之後，其encode的輸出作為第二個SAE的輸入，接著訓練。訓練完後，用第二層的特徵，通過softmax分類器（由於分類器 還是得要帶標籤的資料來訓練，所以，實際應用總，提取特徵後不一定是拿來分類），將所有特徵分為n類，得到n類標籤。訓練網路圖

          step 1

              step 2

                      step 3    

 按照UFLDL的說明，在各自訓練到快要收斂的時候，要對整個網路通過反向傳播做微調，但是不能從一開始就整個網路調節。

兩個SAE訓練完後，每個encode的輸出作為兩個隱層的特徵，然後重新構建網路，新網路不做訓練，只做預測。網路如下：

from __future__ import division, print_function, absolute_import

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

# 下載並匯入MNIST資料集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

# 引數
learning_rate = 0.01#學習率
training_epochs = 20#訓練的週期
batch_size = 256#每一批次訓練的大小
display_step = 1


# 神經網路的引數
n_hidden_1 = 256 # 隱層1的神經元個數
n_hidden_2 = 100 # 隱層2神經元個數
n_input = 784 # MNIST資料集的輸出(img shape: 28*28)
n_output=10

# tf Graph input (only pictures)
X = tf.placeholder("float", [None, n_input])

weights = {
    'encoder_h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
    'encoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
    'decoder_h1': tf.Variable(tf.random_normal([n_hidden_1, n_input])),
    'decoder_h2': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_1])),
    'softmax_w': tf.Variable(tf.random_normal([n_hidden_2, n_output])),
}
biases = {
    'encoder_b1': tf.Variable(tf.random_normal([n_hidden_1])),
    'encoder_b2': tf.Variable(tf.random_normal([n_hidden_2])),
    'decoder_b1': tf.Variable(tf.random_normal([n_input])),
    'decoder_b2': tf.Variable(tf.random_normal([n_hidden_1])),
    'softmax_b': tf.Variable(tf.random_normal([n_output])),
}




#************************* 1st hidden layer **************
X = tf.placeholder("float", [None, n_input])

h1_out =tf.nn.sigmoid(tf.add(tf.matmul(X, weights['encoder_h1']),
                                   biases['encoder_b1']))
keep_prob = tf.placeholder("float")
h1_out_drop = tf.nn.dropout(h1_out,keep_prob)

X_1 = tf.nn.sigmoid(tf.matmul(h1_out_drop,weights['decoder_h1'])+biases['decoder_b1'])

loss1 = tf.reduce_mean(tf.pow(X - X_1, 2))
train_step_1 = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss1)
sess=tf.Session()
sess.run(tf.variables_initializer([weights['encoder_h1'],biases['encoder_b1'],
                                  weights['decoder_h1'],biases['decoder_b1']]))
# training
for i in range(training_epochs):
    batch_x,batch_y =  mnist.train.next_batch(batch_size)
    _,c=sess.run([train_step_1,loss1],feed_dict={X:batch_x, keep_prob:1.0})
    if i%5==0:
        print(c)

#h1_out = tf.nn.sigmoid(tf.add(tf.matmul(X, weights['encoder_h1']),
#                                  biases['encoder_b1']))
#get result of 1st layer as well as the input of next layer

#************************** 2nd hidden layer *************
    
h2_x = tf.placeholder("float", shape = [None, n_hidden_1])

h2_out = tf.nn.sigmoid(tf.matmul(h2_x,weights['encoder_h2']) + biases['encoder_b2'])

h2_out_drop = tf.nn.dropout(h2_out,keep_prob)

h2_in_decode = tf.nn.sigmoid(tf.matmul(h2_out_drop, weights['decoder_h2']) +  biases['decoder_b2'])

loss2 = tf.reduce_mean(tf.pow( h2_x- h2_in_decode, 2))
train_step_2 = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss2)

sess.run(tf.variables_initializer([weights['encoder_h2'],biases['encoder_b2'],
                                   weights['decoder_h2'],biases['decoder_b2']]))

for i in range(training_epochs):
        #batch_x = numpy.reshape(batch_x,[batch_size,sample_length])		
    h1_out=[]
    batch_x,batch_y =  mnist.train.next_batch(batch_size)
    temp=tf.nn.sigmoid(tf.add(tf.matmul(batch_x, weights['encoder_h1']),
                                   biases['encoder_b1']))
    h1_out.extend(sess.run(temp))
    
    _,c=sess.run([train_step_2,loss2],feed_dict={h2_x:h1_out,keep_prob:1.0})
    if i%5==0:
        print(c)
#h2_out = tf.nn.sigmoid(tf.matmul(h1_out,weights['decoder_h2']) + biases['decoder_b2'])
#get result of 2nd layer as well as the input of next layer


#************************** softmax layer ****************
y_ = tf.placeholder("float", shape = [None, n_output])
soft_x = tf.placeholder("float", shape = [None, n_hidden_2])

y_out = tf.nn.softmax(tf.matmul(soft_x, weights['softmax_w']) + biases['softmax_b'])

cross_entropy = -tf.reduce_sum(y_ * tf.log(y_out))
train_step_soft = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_out, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

sess.run(tf.variables_initializer([weights['softmax_w'],biases['softmax_b']]))

for i in range(training_epochs):
    h2_out=[]
    batch_x,batch_y =  mnist.train.next_batch(batch_size)
    for i in range(batch_size):
        temp=tf.nn.sigmoid(tf.add(tf.matmul(batch_x[i].reshape([1,784]), weights['encoder_h1']),
                                       biases['encoder_b1']))

        temp=tf.nn.sigmoid(tf.add(tf.matmul(temp, weights['encoder_h2']),
                                       biases['encoder_b2']))
        h2_out.extend(sess.run(temp))

    sess.run(train_step_soft,feed_dict={soft_x:h2_out, y_:batch_y, keep_prob:1.0})
    if i%5 == 0:
        print(sess.run(accuracy, feed_dict={soft_x:h2_out, y_:batch_y, keep_prob:1.0}))



#fine-tuning
print ("************************* fine-tuning ***************************")
h1_out = tf.nn.sigmoid(tf.add(tf.matmul(X_1, weights['encoder_h1']),
                                   biases['encoder_b1']))
h2_out = tf.nn.sigmoid(tf.matmul(h1_out,weights['encoder_h2']) + biases['encoder_b2'])
        
h2_out_drop = tf.nn.dropout(h2_out,keep_prob)

y_out = tf.nn.softmax(tf.matmul(h2_out_drop, weights['softmax_w']) + biases['softmax_b'])

cross_entropy = -tf.reduce_sum(y_ * tf.log(y_out))
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_out, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

for i in range(training_epochs):
    batch_x,batch_y = mnist.train.next_batch(batch_size)
    #batch_x = numpy.reshape(batch_x,[batch_size,sample_length])		
    sess.run(train_step,feed_dict={X:batch_x, y_:batch_y, keep_prob:0.5})
    if i%5 == 0:
        print (sess.run(accuracy,feed_dict={X:batch_x, y_:batch_y, keep_prob:0.5}))
print (sess.run(accuracy,feed_dict={X:batch_x, y_:batch_y, keep_prob:0.5}))