TensorFlow 學習率 learning rate的設定：指數衰減法。

函式：tr.train.exponential_decay

    初始時使用較大的學習率較快地得到較優解，隨著迭代學習率呈指數逐漸減小。

decayed_learning_rate = learning_rate*(decay_rate^(global_steps/decay_steps)

import tensorflow as tf
import matplotlib.pyplot as plt

learning_rate = 0.1  # 學習速率L
decay_rate = 0.8  # 衰減速率，即每一次學習都衰減為原來的0.8
global_steps = 1000  # 總學習次數
# 如果staircase為True,那麼每decay_steps改變一次learning_rate，
# 改變為learning_rate*(decay_rate^(global_steps/decay_steps)
# 如果為False則，每一步都改變
decay_steps = 100  

global_ = tf.placeholder(dtype=tf.int32)
# 如果staircase=True，那麼每decay_steps更新一次decay_rate，如果是False那麼每一步都更新一次decay_rate。
c = tf.train.exponential_decay(learning_rate, global_, decay_steps, decay_rate, staircase=True)
d = tf.train.exponential_decay(learning_rate, global_, decay_steps, decay_rate, staircase=False)

T_C = []
F_D = []

with tf.Session() as sess:
    for i in range(global_steps):
        T_c = sess.run(c, feed_dict={global_: i})
        T_C.append(T_c)
        F_d = sess.run(d, feed_dict={global_: i})
        F_D.append(F_d)

plt.figure(1)
l1, = plt.plot(range(global_steps), F_D, 'r-')  # staircase=False
l2, = plt.plot(range(global_steps), T_C, 'b-')  # staircase=True

plt.legend(handles=[l1, l2, ], labels=['staircase=False', 'staircase=True'],
           loc='best', ) 
plt.show()