本文作者：hhh5460

本文地址：https://www.cnblogs.com/hhh5460/p/10139738.html

例一的程式碼是函式式編寫的，這裡用面向物件的方式重新擼了一遍。好處是，更便於理解環境(Env)、個體(Agent)之間的關係。

有緣看到的朋友，自己慢慢體會吧。

0.效果圖

1.完整程式碼

import pandas as pd
import random
import time
import pickle
import pathlib

'''
-o---T
# T 就是寶藏的位置, o 是探索者的位置

作者：hhh5460
時間：20181218
地點：Tai Zi Miao
 
'''

class Env(object):
    '''環境類'''
    def __init__(self):
        '''初始化'''
        self.env = list('-----T')
        
    def update(self, state, delay=0.1):
        '''更新環境，並列印'''
        env = self.env[:]
        env[state] = 'o' # 更新環境
        print('\r{}'.format(''.join(env)), end='')
        time.sleep(delay)


 
class Agent(object):
    '''個體類'''
    def __init__(self, alpha=0.01, gamma=0.9):
        '''初始化'''
        self.states = range(6)
        self.actions = ['left', 'right']
        self.rewards = [0,0,0,0,0,1]
        
        self.alpha = alpha
        self.gamma = gamma
        
        self.q_table = pd.DataFrame(data=[[0 for
 
 _ in self.actions] for _ in self.states],
                                    index=self.states, 
                                    columns=self.actions)
    
    def save_policy(self):
        '''儲存Q table'''
        with open('q_table.pickle', 'wb') as f:
            # Pickle the 'data' dictionary using the highest protocol available.
            pickle.dump(self.q_table, f, pickle.HIGHEST_PROTOCOL)
    
    def load_policy(self):
        '''匯入Q table'''
        with open('q_table.pickle', 'rb') as f:
            self.q_table = pickle.load(f)
    
    def choose_action(self, state, epsilon=0.8):
        '''選擇相應的動作。根據當前狀態，隨機或貪婪，按照引數epsilon'''
        if (random.uniform(0,1) > epsilon) or ((self.q_table.ix[state] == 0).all()):  # 探索
            action = random.choice(self.get_valid_actions(state))
        else:
            action = self.q_table.ix[state].idxmax() # 利用（貪婪）
        return action
    
    def get_q_values(self, state):
        '''取狀態state的所有Q value'''
        q_values = self.q_table.ix[state, self.get_valid_actions(state)]
        return q_values
        
    def update_q_value(self, state, action, next_state_reward, next_state_q_values):
        '''更新Q value，根據貝爾曼方程'''
        self.q_table.ix[state, action] += self.alpha * (next_state_reward + self.gamma * next_state_q_values.max() - self.q_table.ix[state, action])
    
    def get_valid_actions(self, state):
        '''取當前狀態下所有的合法動作'''
        valid_actions = set(self.actions)
        if state == self.states[-1]:             # 最後一個狀態（位置），則
            valid_actions -= set(['right']) # 不能向右
        if state == self.states[0]:              # 最前一個狀態（位置），則
            valid_actions -= set(['left'])  # 不能向左
        return list(valid_actions)
    
    def get_next_state(self, state, action):
        '''對狀態執行動作後，得到下一狀態'''
        # l,r,n = -1,+1,0
        if action == 'right' and state != self.states[-1]: # 除非最後一個狀態（位置），向右就+1
            next_state = state + 1
        elif action == 'left' and state != self.states[0]: # 除非最前一個狀態（位置），向左就-1
            next_state = state -1
        else:
            next_state = state
        return next_state
    
    def learn(self, env=None, episode=1000, epsilon=0.8):
        '''q-learning演算法'''
        print('Agent is learning...')
        for _ in range(episode):
            current_state = self.states[0]
            
            if env is not None: # 若提供了環境，則更新之！
                env.update(current_state)
                
            while current_state != self.states[-1]:
                current_action = self.choose_action(current_state, epsilon) # 按一定概率，隨機或貪婪地選擇
                next_state = self.get_next_state(current_state, current_action)
                next_state_reward = self.rewards[next_state]
                next_state_q_values = self.get_q_values(next_state)
                self.update_q_value(current_state, current_action, next_state_reward, next_state_q_values)
                current_state = next_state
                
                if env is not None: # 若提供了環境，則更新之！
                    env.update(current_state)
        print('\nok')
                    
    def play(self, env=None, delay=0.5):
        '''玩遊戲，使用策略'''
        assert env != None, 'Env must be not None!'
        
        if pathlib.Path("q_table.pickle").exists():
            self.load_policy()
        else:
            print("I need to learn before playing this game.")
            self.learn(env, 13)
            self.save_policy()
        
        print('Agent is playing...')
        current_state = self.states[0]
        env.update(current_state, delay)
        while current_state != self.states[-1]:
            current_action = self.choose_action(current_state, 1.) # 1., 不隨機
            next_state = self.get_next_state(current_state, current_action)
            current_state = next_state
            env.update(current_state, delay)
        print('\nCongratulations, Agent got it!')


if __name__ == '__main__':
    env = Env()     # 環境
    agent = Agent() # 個體
    #agent.learn(env, episode=13) # 先學
    #agent.save_policy() # 儲存所學
    #agent.load_policy() # 匯入所學
    agent.play(env)              # 再玩