機器學習實戰精讀--------決策樹
阿新 • • 發佈:2017-08-17
決策樹 機器學習 python
感覺自己像個學走路的孩子,每一步都很吃力和認真!
機器根據數據集創建規則,就是機器學習。
決策樹:從數據集合中提取一系列規則,適用於探索式的知識發現。
決策樹本質:通過一系列規則對數據進行分類的過程。
決策樹算法核心:構建精度高,數據規模小的決策樹。
ID3算法:此算法目的在於減少樹的深度,但是忽略了葉子數目的研究。
C4.5算法:對ID3進行改進,對於預測變量的缺失值處理、剪枝技術、派生規則等方面作了較大改進,既適合分類,又適合回歸。
香農熵:變量的不確定性越大,熵也就越大,把它搞清楚所需要的信息量也就越大。
基尼不純度:一個隨機事件變成它的對立事件的概率。
#coding:utf-8 from math import log import operator #創建數據 def createDataSet(): dataSet = [[1, 1, ‘yes‘], [1, 1, ‘yes‘], [1, 0, ‘no‘], [0, 1, ‘no‘], [0, 1, ‘no‘]] labels = [‘no surfacing‘,‘flippers‘] #change to discrete values return dataSet, labels #計算給定數據集的香農熵 def calcShannonEnt(dataSet): numEntries = len(dataSet) #輸出列表的的長度 值為:5 labelCounts = {} for featVec in dataSet: #對二維數組進行遍歷 currentLabel = featVec[-1] #把每個子列表的最後一個元素賦值給currentLabel if currentLabel not in labelCounts.keys(): labelCounts[currentLabel] = 0 #如果currentLabel 不在字典中,把字典的值設置為0 # keys() 函數以列表返回一個字典所有的鍵。 labelCounts[currentLabel] += 1 #這一步是統計出現的次數,每次匹配到的話,把對應key的值加1 #操作完以後labelCounts字典記錄的是yes出現2次 no出現3次 shannonEnt = 0.0 for key in labelCounts: prob = float(labelCounts[key])/numEntries #prob是字典labelCounts每個key的比率 shannonEnt -= prob * log(prob,2) #log(x, 2) 表示以2為底的對數。 return shannonEnt #熵數字越大,說明混合的數據越多 #按照給定特征劃分數據集 def splitDataSet(dataSet, axis, value): #dataSet:待劃分的數據集;axis:劃分數據集的特征;value:需要返回的特征的值 retDataSet = [] for featVec in dataSet: if featVec[axis] == value: #如果featVec列表的特征值跟valve值一樣 reducedFeatVec = featVec[:axis] #輸出列表featVec開頭到特征值的列表切片 reducedFeatVec.extend(featVec[axis+1:]) retDataSet.append(reducedFeatVec) return retDataSet #選擇最好的數據集劃分方式 def chooseBestFeatureToSplit(dataSet): numFeatures = len(dataSet[0]) - 1 #the last column is used for the labels baseEntropy = calcShannonEnt(dataSet) bestInfoGain = 0.0; bestFeature = -1 for i in range(numFeatures): #iterate over all the features featList = [example[i] for example in dataSet]#create a list of all the examples of this feature uniqueVals = set(featList) #get a set of unique values newEntropy = 0.0 for value in uniqueVals: subDataSet = splitDataSet(dataSet, i, value) prob = len(subDataSet)/float(len(dataSet)) newEntropy += prob * calcShannonEnt(subDataSet) infoGain = baseEntropy - newEntropy #calculate the info gain; ie reduction in entropy if (infoGain > bestInfoGain): #compare this to the best gain so far bestInfoGain = infoGain #if better than current best, set to best bestFeature = i return bestFeature #returns an integer def majorityCnt(classList): classCount={} for vote in classList: if vote not in classCount.keys(): classCount[vote] = 0 classCount[vote] += 1 sortedClassCount = sorted(classCount.iteritems(), key=operator.itemgetter(1), reverse=True) return sortedClassCount[0][0] def createTree(dataSet,labels): classList = [example[-1] for example in dataSet] if classList.count(classList[0]) == len(classList): return classList[0]#stop splitting when all of the classes are equal if len(dataSet[0]) == 1: #stop splitting when there are no more features in dataSet return majorityCnt(classList) bestFeat = chooseBestFeatureToSplit(dataSet) bestFeatLabel = labels[bestFeat] myTree = {bestFeatLabel:{}} del(labels[bestFeat]) featValues = [example[bestFeat] for example in dataSet] uniqueVals = set(featValues) for value in uniqueVals: subLabels = labels[:] #copy all of labels, so trees don‘t mess up existing labels myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value),subLabels) return myTree #使用決策樹的分類函數 def classify(inputTree,featLabels,testVec): firstStr = inputTree.keys()[0] secondDict = inputTree[firstStr] featIndex = featLabels.index(firstStr) key = testVec[featIndex] valueOfFeat = secondDict[key] if isinstance(valueOfFeat, dict): classLabel = classify(valueOfFeat, featLabels, testVec) else: classLabel = valueOfFeat return classLabel #使用pickle模塊存儲決策樹 def storeTree(inputTree,filename): import pickle fw = open(filename,‘w‘) pickle.dump(inputTree,fw) fw.close() def grabTree(filename): import pickle fr = open(filename) return pickle.load(fr)
#coding:utf-8 import matplotlib as mpl mpl.use(‘Agg‘) import matplotlib.pyplot as plt decisionNode = dict(boxstyle="sawtooth", fc="0.8") leafNode = dict(boxstyle="round4", fc="0.8") arrow_args = dict(arrowstyle="<-") def getNumLeafs(myTree): numLeafs = 0 firstStr = myTree.keys()[0] secondDict = myTree[firstStr] for key in secondDict.keys(): if type(secondDict[key]).__name__==‘dict‘:#test to see if the nodes are dictonaires, if not they are leaf nodes numLeafs += getNumLeafs(secondDict[key]) else: numLeafs +=1 return numLeafs def getTreeDepth(myTree): maxDepth = 0 firstStr = myTree.keys()[0] secondDict = myTree[firstStr] for key in secondDict.keys(): if type(secondDict[key]).__name__==‘dict‘:#test to see if the nodes are dictonaires, if not they are leaf nodes thisDepth = 1 + getTreeDepth(secondDict[key]) else: thisDepth = 1 if thisDepth > maxDepth: maxDepth = thisDepth return maxDepth def plotNode(nodeTxt, centerPt, parentPt, nodeType): createPlot.ax1.annotate(nodeTxt, xy=parentPt, xycoords=‘axes fraction‘, xytext=centerPt, textcoords=‘axes fraction‘, va="center", ha="center", bbox=nodeType, arrowprops=arrow_args ) def plotMidText(cntrPt, parentPt, txtString): xMid = (parentPt[0]-cntrPt[0])/2.0 + cntrPt[0] yMid = (parentPt[1]-cntrPt[1])/2.0 + cntrPt[1] createPlot.ax1.text(xMid, yMid, txtString, va="center", ha="center", rotation=30) def plotTree(myTree, parentPt, nodeTxt):#if the first key tells you what feat was split on numLeafs = getNumLeafs(myTree) #this determines the x width of this tree depth = getTreeDepth(myTree) firstStr = myTree.keys()[0] #the text label for this node should be this cntrPt = (plotTree.xOff + (1.0 + float(numLeafs))/2.0/plotTree.totalW, plotTree.yOff) plotMidText(cntrPt, parentPt, nodeTxt) plotNode(firstStr, cntrPt, parentPt, decisionNode) secondDict = myTree[firstStr] plotTree.yOff = plotTree.yOff - 1.0/plotTree.totalD for key in secondDict.keys(): if type(secondDict[key]).__name__==‘dict‘:#test to see if the nodes are dictonaires, if not they are leaf nodes plotTree(secondDict[key],cntrPt,str(key)) #recursion else: #it‘s a leaf node print the leaf node plotTree.xOff = plotTree.xOff + 1.0/plotTree.totalW plotNode(secondDict[key], (plotTree.xOff, plotTree.yOff), cntrPt, leafNode) plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key)) plotTree.yOff = plotTree.yOff + 1.0/plotTree.totalD #if you do get a dictonary you know it‘s a tree, and the first element will be another dict def createPlot(inTree): fig = plt.figure(1, facecolor=‘white‘) fig.clf() axprops = dict(xticks=[], yticks=[]) createPlot.ax1 = plt.subplot(111, frameon=False, **axprops) #no ticks #createPlot.ax1 = plt.subplot(111, frameon=False) #ticks for demo puropses plotTree.totalW = float(getNumLeafs(inTree)) plotTree.totalD = float(getTreeDepth(inTree)) plotTree.xOff = -0.5/plotTree.totalW; plotTree.yOff = 1.0; plotTree(inTree, (0.5,1.0), ‘‘) plt.savefig(‘trees.png‘) ‘‘‘ def createPlot(): fig = plt.figure(1, facecolor=‘white‘) #創建第一張圖,設置背景色是白色 fig.clf() createPlot.ax1 = plt.subplot(111, frameon=False) #ticks for demo puropses plotNode(‘a decision node‘, (0.5, 0.1), (0.1, 0.5), decisionNode) plotNode(‘a leaf node‘, (0.8, 0.1), (0.3, 0.8), leafNode) plt.savefig(‘trees.png‘) ‘‘‘ def retrieveTree(i): listOfTrees =[{‘no surfacing‘: {0: ‘no‘, 1: {‘flippers‘: {0: ‘no‘, 1: ‘yes‘}}}}, {‘no surfacing‘: {0: ‘no‘, 1: {‘flippers‘: {0: {‘head‘: {0: ‘no‘, 1: ‘yes‘}}, 1: ‘no‘}}}} ] return listOfTrees[i]
小結:
一、一棵最優決策樹主要應解決以下三個問題:
① 生成最少數目的葉子節點
② 生成每個葉子節點的深度最小
③ 生成的決策樹葉子節點最少且每個葉子節點深度最小
二、如果葉子節點只能增加少許信息,則可以刪除該節點,將它並入到其它節點中去。
三、ID3算法無法直接處理數值型數據。
四、采用文本方式很難展示決策樹,所以要用Matplotlib註解繪制樹型圖。
本文出自 “付煒超” 博客,謝絕轉載!
機器學習實戰精讀--------決策樹