機器學習實戰精讀--------FP-growth算法
阿新 • • 發佈:2017-09-04
fp-growth算法 頻繁項集
從數據集獲取有趣信息的方法:常用的兩種分別是頻繁項集和關聯規則。
FP-growth:雖然可以高效的發現頻繁項集,但是不能用於發現關聯規則。
FP-growth算法只需要對數據庫進行兩次掃描,速度要比Apriori算法塊。
FP-growth發現頻繁項集的基本過程:
① 構建FP樹
第一遍 對所有元素項的出現次數進行技術,用來統計出現的頻率。
第二遍 只考慮哪些頻繁元素
② 從FP樹種挖掘頻繁項集。
從FP樹種抽取頻繁項集的三個基本步驟:
① 從FP樹種獲得條件模式基
② 利用條件模式基,構建一個條件FP樹
③ 叠代重復步驟(1)步驟(2),知道樹包含一個元素項為止。
條件模式基:是以查找元素項為結尾的路徑集合。
#coding:utf-8 #FP數的類定義 class treeNode: def __init__(self, nameValue, numOccur, parentNode): self.name = nameValue #存放節點名字的變量 self.count = numOccur #計數值 self.nodeLink = None #用於鏈接相似的元素項 self.parent = parentNode #父變量parent來指向擋墻節點的父節點 self.children = {} #空字典用來存放節點的子節點 #對count變量增加給定值 def inc(self, numOccur): self.count += numOccur #用於將數以文本形式展示,非必要,但是便於調試 def disp(self, ind=1): print ‘ ‘*ind, self.name, ‘ ‘, self.count for child in self.children.values(): child.disp(ind+1) #構建FP樹 def createTree(dataSet, minSup=1): #使用數據集以及最小支持度作為參賽來構建FP樹 headerTable = {} #創建空的頭指針表 #遍歷數據集兩次 for trans in dataSet:#first pass counts frequency of occurance for item in trans: headerTable[item] = headerTable.get(item, 0) + dataSet[trans] for k in headerTable.keys(): #刪除出現次數少於minsup的項 if headerTable[k] < minSup: del(headerTable[k]) freqItemSet = set(headerTable.keys()) if len(freqItemSet) == 0: return None, None #如果所有項都不頻繁,就不進行下一步了 for k in headerTable: headerTable[k] = [headerTable[k], None] #reformat headerTable to use Node link retTree = treeNode(‘Null Set‘, 1, None) #create tree for tranSet, count in dataSet.items(): #go through dataset 2nd time localD = {} for item in tranSet: #put transaction items in order if item in freqItemSet: localD[item] = headerTable[item][0] if len(localD) > 0: orderedItems = [v[0] for v in sorted(localD.items(), key=lambda p: p[1], reverse=True)] updateTree(orderedItems, retTree, headerTable, count)#populate tree with ordered freq itemset return retTree, headerTable #return tree and header table def updateTree(items, inTree, headerTable, count): if items[0] in inTree.children:#測試事務中的第一個元素是否作為子節點存在,如果存在,更新該元素項的計數 inTree.children[items[0]].inc(count) else: #如果不存在,則創建一個新的treeNode,並將其作為一個子節點添加到樹中 inTree.children[items[0]] = treeNode(items[0], count, inTree) if headerTable[items[0]][1] == None: #update header table headerTable[items[0]][1] = inTree.children[items[0]] else: updateHeader(headerTable[items[0]][1], inTree.children[items[0]]) #更新頭指針表 if len(items) > 1:#不斷叠代調用自身,每次調用時會去掉列表中的第一個元素 updateTree(items[1::], inTree.children[items[0]], headerTable, count) def updateHeader(nodeToTest, targetNode): #它確保節點鏈接指向樹中該元素項的每一個實例 while (nodeToTest.nodeLink != None): #Do not use recursion to traverse a linked list! nodeToTest = nodeToTest.nodeLink nodeToTest.nodeLink = targetNode def ascendTree(leafNode, prefixPath): #ascends from leaf node to root if leafNode.parent != None: prefixPath.append(leafNode.name) ascendTree(leafNode.parent, prefixPath) def findPrefixPath(basePat, treeNode): #treeNode comes from header table condPats = {} while treeNode != None: prefixPath = [] ascendTree(treeNode, prefixPath) if len(prefixPath) > 1: condPats[frozenset(prefixPath[1:])] = treeNode.count treeNode = treeNode.nodeLink return condPats #遞歸查找頻繁項集 def mineTree(inTree, headerTable, minSup, preFix, freqItemList): bigL = [v[0] for v in sorted(headerTable.items(), key=lambda p: p[1])]# 對頭指針表中的元素項按照其出現頻率進行排序 for basePat in bigL: #start from bottom of header table newFreqSet = preFix.copy() newFreqSet.add(basePat) freqItemList.append(newFreqSet) condPattBases = findPrefixPath(basePat, headerTable[basePat][1]) myCondTree, myHead = createTree(condPattBases, minSup) if myHead != None: #如果樹中有元素項,遞歸調用mineTree()函數 mineTree(myCondTree, myHead, minSup, newFreqSet, freqItemList) def loadSimpDat(): simpDat = [[‘r‘, ‘z‘, ‘h‘, ‘j‘, ‘p‘], [‘z‘, ‘y‘, ‘x‘, ‘w‘, ‘v‘, ‘u‘, ‘t‘, ‘s‘], [‘z‘], [‘r‘, ‘x‘, ‘n‘, ‘o‘, ‘s‘], [‘y‘, ‘r‘, ‘x‘, ‘z‘, ‘q‘, ‘t‘, ‘p‘], [‘y‘, ‘z‘, ‘x‘, ‘e‘, ‘q‘, ‘s‘, ‘t‘, ‘m‘]] return simpDat def createInitSet(dataSet): retDict = {} for trans in dataSet: retDict[frozenset(trans)] = 1 return retDict import twitter from time import sleep import re def textParse(bigString): urlsRemoved = re.sub(‘(http:[/][/]|www.)([a-z]|[A-Z]|[0-9]|[/.]|[~])*‘, ‘‘, bigString) #re.sub使用給定的替換內容將匹配模式的子字符串 listOfTokens = re.split(r‘\W*‘, urlsRemoved) #通過正則表達式將字符串分離 return [tok.lower() for tok in listOfTokens if len(tok) > 2] #列表推導式生成新列表 #處理認證然後創建一個空列表 def getLotsOfTweets(searchStr): CONSUMER_KEY = ‘get when you create an app‘ CONSUMER_SECRET = ‘get when you create an app‘ ACCESS_TOKEN_KEY = ‘get from Oauth,apecific to a user‘ ACCESS_TOKEN_SECRET = ‘get from Oauth,apecific to a user‘ api = twitter.Api(consumer_key=CONSUMER_KEY, consumer_secret=CONSUMER_SECRET, access_token_key=ACCESS_TOKEN_KEY, access_token_secret=ACCESS_TOKEN_SECRET) #you can get 1500 results 15 pages * 100 per page resultsPages = [] for i in range(1,15): print "fetching page %d" % i searchResults = api.GetSearch(searchStr, per_page=100, page=i) resultsPages.append(searchResults) sleep(6) return resultsPages #構建FP樹並對其進行挖掘,最後返回所有頻繁項集組成的列表 def mineTweets(tweetArr, minSup=5): parsedList = [] for i in range(14): for j in range(100): parsedList.append(textParse(tweetArr[i][j].text)) initSet = createInitSet(parsedList) myFPtree, myHeaderTab = createTree(initSet, minSup) myFreqList = [] mineTree(myFPtree, myHeaderTab, minSup, set([]), myFreqList) return myFreqList
本文出自 “付煒超” 博客,謝絕轉載!
機器學習實戰精讀--------FP-growth算法