作者：faaronzheng 轉載請註明出處！

最近再看Machine Learning in Action. k近鄰演算法這一章節提供了不少例子，本著Talk is cheap的原則，我們用手寫數字識別來實際測試一下。 簡單的介紹一下k近鄰演算法（KNN）：給定測試樣本，基於某種距離度量找出訓練集中與其最靠近的k個訓練樣本，然後基於這k個“鄰居”的資訊來進行預測。如下圖所示：

x為測試樣本，小黑點是一類樣本，小紅點是另一類樣本。在測試樣本x的周圍畫一個圈，這個圈就是依據某種距離度量畫出的，可以看到我們選擇的是5近鄰。現在我們要做出一個預測，就是這個測試樣本x是屬於小黑點那一類還是小紅點那一類呢？很簡單，我們只要看看選中的近鄰中哪一類樣本多就把這類樣本的標籤賦給測試樣本就可以了。圖中自然就是小黑點，所以我們預測x是小黑點。

正文：

第一步：準備實驗資料。Machine Learning in Action書中的資料使用的是“手寫數字資料集的光學識別”一文中的資料。具體可以參考書中的相關介紹。所有的資料是以Txt形式儲存的，由32行32列的0/1元素組成。下圖就是一個手寫數字0的儲存資料。可以看出，數字所在的位置用1表示，空白的用0表示。

除此之外，為了能識別自己手寫的數字，我們在原來實驗的基礎上新增畫板的功能，使其能採集自己手寫的數字並按照相同的格式儲存下來。如下圖所示，當點選CustomizeTestData後會出現一個畫板，當我們在畫板上寫上數字後，按下ESC鍵儲存圖片並退出，接下來將儲存的圖片處理成我們想要的格式，就可以用演算法對其進行預測了。畫板的實現使用了pygame。

下面是畫板功能的具體實現：

import pygame
from pygame.locals import *
import math
from sys import exit
#向sys模組借一個exit函式用來退出程式
pygame.init()
#初始化pygame,為使用硬體做準備
 
class Brush():
 def __init__(self, screen):
  self.screen = screen
  self.color = (0, 0, 0)
  self.size = 4
  self.drawing = False
  self.last_pos = None
  self.space = 1
  # if style is True, normal solid brush
  # if style is False, png brush
  self.style = False
  # load brush style png
  self.brush = pygame.image.load("brush.png").convert_alpha()
  # set the current brush depends on size
  self.brush_now = self.brush.subsurface((0,0), (1, 1))
 
 def start_draw(self, pos):
  self.drawing = True
  self.last_pos = pos
 def end_draw(self):
  self.drawing = False
 
 def set_brush_style(self, style):
  print "* set brush style to", style
  self.style = style
 def get_brush_style(self):
  return self.style
 
 def get_current_brush(self):
  return self.brush_now
 
 def set_size(self, size):
  if size < 0.5: size = 0.5
  elif size > 32: size = 32
  print "* set brush size to", size
  self.size = size
  self.brush_now = self.brush.subsurface((0,0), (size*2, size*2))
 def get_size(self):
  return self.size
 
 def set_color(self, color):
  self.color = color
  for i in xrange(self.brush.get_width()):
   for j in xrange(self.brush.get_height()):
    self.brush.set_at((i, j),
      color + (self.brush.get_at((i, j)).a,))
 def get_color(self):
  return self.color
 
 def draw(self, pos):
  if self.drawing:
   for p in self._get_points(pos):
    # draw eveypoint between them
    if self.style == False:
     pygame.draw.circle(self.screen, self.color, p, self.size)
    else:
     self.screen.blit(self.brush_now, p)
 
   self.last_pos = pos
 
 def _get_points(self, pos):
  """ Get all points between last_point ~ now_point. """
  points = [ (self.last_pos[0], self.last_pos[1]) ]
  len_x = pos[0] - self.last_pos[0]
  len_y = pos[1] - self.last_pos[1]
  length = math.sqrt(len_x ** 2 + len_y ** 2)
  step_x = len_x / length
  step_y = len_y / length
  for i in xrange(int(length)):
   points.append(
     (points[-1][0] + step_x, points[-1][1] + step_y))
  points = map(lambda x:(int(0.5+x[0]), int(0.5+x[1])), points)
  # return light-weight, uniq integer point list
  return list(set(points))
 
class Menu():
 def __init__(self, screen):
  self.screen = screen
  self.brush = None

 def set_brush(self, brush):
  self.brush = brush

 
class Painter():
 def __init__(self):
  self.screen = pygame.display.set_mode((100, 100))
 # self.menu = pygame.display.set_mode((80, 600))
  pygame.display.set_caption("Painter")
  self.clock = pygame.time.Clock()
  self.brush = Brush(self.screen)
  self.menu = Menu(self.screen)
  self.menu.set_brush(self.brush)
 
 def run(self):
  self.screen.fill((255, 255, 255))
  while True:
   # max fps limit
   self.clock.tick(30)
   for event in pygame.event.get():
    if event.type == QUIT:
        pygame.quit()
     #   break
    elif event.type == KEYDOWN:
     # press esc to clear screen
     if event.key == K_ESCAPE:
      fname = "test.png"
      pygame.image.save(self.screen, fname)    
      pygame.quit()
      #break
    elif event.type == MOUSEBUTTONDOWN:
     # <= 74, coarse judge here can save much time
     if ((event.pos)[0] <= 74 and
       self.menu.click_button(event.pos)):
      # if not click on a functional button, do drawing
      pass
     else:
      self.brush.start_draw(event.pos)
    elif event.type == MOUSEMOTION:
     self.brush.draw(event.pos)
    elif event.type == MOUSEBUTTONUP:
     self.brush.end_draw()
    self.menu.draw()
    pygame.display.update()
 

KNN演算法--KNN的關鍵在我看來是距離度量的選擇。不同的距離度量會對最終的結果產生比較大的影響。首先將手寫數字變化為一個一維的向量，通過計算測試樣例（向量）和每個訓練樣本（向量）之間的距離然後進行排序。最後選最近的k個進行投票產生對測試樣例的預測。

import pygame
from numpy import *
import operator
from os import listdir
from Board import *
import Tkinter
import tkFileDialog
import tkMessageBox
import Image  
from KNN import dot
pygame.init()


def classify0(inX, dataSet, labels, k):           #k控制選取最近的k個近鄰然後投票
    dataSetSize = dataSet.shape[0]
    #計算歐式距離（其實比較的是兩個向量之間的距離）
    diffMat = tile(inX, (dataSetSize,1)) - dataSet
    sqDiffMat = diffMat**2
    sqDistances = sqDiffMat.sum(axis=1)
    distances = sqDistances**0.5
    sortedDistIndicies = distances.argsort()     
    classCount={}          
    #投票
    for i in range(k):
        voteIlabel = labels[sortedDistIndicies[i]]
        classCount[voteIlabel] = classCount.get(voteIlabel,0) + 1
    sortedClassCount = sorted(classCount.iteritems(), key=operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]
def classify1(inX,dataSet,labels, k):
 dataSetSize = dataSet.shape[0]
 diffMat = tile(inX, (dataSetSize,1)) - dataSet
 diffMatT=(diffMat.T)
 sqDiffMat = dot(diffMat,diffMat.T)
 distances = sqrt(sqDiffMat)   
 sortedDistIndicies=distances.argsort() 
 classCount={}         
#投票
 for i in range(k):
    voteIlabel = labels[sortedDistIndicies[i]]
    classCount[voteIlabel] = classCount.get(voteIlabel,0) + 1
 sortedClassCount = sorted(classCount.iteritems(), key=operator.itemgetter(1), reverse=True)
 return sortedClassCount[0][0]
# 將檔案轉化為向量
def img2vector(filename):
    returnVect = zeros((1,1024))
    fr = open(filename)
    for i in range(32):
        lineStr = fr.readline()
        for j in range(32):
            returnVect[0,32*i+j] = int(lineStr[j])
    return returnVect

def handwritingClassTest(TrainDataPath):
    hwLabels = []
    trainingFileList = listdir(TrainDataPath)           #load the training set
    m = len(trainingFileList)
    trainingMat = zeros((m,1024))
    for i in range(m):
        fileNameStr = trainingFileList[i]
        fileStr = fileNameStr.split('.')[0]     #take off .txt
        classNumStr = int(fileStr.split('_')[0])
        hwLabels.append(classNumStr)
        trainingMat[i,:] = img2vector(TrainDataPath+'/%s' % fileNameStr)
    testFileList = listdir('C:/Users/HP/Desktop/MLiA_SourceCode/machinelearninginaction/Ch02/testDigits')        #iterate through the test set
    errorCount = 0.0
    mTest = len(testFileList)
    for i in range(mTest):
        fileNameStr = testFileList[i]
        fileStr = fileNameStr.split('.')[0]     #take off .txt
        classNumStr = int(fileStr.split('_')[0])
        vectorUnderTest = img2vector('C:/Users/HP/Desktop/MLiA_SourceCode/machinelearninginaction/Ch02/testDigits/%s' % fileNameStr)
        classifierResult = classify0(vectorUnderTest, trainingMat, hwLabels, 3)
        print "the classifier came back with: %d, the real answer is: %d" % (classifierResult, classNumStr)
        if (classifierResult != classNumStr): errorCount += 1.0
    print "\nthe total number of errors is: %d" % errorCount
    print "\nthe total error rate is: %f" % (errorCount/float(mTest))

top = Tkinter.Tk()

def TrainDataCallBack():
    TrainDataPath=tkFileDialog.askdirectory()
    handwritingClassTest(TrainDataPath)

def CustomizeTestDataCallBack():
    board = Painter()
    board.run()
    
def TestingCustomizeTestDataCallBack():
    ResizePic()
    TransformArray()
TrainDataButton = Tkinter.Button(top, text ="TrainData", command = TrainDataCallBack)
CustomizeTestDataButton = Tkinter.Button(top, text ="CustomizeTestData", command = CustomizeTestDataCallBack)
TestingButton = Tkinter.Button(top, text ="TestingCustomizeTestData", command = TestingCustomizeTestDataCallBack)

def ResizePic():
    im = Image.open("test.png")  
    w,h = im.size  
    im_ss = im.resize((int(32), int(32)))  
    im_ss.save("test.png")  

def TransformArray():
    TestArray = zeros((1,1024))
    im = Image.open("test.png")  
    width,height = im.size  
    for h in range(0, height):  
      for w in range(0, width):  
        pixel = im.getpixel((w, h))      
        if pixel!=(255,255,255):
            TestArray[0,32*h+w]=int(1)
    handwritingTesting(TestArray)


def handwritingTesting(TestArray):
  #  TrainDataPath=tkFileDialog.askdirectory()
    TrainDataPath="C:/Users/HP/Desktop/MLiA_SourceCode/machinelearninginaction/Ch02/trainingDigits"
    hwLabels = []
    trainingFileList = listdir(TrainDataPath)           #load the training set
    m = len(trainingFileList)
    trainingMat = zeros((m,1024))
    for i in range(m):
        fileNameStr = trainingFileList[i]
        fileStr = fileNameStr.split('.')[0]     #take off .txt
        classNumStr = int(fileStr.split('_')[0])
        hwLabels.append(classNumStr)
        trainingMat[i,:] = img2vector(TrainDataPath+'/%s' % fileNameStr)      
    classifierResult = classify0(TestArray, trainingMat, hwLabels, 100)
    classifierResult1 = classify1(TestArray, trainingMat, hwLabels, 100)
    print "the classifier came back with: %d"  %  classifierResult
    print "the classifier came back with: %d"  %  classifierResult1
       
TrainDataButton.pack()
CustomizeTestDataButton.pack()
TestingButton.pack()
top.mainloop()

未完待續。。。