Machine Learning - SVC

一.基本原理

二.程式碼實現

import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
import warnings
warnings.filterwarnings('ignore')

data = datasets.load_breast_cancer()
x = data.data;y = data.target
y[y==0] = -1
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.2,random_state=1)

omega = np.zeros(x.shape[1])
b = 0
lr = 0.01
C = 1
maxgen = 500

for L in range(maxgen):
    error = 1 - (np.dot(x_train,omega)+b)*y_train
    index = np.argmax(error)
    if error[index] > 0:
        omega = (1-lr)*omega + lr*C*y_train[index]*x_train[index,:]
        b = b + lr*C*y_train[index]
    else:
        break

predict_train = np.sign(np.dot(x_train,omega)+b)
train_acc = len(np.where(predict_train==y_train)[0])/len(y_train)
predict_test = np.sign(np.dot(x_test,omega)+b)
test_acc = len(np.where(predict_test==y_test)[0])/len(y_test)
print('Train acc = ',round(train_acc,4),' Test acc = ',round(test_acc,4))

三.SMO演算法實現

import numpy as np
from sklearn import datasets
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
import warnings
warnings.filterwarnings('ignore')

data = datasets.load_breast_cancer()
x = data.data;x = preprocessing.MinMaxScaler().fit_transform(x)
y = data.target;y[y==0] = -1
x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.2,random_state=1)

alpha = np.zeros(x_train.shape[0])
b = 0
C = 1
p = np.zeros(x_train.shape[0])
maxgen = 100

def gramMat(x):
    v = []
    for i in x:
        for j in x:
            k = np.dot(i,j.T)
            v.append(k)
    gram = np.array(v).reshape(len(x),len(x))
    return gram

K = gramMat(x_train)
            
def selectFirstSample(K,y,p,alpha,b,C):
    threshold = 0.001
    c = y*p-1
    c1 = c.copy();c2=c.copy();c3=c.copy()
    c1[(alpha > 0) & (c >= 0)] = 0
    c2[((alpha==0) | (alpha==C)) & (c==0)] = 0
    c3[(alpha < 0) & (c <= 0)] = 0
    error = c1**2+c2**2+c3**2
    index = np.argmax(error)
    if error[index] >= threshold:
        return index
    else:
        return None

def selectSecondSample(id1,y_train):
    id2 = np.random.randint(len(y_train))
    while id1 == id2:
        id2 = np.random.randint(len(y_train))
    return id2

def boundary(y,id1,id2,C):
    if y[id1] == y[id2]:
        lb = max(0,alpha[id1]+alpha[id2]-C)
        ub = min(C,alpha[id1]+alpha[id2])
    else:
        lb = max(0,alpha[id2]-alpha[id1])
        ub = min(C,C+alpha[id2]-alpha[id1])
    return lb,ub

def updateAlpha(K,y,p,id1,id2,lb,ub,C):
    old_alpha1 = alpha[id1];old_alpha2 = alpha[id2]
    p1 = p[id1];p2 = p[id2]
    y1 = y[id1];y2 = y[id2]
    E1 = p1 - y1;E2 = p2 - y2
    beta = 2*K[id1,id2] - K[id1,id1] - K[id2,id2]
    
    new_alpha2 = old_alpha2 - y2*(E1-E2)/beta
    if new_alpha2 > ub:
        new_alpha2 = ub
    if new_alpha2 < lb:
        new_alpha2 = lb
    alpha[id2] = new_alpha2
    
    new_alpha1 = old_alpha1 - y1*y2*(new_alpha2 - old_alpha2)
    alpha[id1] = new_alpha1
    
    deta1 = new_alpha1 - old_alpha1
    deta2 = new_alpha2 - old_alpha2
    dw = [y1*deta1,y2*deta2]
    b1 = -E1 - y1*K[id1,id1]*deta1 - y2*K[id1,id2]*deta2
    b2 = -E2 - y1*K[id1,id2]*deta1 - y2*K[id2,id2]*deta2
    if new_alpha1 >=0 and new_alpha1 <= C:
        db = b1
    elif new_alpha2 >=0 and new_alpha2 <= C:
        db = b2
    else:
        db = (b1 + b2)/2
    
    p = p + dw[0]*K[id1,:].T + dw[1]*K[id2,:].T + db
    return p

for L in range(maxgen):
    id1 = selectFirstSample(K,y_train,p,alpha,b,C)
    id2 = selectSecondSample(id1,y_train)
    lb,ub = boundary(y_train,id1,id2,C)
    p = updateAlpha(K,y,p,id1,id2,lb,ub,C)

index = np.argmax((alpha!=0)&(alpha!=C))
b = y_train[index] - np.sum(alpha*y_train*K[:,index])

predict_train = np.sign(np.sum(alpha*y_train*K,axis=0)+b)
train_acc = len(np.where(predict_train==y_train)[0])/len(y_train)

predict_test = []
for arr in x_test:
    v = np.sum(alpha*y_train*np.dot(x_train,arr.T))+b
    predict_test.append(np.sign(v))   
test_acc = len(np.where(predict_test==y_test)[0])/len(y_test)

print('self-written ==> Train acc = ',round(train_acc,4),' Test acc = ',round(test_acc,4))
    
model = SVC()
model.fit(x_train,y_train)
train_acc = model.score(x_train,y_train)
test_acc = model.score(x_test,y_test)
print('sklearn ==> Train acc = ',round(train_acc,4),' Test acc = ',round(test_acc,4))