基於SKLearn的SVM模型垃圾郵件分類——程式碼實現及優化

 1 #coding:utf-8
 2 import jieba
 3 import pandas as pd
 4 import numpy as np
 5
 6 from sklearn.feature_extraction.text import CountVectorizer
 7 from sklearn.svm import SVC
 8 from sklearn.model_selection import train_test_split
 9 from sklearn.externals import joblib
10
11 path_list_spam = []
12 with open('./data/CN_index_spam','r',encoding='utf-8') as fin:
13     for line in fin.readlines():
14         path_list_spam.append(line.strip())
15
16 path_list_ham = []
17 with open('./data/CN_index_ham','r',encoding='utf-8') as fin:
18     for line in fin.readlines():
19         path_list_ham.append(line.strip())
20
21 stopWords = []
22 with open('./data/CN_stopWord','r',encoding='utf-8') as fin:
23     for i in fin.readlines():
24         stopWords.append(i.strip())
25
26 # 定義一些超引數
27 MAX_EMAIL_LENGTH = 200   #最長郵件長度
28 THRESHOLD = len(path_list_ham)/50   # 超過這麼多次的詞彙，入選dict
29
30 path_list_spam = path_list_spam[:5000]  # 將正例、負例樣本取子集，先各取5000個做實驗
31 path_list_ham = path_list_ham[:5000]
32 # 下方定義數值類字串檢驗函式 ，預處理時需要將數值資訊清洗掉
33 def is_number(s):
34     try:
35         float(s)
36         return True
37     except ValueError:
38         pass
39     try:
40         import unicodedata
41         unicodedata.numeric(s)
42         return True
43     except (TypeError, ValueError):
44         pass
45     return False

 1 def email_cut(path_list):
 2     emali_str_list = []
 3     for i in range(len(path_list)):
 4         print('====== ',i,' =======')
 5         print(path_list[i])
 6         with open(path_list[i],'r',encoding='gbk') as fin:
 7             words = []
 8             begin_tag = 0
 9             wrong_tag = 0
10             while(True):
11                 if wrong_tag > 20 or len(words)>MAX_EMAIL_LENGTH:
12                     break
13                 try:
14                     line = fin.readline()
15                     wrong_tag = 0
16                 except:
17                     wrong_tag += 1
18                     continue
19                 if (not line):
20                     break
21                 if(begin_tag == 0):
22                     if(line=='\n'):
23                         begin_tag = 1
24                     continue
25                 else:
26                     l = jieba.cut(line.strip())
27                     ll = list(l)
28                     for word in ll:
29                         if word not in stopWords and word != '\n' and word != '\t' and word != ' ' and not is_number(word):  #
30                             words.append(word)
31                         if len(words)>MAX_EMAIL_LENGTH:  # 一封email最大詞彙量設定
32                             break
33             wordStr = ' '.join(words)
34             emali_str_list.append(wordStr)
35     return emali_str_list

 1 def textToMatrix(text):
 2     cv = CountVectorizer()
 3     cv.fit(text)
 4     vocabulary = cv.vocabulary_
 5     vector = cv.transform(text)
 6     result = pd.DataFrame(vector.toarray())
 7     del(vector) # 及時刪除以節省記憶體空間
 8     features = []# 儲存特徵值
 9     for key, value in vocabulary.items():  # key, value 示例  '孔子'， 23772  即 詞彙,字串 的形式
10         if result[value].sum() >= THRESHOLD:
11             features.append(key) # 加入詞彙表
12             result.rename(columns={value:key}, inplace=True) # 本來的列名是索引值value，現在改成key ('孔子'、'後人'、'家鄉' ..等詞彙)
13     return result[features]  # 縮減特徵矩陣規模，僅將特徵詞彙表中的列留下

 1 ham_str_all = email_cut(path_list_ham)
 2 spam_str_all = email_cut(path_list_spam)
 3 allWord = []
 4 allWord.extend(ham_str_all)
 5 allWord.extend(spam_str_all)
 6 labels = []#標籤
 7 labels.extend(np.ones(len(path_list_ham)))
 8 labels.extend(np.zeros(len(path_list_spam)))
 9 vector = textToMatrix(allWord)#獲取特徵向量
10 print(vector)
11 feature = list(vector.columns)
12 print("feature length: ",len(feature))
13 with open('./model/CN_features.txt', 'w', encoding="UTF-8") as f:
14     s = ' '.join(feature)
15     f.write(s)
16 svm = SVC(kernel='linear', C=0.5, random_state=0)  # 線性核，C的值較小時可以允許一些錯誤 可選核： 'linear', 'poly', 'rbf', 'sigmoid', 'precomputed'
17 # 將資料分成測試集和訓練集
18 X_train, X_test, y_train, y_test = train_test_split(vector, labels, test_size=0.3, random_state=0)
19 svm.fit(X_train, y_train)
20 print(svm.score(X_test, y_test))
21 model = joblib.dump(svm,'./model/svm_model.m')

 1 def textToMatrix(text):
 2     cv = CountVectorizer()
 3     cv.fit(text)
 4     vocabulary = cv.vocabulary_
 5     vector = cv.transform(text)
 6     result = pd.DataFrame(vector.toarray())
 7     del(vector)
 8     features = []# 儲存特徵值
 9     origin_data = np.zeros((len(result),1)) # 新建的資料表
10     for key, value in vocabulary.items():
11         if result[value].sum() >= THRESHOLD:
12             features.append(key)
13             origin_data = np.column_stack((origin_data,np.array(result[value])))  # 按列堆疊到新資料表
14     origin_data = origin_data[:,1:]  # 刪掉初始化的第一列全0資料
15     print('origin_data shape: ',origin_data.shape)
16     origin_data = pd.DataFrame(origin_data)    # 轉換為DataFrame物件
17     origin_data.columns = features   # 批量修改列名
18     print('features length: ',len(features))
19     return origin_data