#!/usr/bin/python  
#-*- encoding:utf-8 -*-  
import jieba                                           #匯入jieba模組
import re 
jieba.load_userdict("newdict.txt")                     #載入自定義詞典  
import jieba.posseg as pseg 

def splitSentence(inputFile, outputFile):
    #把停用詞做成字典
    stopwords = {}
    fstop 
 
= open('stop_words.txt', 'r')
    for eachWord in fstop:
        stopwords[eachWord.strip().decode('utf-8', 'ignore')] = eachWord.strip().decode('utf-8', 'ignore')
    fstop.close()

    fin = open(inputFile, 'r')                                  #以讀的方式開啟檔案  
    fout = open(outputFile, 'w')                                #
 
以寫得方式開啟檔案  
    jieba.enable_parallel(4)                                    #並行分詞
    for eachLine in fin:
        line = eachLine.strip().decode('utf-8', 'ignore')       #去除每行首尾可能出現的空格，並轉為Unicode進行處理 
        line1 = re.sub("[0-9\s+\.\!\/_,$%^*()?;；:-【】+\"\']+|[+——！，;:。？、[email protected]#￥%……&*（）]+
 
".decode("utf8"), "".decode("utf8"),line)
        wordList = list(jieba.cut(line1))                        #用結巴分詞，對每行內容進行分詞  
        outStr = ''  
        for word in wordList:
            if word not in stopwords:  
                outStr += word  
                outStr += ' '  
        fout.write(outStr.strip().encode('utf-8') + '\n')       #將分詞好的結果寫入到輸出檔案
    fin.close()  
    fout.close()  
  
splitSentence('ss.txt', 'tt.txt')