tensorflow版本：

In[33]: tf.__version__
Out[33]:'1.2.1'

首先是資料獲取：

curl  -O    "http://thuctc.thunlp.org/source/THUCNews.zip"

抽取資料的python程式碼如下，原部落格寫了個shell指令碼，我用python寫了個：

import
 
 os
import glob
import shutil
import random
basepath="/Users/shuubiasahi/Desktop/THUCNews/"
newpath="/Users/shuubiasahi/Desktop/tensorflow/text/"
listpath=list(map(lambda  x:basepath+str(x)+"/",list(filter(lambda  x:not str(x).startswith("."),os.listdir(basepath)))))
def copy(listpath,MAXCOUNT=6500):
for  path in listpath
 
:
        newdir=newpath+ str(path).split("/")[-2]
        print(newdir)
if not os.path.exists(newdir):
            os.mkdir(newdir)
        files=glob.glob(path+"*.txt")
if len(files)<MAXCOUNT:
            resultlist=[]
for i in range(MAXCOUNT):
                resultlist.append(random.choice(files))
else:
 
            resultlist=random.sample(files,MAXCOUNT)
for file in resultlist:
            shutil.copy(file,newdir)
if  __name__=='__main__':
    copy(listpath)
    print("抽取成功")

把資料整合到一個檔案裡面去，格式如下：

標籤+“\t”+實際文字內容

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
將文字整合到 train、test、val 三個檔案中
"""
import  os
basepath="/Users/shuubiasahi/Desktop/tensorflow/text/"
trainpath="/Users/shuubiasahi/Desktop/tensorflow/train/"
def _read_file(filename):
    with open(filename,'r',encoding='utf-8') as f:
"""
           u3000中文全形下的空格
"""
return  f.read().replace('\n','').replace('\t','').replace('\u3000','')
def  save_file(dirname):
"""
將多個檔案整合並存到3個檔案中
        dirname:原資料目錄
檔案內容格式:類別\t內容
"""
    f_train=open(trainpath+"cnews.train.txt",'w',encoding='utf-8')
    f_test = open(trainpath +"cnews.test.txt",'w', encoding='utf-8')
    f_val = open(trainpath +"cnews.val.txt",'w', encoding='utf-8')
for category in os.listdir(dirname):
        catdir=os.path.join(dirname,category)
if not os.path.isdir(catdir):
continue
        files=os.listdir(catdir)
        print(len(files))
        count=0
for cur_file in files:
            filename=os.path.join(catdir,cur_file)
            content=_read_file(filename)
if count<5000:
                f_train.write(category+"\t"+content+"\n")
            elif count<6000:
                f_test.write(category+"\t"+content+"\n")
else:
                f_val.write(category +'\t'+ content +'\n')
            count+=1
        print("finish:",category)
    f_train.close()
    f_test.close()
    f_val.close()
if  __name__=='__main__':
    save_file(basepath)
    print(len(open(trainpath+"cnews.train.txt",'r', encoding='utf-8').readlines()))
    print(len(open(trainpath +"cnews.test.txt",'r', encoding='utf-8').readlines()))
    print(len(open(trainpath +"cnews.val.txt",'r', encoding='utf-8').readlines()))

預處理程式碼說明：

read_file()：讀取上一部分生成的資料檔案，將內容和標籤分開返回;
_build_vocab():構建詞彙表，這裡不需要對文件進行分詞，單字的效果已經很好，這一函式會將詞彙表儲存下來，避免每一次重複處理;
_read_vocab():讀取上一步儲存的詞彙表，轉換為{詞：id}表示;
_read_category():將分類目錄固定，轉換為{類別: id}表示;
_file_to_ids():基於上面定義的函式，將資料集從文字轉換為id表示;
to_words():將一條由id表示的資料重新轉換為文字;
preocess_file():一次性處理所有的資料並返回;
batch_iter():為神經網路的訓練準備批次的資料。

#!/usr/bin/python
# -*- coding: utf-8 -*-
from collections  importCounter
import tensorflow.contrib.keras as kr
import  numpy as  np
import os
trainpath="/Users/shuubiasahi/Desktop/tenso