Python深度學習筆記(五)：使用NLTK進行自然語言處理
摘要：
安裝NLTK
pip install nltk
安裝NLTK包
import nltk
nltk.download()
#跳出GUI介面，下載需要的資料
計算單字頻率並繪圖
from bs4 import BeautifulSoupimport urllib.request...
安裝NLTK
pip install nltk
安裝NLTK包
import nltk nltk.download() #跳出GUI介面，下載需要的資料
計算單字頻率並繪圖
from bs4 import BeautifulSoup
import urllib.request
import nltk
response = urllib.request.urlopen('http://php.net/')
html = response.read()
soup = BeautifulSoup(html,"html5lib")
text = soup.get_text(strip=True)
tokens = [t for t in text.split()]
freq = nltk.FreqDist(tokens)
for key,val in freq.items():
print (str(key) + ':' + str(val))
<strong>freq.plot(20, cumulative=False)</strong>
移除停用詞Stop Words
停用詞大致分為兩類。 1)人類語言中包含的功能詞，如'the'、'is'、'at'、'which'、'on'等。 2)詞彙詞，比如'want'等，這些詞應用十分廣泛，但是對這樣的詞搜尋引擎無法保證能夠給出真正相關的搜尋結果。
<strong>#stopwords必須使用nltk.download()下載</strong>
from bs4 import BeautifulSoup
import urllib.request
import nltk
from nltk.corpus import stopwords
response = urllib.request.urlopen('http://php.net/')
html = response.read()
soup = BeautifulSoup(html,"html5lib")
text = soup.get_text(strip=True)
tokens = [t for t in text.split()]
clean_tokens = tokens[:]
sr = stopwords.words('english')
for token in tokens:
if token in stopwords.words('english'):
clean_tokens.remove(token)
freq = nltk.FreqDist(clean_tokens)
for key,val in freq.items():
print (str(key) + ':' + str(val))
freq.plot(20,cumulative=False)
<strong>可使用1984這本小說來分析看看，哪些是高頻率單字</strong>
如下連結
<a href="http://gutenberg.net.au/ebooks01/0100021.txt" rel="nofollow noopener noreferrer" target="_blank">http://gutenberg.net.au/ebooks01/0100021.txt</a>
分開英文句子
from nltk.tokenize import <strong>sent_tokenize</strong>
mytext = "Hello Adam, how are you? I hope everything is going well. Today is a good day, see you dude."
print(sent_tokenize(mytext))
<strong>['Hello Adam, how are you?', 'I hope everything is going well.', 'Today is a good day, see you dude.']</strong>
分開英文單字
from nltk.tokenize import <strong>word_tokenize</strong>
mytext = "Hello Mr. Adam, how are you? I hope everything is going well. Today is a good day, see you dude."
print(word_tokenize(mytext))
<strong>['Hello', 'Mr.', 'Adam', ',', 'how', 'are', 'you', '?', 'I', 'hope', 'everything', 'is', 'going', 'well', '.', 'Today', 'is', 'a', 'good', 'day', ',', 'see', 'you', 'dude', '.']</strong>
分開非英文文字
from nltk.tokenize import sent_tokenize
mytext = "Bonjour M. Adam, comment allez-vous? J'espère que tout va bien. Aujourd'hui est un bon jour."
print(sent_tokenize(mytext,"<strong>french</strong>"))
['Bonjour M. Adam, comment allez-vous?', "J'espère que tout va bien.", "Aujourd'hui est un bon jour."]
使用WordNet顯示同義/反義詞
<strong>#wordnet必須使用nltk.download()下載</strong>
from nltk.corpus import wordnet
syn = wordnet.synsets("pain")
print(syn[0].definition())
print(syn[0].examples())
<strong>#用For迴圈取得大量相關代名詞</strong>
from nltk.corpus import wordnet
synonyms = []
for syn in wordnet.synsets('Computer'):
for lemma in syn.lemmas():
synonyms.append(lemma.name())
print(synonyms)
#用For迴圈取得大量反義詞
from nltk.corpus import wordnet
antonyms = []
for syn in wordnet.synsets("small"):
for l in syn.lemmas():
if l.antonyms():
antonyms.append(l.antonyms()[0].name())
print(antonyms)
去除字尾
from nltk.stem import PorterStemmer
stemmer = PorterStemmer()
print(stemmer.stem('working'))
#顯示work -----------------------------------------------
<strong>支援去除以下語言的字尾</strong>
from nltk.stem import SnowballStemmer
print(SnowballStemmer.languages)
<strong>('arabic', 'danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', 'italian', 'norwegian', 'porter', 'portuguese', 'romanian', 'russian', 'spanish', 'swedish')</strong>
------------------------------------------------
去除德文字尾
from nltk.stem import SnowballStemmer
french_stemmer = SnowballStemmer('german')
print(french_stemmer.stem("Guten"))
更精確的去除字尾
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
print(lemmatizer.lemmatize('playing', pos="v"))
print(lemmatizer.lemmatize('playing', pos="n"))
print(lemmatizer.lemmatize('playing', pos="a"))
print(lemmatizer.lemmatize('playing', pos="r"))