由於資料抓取的需要，將網頁下載完後所有的都是html原始碼，需要抓取某一資訊時，需要對html做特定的分析，然後按照class或者id進行抓取。如果不瞭解抓取頁面的html標籤時，所得到的資訊是非常的難看，也很難從其中發現有用的資訊。也是現在我開始做頁面挖掘最大的難點。

以下程式碼是結合前面對html頁面空格處理以及特定抓取和儲存的程式碼:

下載頁面程式碼：

</pre><pre class="java" name="code">package com.dazhihui;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

public class MyJsoup {
	public static boolean downloadPage(String url, File file){
		try {
			Document doc = Jsoup.connect(url).data("jquery","java").userAgent("Mozilla").cookie("auth", "tiken").timeout(5000).get();
			String pageHtml = doc.toString();
			OutputStream out = new FileOutputStream(file);
			out.write(pageHtml.toString().getBytes());
			out.close();	
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		}
		return true;
	}
}
 

處理空格程式碼：

package com.dazhihui;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

public class ReplaceAllFileString {
	//引數說明：oldFile為所需要替換的檔案，即為原檔案；   newFile為替換後新的檔案 ；oldString為所需要替換的字串；newString為替換字串
	public static boolean replaceAllFileString(File oldFile, File newFile, String oldString, String newString){
		try {
		BufferedReader reader = new BufferedReader(new FileReader(oldFile));
		BufferedWriter writer = new BufferedWriter(new FileWriter(newFile));
		String teamString = null;
		while((teamString = reader.readLine()) != null){
			String str = teamString.replaceAll(oldString, newString);
			writer.write(str);
		}
		reader.close();
		writer.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return false;
		}
		return true;
	}
}
 

提取大智慧公司概況程式碼：

package com.dazhihui;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class DazhihuiResolveCompanyProfile {
	public static ArrayList<String> resolvePageText(File file){
		ArrayList<String> list=null;
		try {
			Document doc = Jsoup.parse(file, "GBK");
			Elements elements = doc.getElementsByClass("table_style_e");
			list = new ArrayList<String>();
			//select("table#table_style_e");
			for(Element element:elements){
				if(element.text()!=null&& !"".equals(element.text())){
					Elements es = element.select("tr");
					for(Element tdelement:es){
						Elements tdes = tdelement.select("td");
						for(int i = 0; i < tdes.size(); i++){
							list.add(tdes.get(i).text());
							//System.out.println(tdes.get(i).text());
						}
					}
				}
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}	
		return list;
	}
}
 

主程式碼：

package com.dazhihui;

import java.io.File;
import java.util.ArrayList;


public class Dazhihui {

	
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String url = "http://cj.gw.com.cn/news/stock/601288.shtml";
		File file = new File("C:/myjsoup/dazhihui/dazhihui.txt");
		File newFile = new File("C:/myjsoup/dazhihui/newdazhihui.txt");
		boolean mark =  MyJsoup.downloadPage(url, file);
		System.out.println(mark);
		boolean mark2 = ReplaceAllFileString.replaceAllFileString(file, newFile, " ", "");
		System.out.println(mark2);
		ArrayList<String> list =DazhihuiResolveCompanyProfile.resolvePageText(newFile);
		for(int i = 0; i < list.size(); i++){
			System.out.println(list.get(i));
		}
	}

}