1. 程式人生 > >使用正則表示式抓取網易雲課堂中的資料

使用正則表示式抓取網易雲課堂中的資料

 要抓取資料的頁面如下:


程式碼:

package com.url;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class PaChong {
	static Vector<String> url1 = new Vector<>();
    public static void FindOne(URL url) throws Exception{
      URLConnection Conn = url.openConnection();
      Conn.setReadTimeout(10000);
         
      BufferedReader read = new BufferedReader(new InputStreamReader(Conn.getInputStream(),"UTF-8"));
      String line = "";
      while((line = read.readLine())!=null){
    	  int index = line.indexOf("about/aboutus.htm#/about?"); //擷取URL搜尋到的網頁原始碼中的包含該欄位的原始碼
    	  if(index>=0){
    		  String URL ="http://study.163.com/"+line.substring(index);
              try {
				URL = URL.substring(0, URL.indexOf("\""));
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
              url1.add(URL);
    		  System.out.println(URL);
    	  }
       } 
   }
    
    
    public static void FindTitle(URL url) throws Exception{
        URLConnection Conn = url.openConnection();
        Conn.setReadTimeout(10000);
           
        BufferedReader read = new BufferedReader(new InputStreamReader(Conn.getInputStream(),"UTF-8"));
        String line = "";
        while((line = read.readLine())!=null){
      	  int Titleindex = line.indexOf("<title>");
      	  if(Titleindex>=0){
      		  System.out.println(PaChong.getChinese(line));
         } 
        }
     }
    
    public static void FindContent(URL url) throws Exception{
        URLConnection Conn = url.openConnection();
        Conn.setReadTimeout(10000);
           
        BufferedReader read = new BufferedReader(new InputStreamReader(Conn.getInputStream(),"UTF-8"));
        String line = "";
        while((line = read.readLine())!=null){
//          int Contentindex = line.indexOf("<a data-index=");
          int Contentindex = line.indexOf("data-name=");
      	  if(Contentindex>=0){
   		  String content = line.substring(line.indexOf("\""));
      	  System.out.println(PaChong.getChinese(content));
         } 
        }
     }
    
    //正則表示式提取搜尋到網頁中需要的中文字元
    public static String getChinese(String paramValue) {
    	String regex = "([\u4e00-\u9fa5]+)";
    	String str = "";
    	Matcher matcher = Pattern.compile(regex).matcher(paramValue);
    	while (matcher.find()) {
    		str+= matcher.group(0);
    		str+= "  ";
    	}
    	return str;
    }
	public static void main(String[] args) throws Exception {
		// TODO Auto-generated method stub
       URL url = new URL("http://study.163.com/courses-search?keyword=JAVA");  //爬取的連結
       System.err.println("提取的相關介紹網頁如下:");
       FindOne(url);
       System.err.println("提取的網頁標題如下:");
       FindTitle(url);
       System.err.println("提取的網頁內容如下:");
       FindContent(url);
	}

}

截圖: