記錄防止忘記

包：

jsoup-1.4.1 html解析

httpcore-4.0.1_1

httpclient-4.0.1

程式碼：

已經訪問的url佇列

//已經訪問連結佇列
public class VisitedUrlQueue {
	public static HashSet<String> visitedUrlQueue = new HashSet<String>();  
	  
    public synchronized static void addElem(String url) {  
        visitedUrlQueue.add(url);  
    }  
  
    public synchronized static boolean isContains(String url) {  
        return visitedUrlQueue.contains(url);  
    }  
  
    public synchronized static int size() {  
        return visitedUrlQueue.size();  
    }  
}
 

//未訪問url佇列
public class UrlQueue {
	/** 超連結佇列 */  
    public static LinkedList<String> urlQueue = new LinkedList<String>();  
  
    /** 佇列中對應最多的超連結數量 */  
    public static final int MAX_SIZE = 10000;  
  
    public synchronized static void addElem(String url) {  
        urlQueue.add(url);  
    }  
  
    public synchronized static String outElem() {  
        return urlQueue.removeFirst();  
    }  
  
    public synchronized static boolean isEmpty() {  
        return urlQueue.isEmpty();  
    }  
  
    public static int size() {  
        return urlQueue.size();  
    }  
  
    public static boolean isContains(String url) {  
        return urlQueue.contains(url);  
    }  
}
 

public class DownloadPage {
	
	public static String getContentFormUrl(String url) throws Exception {  
		HttpClient client = new DefaultHttpClient();  
        HttpGet getHttp = new HttpGet(url);  
  
        String content = null;  
  
        HttpResponse response;  
        try {  
            /* 獲得資訊載體 */  
            response = client.execute(getHttp);  
            HttpEntity entity = response.getEntity();  
            
            //已經訪問url
            VisitedUrlQueue.addElem(url);  
  
            if (entity != null) {  
                /* 轉化為文字資訊 */  
                content = EntityUtils.toString(entity);  
            }  
  
        } catch (ClientProtocolException e) {  
            e.printStackTrace();  
        } catch (IOException e) {  
            e.printStackTrace();  
        } finally {  
            client.getConnectionManager().shutdown();  
        }  
  
        return content;  
    }  
	
}
 

public class ParseOfPage {
	 /** 
     * 獲得url頁面原始碼中超連結 
	 * @throws Exception 
     */  
    public static void getHrefOfContent(String content) throws Exception { 
    	Document doc = Jsoup.parse(content);
    	for(Element e:doc.getElementsByTag("a")){
    		String linkHref = e.attr("href");
    		if(linkHref.startsWith("/album")){ //進行連結篩選
    			linkHref = "http://www.kuwo.cn"+linkHref;//進行連結補充
    		}
    		if(linkHref.startsWith("http://www.kuwo.cn/album")){ //連結篩選，佇列判斷重複後加入佇列
    			if (!UrlQueue.isContains(linkHref)   
    	                 && !VisitedUrlQueue.isContains(linkHref)) { 
    				String urlNew = linkHref.replace(" ","%20");
    				//System.out.println(urlNew);
    	             UrlQueue.addElem(urlNew);  
    	        }
    		}
    		
    	}
    } 
  

    
    //進行自定義解析
    public static void getDataOfContentForSinger(String content) throws Exception {
    	SingerPo po = new SingerPo();
    	
    	Document doc = Jsoup.parse(content);
    	for(Element e:doc.getElementsByClass("artistTop")){
    		po.setPhotourl(e.childNode(1).attr("data-src")); //設定圖片
    	}
    	
    
    	

    	
    }
}