1. 程式人生 > >JAVA使用爬蟲代理的方案之一

JAVA使用爬蟲代理的方案之一

字符串 dem req soc protect .net mes authent 解析

Connection
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.URL;
import java.util.Random;

    class ProxyAuthenticator extends Authenticator {
        private String user, password;

        public ProxyAuthenticator(String user, String password) {
            this.user     = user;
            this.password = password;
        }

        protected PasswordAuthentication getPasswordAuthentication() {
            return new PasswordAuthentication(user, password.toCharArray());
        }
    }

    /**
     * 註意:下面代碼僅僅實現HTTP請求鏈接,每一次請求都是無狀態保留的,僅僅是這次請求是更換IP的,如果下次請求的IP地址會改變
     * 如果是多線程訪問的話,只要將下面的代碼嵌入到你自己的業務邏輯裏面,那麽每次都會用新的IP進行訪問,如果擔心IP有重復,
     * 自己可以維護IP的使用情況,並做校驗。
     */
    public class Demo {
        public static void main(String args[]) throws Exception {
            // 要訪問的目標頁面
            String targetUrl = "http://httpbin.org/ip";

            // 代理服務器
            String proxyServer = "t.16yun.cn";
            int proxyPort      = 31111;

            // 代理隧道驗證信息
            String proxyUser  = "username";
            String proxyPass  = "password";

            try {
                URL url = new URL(targetUrl);

                Authenticator.setDefault(new ProxyAuthenticator(proxyUser, proxyPass));

                // 創建代理服務器地址對象
                InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
                // 創建HTTP類型代理對象
                Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);

                // 設置通過代理訪問目標頁面
                HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
                // 設置Proxy-Tunnel
                // Random random = new Random();
                // int tunnel = random.nextInt(10000);
                // connection.setRequestProperty("Proxy-Tunnel",String.valueOf(tunnel));

                // 解析返回數據
                byte[] response = readStream(connection.getInputStream());

                System.out.println(new String(response));
            } catch (Exception e) {
                System.out.println(e.getLocalizedMessage());
            }
        }

        /**
         * 將輸入流轉換成字符串
         *
         * @param inStream
         * @return
         * @throws Exception
         */
        public static byte[] readStream(InputStream inStream) throws Exception {
            ByteArrayOutputStream outSteam = new ByteArrayOutputStream();
            byte[] buffer = new byte[1024];
            int len = -1;

            while ((len = inStream.read(buffer)) != -1) {
                outSteam.write(buffer, 0, len);
            }
            outSteam.close();
            inStream.close();

            return outSteam.toByteArray();
        }
    }

JAVA使用爬蟲代理的方案之一