apache HttpClient 4.3.4自动登录并抓取中国联通网页用户基本信息和账单数据

1.新建一个maven项目httpclient

2.登录中国联通并抓取数据

3.使用Get模拟登录,抓取每月账单数据

中国联通有两种登录方式:

上面两图的区别一个是带验证码,一个是不带验证码, 下面将先解决不带验证码的登录.

这里有两个难点,一是验证码,二uvc码;

验证码,这里将其写到本地,然后人工输入,这个还比较好解决.

uvc码,很重要,这个是在cookie里的,httpclient操作cookie的方法网上找了很久都没有找到,后来看其源码才看到.

package com.httpclient.asm.demo;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.*;
import org.apache.http.util.EntityUtils;

import javax.swing.*;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;

public class LoginChinaUnicomWithCaptcha {

    /**
     * 登录并抓取中国联通数据
     * 带验证码登录
     * @author Edson.di
     * @date 2015年3月4日
     * @version 1.0
     * @throws IOException
     */
        public static void main(String args[]) throws Exception {

            String name = "联通手机号";
            String pwd = "手机服务密码";

            //生成验证码的链接
            String createCaptchaUrl = "http://uac.10010.com/portal/Service/CreateImage";
            HttpClient httpClient = new DefaultHttpClient();

            //这里可自定义所需要的cookie
            CookieStore cookieStore = new BasicCookieStore();

            CloseableHttpClient httpclient = HttpClients.custom()
                    .setDefaultCookieStore(cookieStore)
                    .build();

            //get captcha,获取验证码
            HttpGet captchaHttpGet = new HttpGet(createCaptchaUrl);
            HttpResponse capthcaResponse = httpClient.execute(captchaHttpGet);

            if (capthcaResponse.getStatusLine().getStatusCode() == 200) {
                //将验证码写入本地
                saveToLocal(capthcaResponse.getEntity(), "chinaunicom.capthca." + System.currentTimeMillis()+".png");
            }


            //手工输入验证码并验证
            HttpResponse verifyResponse = null;
            String capthca = null;
            String uvc = null;

            do {
                //输入验证码,读入键盘输入
                //1)
//                InputStream inputStream = System.in;
//                BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
//                System.out.println("请输入验证码:");
//                capthca = bufferedReader.readLine();

                capthca=JOptionPane.showInputDialog("请输入图片验证码:");

                //2)
                //Scanner scanner = new Scanner(System.in);
                //capthca = scanner.next();
               // http://uac.10010.com/portal/Service/CtaIdyChk?callback=jsonp1404716227598&verifyCode=4m3e&verifyType=1
                String verifyCaptchaUrl = "http://uac.10010.com/portal/Service/CtaIdyChk?verifyCode=" + capthca + "&verifyType=1";
                HttpGet verifyCapthcaGet = new HttpGet(verifyCaptchaUrl);
                verifyResponse = httpClient.execute(verifyCapthcaGet);
                AbstractHttpClient abstractHttpClient = (AbstractHttpClient) httpClient;
                for (Cookie cookie : abstractHttpClient.getCookieStore().getCookies()) {
                    System.out.println(cookie.getName() + ":" + cookie.getValue());
                    if (cookie.getName().equals("uacverifykey")) {
                        uvc = cookie.getValue();
                    }
                }
            } while (!EntityUtils.toString(verifyResponse.getEntity()).contains("true"));

            //登录
            String loginurl = "https://uac.10010.com/portal/Service/MallLogin?userName=" + name + "&password=" + pwd + "&pwdType=01&productType=01&verifyCode=" + capthca + "&redirectType=03&uvc=" + uvc;
            HttpGet loginGet = new HttpGet(loginurl);
            CloseableHttpResponse loginResponse = httpclient.execute(loginGet);
            System.out.print("result:" + EntityUtils.toString(loginResponse.getEntity()));

            //抓取基本信息数据
            //jsonp1404663560635({resultCode:"7072",redirectURL:"http://www.10010.com",errDesc:"null",msg:'系统忙，请稍后再试。',needvode:"1"});
            HttpPost basicHttpGet = new HttpPost("http://iservice.10010.com/ehallService/static/acctBalance/execute/YH102010005/QUERY_AcctBalance.processData/Result");
            saveToLocal(httpclient.execute(basicHttpGet).getEntity(), "chinaunicom.basic.html");

        }
        /**
         * 写文件到本地
         * 
         * @param httpEntity
         * @param filename
         */
        public static void saveToLocal(HttpEntity httpEntity, String filename) {
        
            try {
        
                File dir = new File("/JEE/sz-588/workspace/maven-httpclient-demo");
                if (!dir.isDirectory()) {
                    dir.mkdir();
                }
        
                File file = new File(dir.getAbsolutePath() + "/" + filename);
                FileOutputStream fileOutputStream = new FileOutputStream(file);
                InputStream inputStream = httpEntity.getContent();
        
                if (!file.exists()) {
                    file.createNewFile();
                }
                byte[] bytes = new byte[1024];
                int length = 0;
                while ((length = inputStream.read(bytes)) > 0) {
                    fileOutputStream.write(bytes, 0, length);
                }
                inputStream.close();
                fileOutputStream.close();
            } catch (Exception e) {
                e.printStackTrace();
            }

   }
}

生成文件

json格式输出