package com.search.impl;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.search.entitys.Message;
import com.search.interfaces.ISpider;

/**
 * @Description: TODO(爬虫实现类)
 * @Date: 2017年1月15日
 * @author: luoshao
 * @Copyright (c) 2017, www.panyixia.cn , 792435323@qq.com All Rights Reserved.
 */
public class Spider implements ISpider {

	protected String crawlSource(String url) throws ClientProtocolException,
			IOException {
		CloseableHttpClient httpClient = HttpClientBuilder.create().build();
		CloseableHttpResponse httpResponse = httpClient.execute(new HttpGet(url));
		String result = EntityUtils.toString(httpResponse.getEntity());
		return result;
	}

	@Override
	public List<Message> getList(String url) {
		String result = null;
		List<Message> list = new ArrayList<Message>();
		Message msg = null;
		try {
			result = crawlSource(url);
			Document doc = Jsoup.parse(result);
			Elements es = doc.select(".b_algo h2 a");

			for (Element e : es) {
				msg = new Message();
				msg.setTitle(e.html());
				msg.setUrl(e.attr("href"));
				msg.setContent(e.parent().parent().select(".b_caption").html());
				list.add(msg);
			}

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return list;
	}

}