Java SpiderThreadLocal类代码示例

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static void log(String message, Class<? extends SpiderBean> spiderBeanClass, Throwable cause) {
	boolean debug = SpiderThreadLocal.get().getEngine().isDebug();
	log.error(spiderBeanClass.getName() + " render error : " + message);
	if(debug && cause != null) {
		log.error(message, cause);
	}
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static void log(Field field, String message, Throwable cause) {
	boolean debug = SpiderThreadLocal.get().getEngine().isDebug();
	log.error(field.getName() + " render error : " + message);
	if(debug && cause != null) {
		log.error(message, cause);
	}
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public Elements $(String selector) {
	Elements elements = document.select(selector);
	if (SpiderThreadLocal.get().getEngine().isDebug()) {
		if (!selector.equalsIgnoreCase("script")) {
			// log.debug("["+selector+"]--->["+elements+"]");
			System.out.println("[" + selector + "]--->[" + elements + "]");
		}
	}
	return elements;
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static Proxys get() {
	Spider spider = SpiderThreadLocal.get();
	if(spider == null) {
		return null;
	}
	return spider.getEngine().getProxysLoader();
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static boolean isEnableProxy() {
	Spider spider = SpiderThreadLocal.get();
	if(spider == null) {
		return false;
	}
	return spider.getEngine().isProxy();
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public HttpResponse download(HttpRequest request, int timeout) throws DownloadException {
	try {
		URL url = new URL(request.getUrl());
		WebRequest webRequest = new WebRequest(url);
		webRequest.setHttpMethod(HttpMethod.GET);
		if(request instanceof HttpPostRequest) {//post
			HttpPostRequest post = (HttpPostRequest)request;
			webRequest.setHttpMethod(HttpMethod.POST);
			List<NameValuePair> requestParameters = new ArrayList<NameValuePair>();
			for(Map.Entry<String, Object> entry : post.getFields().entrySet()) {
				NameValuePair nvp = new NameValuePair(entry.getKey(), entry.getValue().toString());
				requestParameters.add(nvp);
			}
			webRequest.setRequestParameters(requestParameters);	
		}
		//header
		boolean isMobile = SpiderThreadLocal.get().getEngine().isMobile();
		webRequest.setAdditionalHeader("User-Agent", UserAgent.getUserAgent(isMobile));
		webRequest.setAdditionalHeaders(request.getHeaders());
		//proxy
		HttpHost proxy = Proxys.getProxy();
		if(proxy != null) {
			webRequest.setProxyHost(proxy.getHostName());
			webRequest.setProxyPort(proxy.getPort());
		}
		//timeout
		this.webClient.getOptions().setTimeout(timeout);
		//request,response
		webClient.getPage(webRequest);
		HtmlPage page = webClient.getPage(request.getUrl());
		HttpResponse resp = new HttpResponse();
		WebResponse webResponse = page.getWebResponse();
		int status = webResponse.getStatusCode();
		resp.setStatus(status);
		if(status == 302 || status == 301) {
			String redirectUrl = webResponse.getResponseHeaderValue("Location");
			resp.setContent(UrlUtils.relative2Absolute(request.getUrl(), redirectUrl));
		} else if(status == 200) {
			String content = page.asXml();
			resp.setContent(content);
			resp.setRaw(webResponse.getContentAsStream());
			String contentType = webResponse.getContentType();
			resp.setContentType(contentType);
			String charset = getCharset(request.getCharset(), contentType);
			resp.setCharset(charset);
		} else {
			throw new DownloadException("ERROR : " + status);
		}
		return resp;
	} catch(Exception ex) {
		throw new DownloadException(ex);
	}
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static Render getRender(RenderType type){
	return SpiderThreadLocal.get().getEngine().getSpiderBeanFactory().getRenderFactory().getRender(type);
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static Object getRet(){
    return SpiderThreadLocal.get().getEngine().getRet();
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static void setRet(Object o){
    SpiderThreadLocal.get().getEngine().setRet(o);
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public HttpClientDownloader() {
	
	cookieContext = HttpClientContext.create();
	cookieContext.setCookieStore(new BasicCookieStore());
	
	Registry<ConnectionSocketFactory> socketFactoryRegistry = null;
	try {
		//构造一个信任所有ssl证书的httpclient
		SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
			@Override
			public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
				return true;
			}
		}).build();
		SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
		socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
		           .register("http", PlainConnectionSocketFactory.getSocketFactory())  
		           .register("https", sslsf)  
		           .build();
	} catch(Exception ex) {
		socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
           .register("http", PlainConnectionSocketFactory.getSocketFactory())
           .register("https", SSLConnectionSocketFactory.getSocketFactory())
           .build();
	}
	RequestConfig clientConfig = RequestConfig.custom().setRedirectsEnabled(false).build();
	PoolingHttpClientConnectionManager syncConnectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
	syncConnectionManager.setMaxTotal(1000);
	syncConnectionManager.setDefaultMaxPerRoute(50);
	httpClient = HttpClientBuilder.create()
			.setDefaultRequestConfig(clientConfig)
			.setConnectionManager(syncConnectionManager)
			.setRetryHandler(new HttpRequestRetryHandler() {
				@Override
				public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
					int retryCount = SpiderThreadLocal.get().getEngine().getRetry();
					boolean retry = (executionCount <= retryCount);
					if(log.isDebugEnabled() && retry) {
						log.debug("retry : " + executionCount);
					}
					return retry;
				}
			}).build();
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static HttpResponse download(HttpRequest request) throws DownloadException {
	SpiderBeanContext context = SpiderThreadLocal.get().getSpiderBeanContext();
	return context.getDownloader().download(request, context.getTimeout());
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static HttpResponse defaultDownload(HttpRequest request) throws DownloadException {
	SpiderBeanContext context = SpiderThreadLocal.get().getSpiderBeanContext();
	Downloader downloader = SpiderThreadLocal.get().getEngine().getSpiderBeanFactory().getDownloaderFactory().defaultDownloader();
	return downloader.download(request, context.getTimeout());
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static void into(HttpRequest request) {
	SpiderThreadLocal.get().getSpiderScheduler().into(request);
}
 

import com.geccocrawler.gecco.spider.SpiderThreadLocal; //导入依赖的package包/类
public static void into(HttpRequest request) {
	SpiderThreadLocal.get().getEngine().getScheduler().into(request);
}