Java HttpGet.setConfig方法代码示例

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
/**
 * get a redirect for an url: this method shall be called if it is expected that a url
 * is redirected to another url. This method then discovers the redirect.
 * @param urlstring
 * @param useAuthentication
 * @return the redirect url for the given urlstring
 * @throws IOException if the url is not redirected
 */
public static String getRedirect(String urlstring) throws IOException {
    HttpGet get = new HttpGet(urlstring);
    get.setConfig(RequestConfig.custom().setRedirectsEnabled(false).build());
    get.setHeader("User-Agent", ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName).userAgent);
    CloseableHttpClient httpClient = getClosableHttpClient();
    HttpResponse httpResponse = httpClient.execute(get);
    HttpEntity httpEntity = httpResponse.getEntity();
    if (httpEntity != null) {
        if (httpResponse.getStatusLine().getStatusCode() == 301) {
            for (Header header: httpResponse.getAllHeaders()) {
                if (header.getName().equalsIgnoreCase("location")) {
                    EntityUtils.consumeQuietly(httpEntity);
                    return header.getValue();
                }
            }
            EntityUtils.consumeQuietly(httpEntity);
            throw new IOException("redirect for  " + urlstring+ ": no location attribute found");
        } else {
            EntityUtils.consumeQuietly(httpEntity);
            throw new IOException("no redirect for  " + urlstring+ " fail: " + httpResponse.getStatusLine().getStatusCode() + ": " + httpResponse.getStatusLine().getReasonPhrase());
        }
    } else {
        throw new IOException("client connection to " + urlstring + " fail: no connection");
    }
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
@Override
public boolean checkProxy(Proxy proxy){
    HttpGet httpGet = new HttpGet("http://icanhazip.com/");
    CloseableHttpResponse response = null;
    RequestConfig requestConfig = RequestConfig.custom().setProxy(new HttpHost(proxy.getIp(),proxy.getPort())).build();
    httpGet.setConfig(requestConfig);
    httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36");
    try {
        response = this.httpClient.execute(httpGet);
        String content = EntityUtils.toString(response.getEntity(), Charset.forName("UTF-8"));
        response.close();
        return proxy.getIp().equals(content.trim());
    } catch (IOException e) {
        e.printStackTrace();
        return false;
    }

}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
public void checkProxys(){
    while(true){
        Proxy proxy = this.poplProxy();
        HttpHost host = new HttpHost(proxy.getIp(),proxy.getPort());
        RequestConfig config = RequestConfig.custom().setProxy(host).build();
        HttpGet httpGet = new HttpGet(PROXY_TEST_URL);
        httpGet.setConfig(config);
        try {
            CloseableHttpResponse response = this.httpClient.execute(httpGet);
            String content = EntityUtils.toString(response.getEntity(), Charset.forName("UTF-8"));
            if(content!=null&&content.trim().equals(proxy.getIp()))
                this.pushrProxy(proxy);
        } catch (IOException e) {
        }
    }
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private Map<String, RouteConfig> getRouterMap() {
	RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.DEFAULT_TIMEOUT)
			.setConnectionRequestTimeout(Constants.DEFAULT_TIMEOUT).setSocketTimeout(Constants.DEFAULT_TIMEOUT)
			.build();
	HttpClient httpClient = this.httpPool.getResource();
	try {
		StringBuilder sb = new StringBuilder(50);
		sb.append(this.getServerDesc().getRegistry())
				.append(this.getServerDesc().getRegistry().endsWith("/") ? "" : "/")
				.append(this.getServerDesc().getServerApp()).append("/routers");
		HttpGet get = new HttpGet(sb.toString());
		get.setConfig(requestConfig);
		// 创建参数队列
		HttpResponse response = httpClient.execute(get);
		HttpEntity entity = response.getEntity();
		String body = EntityUtils.toString(entity, "UTF-8");
		ObjectMapper mapper = JsonMapperUtil.getJsonMapper();
		return mapper.readValue(body, Map.class);
	} catch (Exception e) {
		logger.error(e.getMessage(), e);
		throw new RuntimeException(e);
	} finally {
		this.httpPool.release(httpClient);
	}
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
private String getEntity(URI url) throws IOException {
    final HttpGet get = new HttpGet(url);
    get.setConfig(requestConfig);
    get.setHeader("Accept", "application/json");

    HttpClientBuilder clientBuilder = HttpClients.custom();
    if (sslContext != null) {
        clientBuilder.setSslcontext(sslContext);
    }

    try (CloseableHttpClient httpClient = clientBuilder.build()) {

        try (CloseableHttpResponse response = httpClient.execute(get)) {

            final StatusLine statusLine = response.getStatusLine();
            final int statusCode = statusLine.getStatusCode();
            if (200 != statusCode) {
                final String msg = String.format("Failed to get entity from %s, response=%d:%s",
                        get.getURI(), statusCode, statusLine.getReasonPhrase());
                throw new RuntimeException(msg);
            }
            final HttpEntity entity = response.getEntity();
            return EntityUtils.toString(entity);
        }
    }
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
/**
 * HttpClient get方法请求返回Entity
 */
private static byte[] getMethodGetContent(String address,
                                          RequestConfig config) throws Exception {
    HttpGet get = new HttpGet(address);
    try {
        get.setConfig(config);
        HttpResponse response = CLIENT.execute(get);
        if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
            int code = response.getStatusLine().getStatusCode();
            throw new RuntimeException("HttpGet Access Fail , Return Code("
                    + code + ")");
        }
        response.getEntity().getContent();
        return convertEntityToBytes(response.getEntity());
    } finally {
        if (get != null) {
            get.releaseConnection();
        }
    }
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
@Override
public String execute(CloseableHttpClient httpclient, HttpHost httpProxy, String uri, String queryParam) throws WxErrorException, IOException {
  if (queryParam != null) {
    if (uri.indexOf('?') == -1) {
      uri += '?';
    }
    uri += uri.endsWith("?") ? queryParam : '&' + queryParam;
  }
  HttpGet httpGet = new HttpGet(uri);
  if (httpProxy != null) {
    RequestConfig config = RequestConfig.custom().setProxy(httpProxy).build();
    httpGet.setConfig(config);
  }

  try (CloseableHttpResponse response = httpclient.execute(httpGet)) {
    String responseContent = Utf8ResponseHandler.INSTANCE.handleResponse(response);
    WxError error = WxError.fromJson(responseContent);
    if (error.getErrorCode() != 0) {
      throw new WxErrorException(error);
    }
    return responseContent;
  } finally {
    httpGet.releaseConnection();
  }
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
/**
 * returns the location of a host.
 *
 * @param canonicalHostName
 *            the host name to use.
 * @return the location, as JSON.
 */
public static JSONObject getLocation(final String canonicalHostName) {
	try {
		final String urlStr = "https://freegeoip.net/json/" + canonicalHostName;
		final HttpGet get = new HttpGet(urlStr);
		final RequestConfig requestConfig = RequestConfig.custom().build();
		get.setConfig(requestConfig);
		final CloseableHttpClient client = HttpClients.createDefault();
		final CloseableHttpResponse response = client.execute(get);
		final HttpEntity entity = response.getEntity();
		final String str = EntityUtils.toString(entity);
		final JSONObject outputJson = new JSONObject(str);
		LOG.info("outputJson:{}", outputJson);
		return outputJson;
	} catch (final IOException e) {
		throw new RuntimeException(e);
	}
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
private void bindRequest(){
    setDefaultParameter();
    if(mHttpType == HttpType.GET){
        QuickURL quickURL = new QuickURL(mUrl,mParames);
        mUrl = quickURL.fullUrl();
        if(isDebug){
            log("GET "+mUrl);
        }
        mRequest = new HttpGet(mUrl);
    }else if(mHttpType == HttpType.POST){
        HttpPost httpPost= new HttpPost(mUrl);
        if(isDebug){
            log("POST "+mUrl);
        }
        setupEntity(httpPost);
        mRequest = httpPost;
    }
    mRequest.setConfig(getDefaultRequestConfig());
    setupHeaders();
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
/**
 * 对上一个方法的重载，使用本机ip进行网站爬取
 */

public static String getHtml(String url) throws ClassNotFoundException,
        IOException {
    String entity = null;
    CloseableHttpClient httpClient = HttpClients.createDefault();

    //设置超时处理
    RequestConfig config = RequestConfig.custom().setConnectTimeout(5000).
            setSocketTimeout(5000).build();
    HttpGet httpGet = new HttpGet(url);
    httpGet.setConfig(config);

    httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;" +
            "q=0.9,image/webp,*/*;q=0.8");
    httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch");
    httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
    httpGet.setHeader("Cache-Control", "no-cache");
    httpGet.setHeader("Connection", "keep-alive");
    httpGet.setHeader("Cookie", "_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJTRkYjMyM" +
            "TU3NGRjMWVhM2JlMDA5Y2IyNzZlZmVlZTYwBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMUhtT0pjcnRT" +
            "bm9CZEllSXNTYkNZZWk2Nnp3NGNDcFFSQVFodzk1dmpLZWM9BjsARg%3D%3D--09d8736fbfb9a8544" +
            "b46eef48bb320c2b40ee721; Hm_lvt_0cf76c77469e965d2957f0553e6ecf59=1492128157,149" +
            "2160558,1492347839,1492764281; Hm_lpvt_0cf76c77469e965d2957f0553e6ecf59=1492764295");
    httpGet.setHeader("Host", "www.xicidaili.com");
    httpGet.setHeader("Pragma", "no-cache");
    httpGet.setHeader("Upgrade-Insecure-Requests", "1");
    httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " +
            "(KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

    try {
        //客户端执行httpGet方法，返回响应
        CloseableHttpResponse httpResponse = httpClient.execute(httpGet);

        //得到服务响应状态码
        if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
            entity = EntityUtils.toString(httpResponse.getEntity(), "utf-8");
        }

        httpResponse.close();
        httpClient.close();
    } catch (ClientProtocolException e) {
        e.printStackTrace();
    }

    return entity;
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
public void makeGetRequest(String alias, String uri, Map<String, String> headers, Map<String, String> parameters,
		boolean allowRedirects) {
	logger.debug("Making GET request");
	HttpGet getRequest = new HttpGet(this.buildUrl(alias, uri, parameters));
	getRequest = this.setHeaders(getRequest, headers);
	getRequest.setConfig(RequestConfig.custom().setRedirectsEnabled(allowRedirects).build());
	Session session = this.getSession(alias);
	this.makeRequest(getRequest, session);
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
private HttpGet getHttpGet() {
	HttpGet httpGet = new HttpGet();
	if(config != null) {
		httpGet.setConfig(config);  
	}
	return httpGet;
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
private String executeGetRequest(String url) throws IOException {
    RequestConfig requestConfig = RequestConfig.custom()
            .setConnectTimeout(15000)
            .setSocketTimeout(15000)
            .build();
    HttpGet method = new HttpGet(url);
    method.setConfig(requestConfig);
    try (CloseableHttpClient client = HttpClients.createDefault()) {
        ResponseHandler<String> responseHandler = new BasicResponseHandler();
        return client.execute(method, responseHandler);
    }
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
/**
 * Returns the controller properties for {@code controller}.
 * @param controller the controller to contact.
 * @return the controller properties.
 * @throws IOException when properties cannot be read.
 */
public static ControllerProperties getInitialization(Controller controller) throws IOException {
  ControllerProperties props;
  String initResource = controller.getInitResource();
  if (HttpManager.isHttpUrl(initResource)) {
    CloseableHttpClient client = HttpClients.createDefault();
    HttpHost proxy = controller.getProxy(AppConfigurationService.getConfigurations().getProxy())
        .toHttpHost();
    HttpGet httpGet = new HttpGet(initResource);
    controller.getAuthentication(AppConfigurationService.getConfigurations().getAuthentication())
        .forEach(httpGet::setHeader);
    if (proxy != null) httpGet.setConfig(RequestConfig.custom().setProxy(proxy).build());
    try (CloseableHttpResponse response = client.execute(httpGet)) {
      HttpEntity entity = response.getEntity();
      try (InputStream in = entity.getContent()) {
        props = ControllerService.from(ControllerPropertiesFormat.JSON, in);
      }
      EntityUtils.consume(entity);
    } finally {
      client.close();
    }
  } else {
    try (InputStream in = IOManager.getInputStream(initResource)) {
      props = ControllerService.from(ControllerPropertiesFormat.JSON, in);
    }
  }
  return props;
}
 

import org.apache.http.client.methods.HttpGet; //导入方法依赖的package包/类
private HttpGet createHttpGet(String uri) {
	HttpGet httpGet = new HttpGet(uri);
	if (basicAuthorizationHeader != null) {
		httpGet.addHeader(basicAuthorizationHeader);
	}
	httpGet.addHeader(apikeyHeader);

	if (proxy != null) {
		RequestConfig requsetConfig = RequestConfig.custom().setProxy(proxy).build();
		httpGet.setConfig(requsetConfig);
	}

	return httpGet;
}