Java HttpConnectionManagerParams.setSendBufferSize方法代码示例

import org.apache.commons.httpclient.params.HttpConnectionManagerParams; //导入方法依赖的package包/类
/**
 * Builds an HTTP client with the given settings. Settings are NOT reset to their default values after a client has
 * been created.
 * 
 * @return the created client.
 */
public HttpClient buildClient() {
    if (httpsProtocolSocketFactory != null) {
        Protocol.registerProtocol("https", new Protocol("https", httpsProtocolSocketFactory, 443));
    }

    HttpClientParams clientParams = new HttpClientParams();
    clientParams.setAuthenticationPreemptive(isPreemptiveAuthentication());
    clientParams.setContentCharset(getContentCharSet());
    clientParams.setParameter(HttpClientParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(
            connectionRetryAttempts, false));

    HttpConnectionManagerParams connMgrParams = new HttpConnectionManagerParams();
    connMgrParams.setConnectionTimeout(getConnectionTimeout());
    connMgrParams.setDefaultMaxConnectionsPerHost(getMaxConnectionsPerHost());
    connMgrParams.setMaxTotalConnections(getMaxTotalConnections());
    connMgrParams.setReceiveBufferSize(getReceiveBufferSize());
    connMgrParams.setSendBufferSize(getSendBufferSize());
    connMgrParams.setTcpNoDelay(isTcpNoDelay());

    MultiThreadedHttpConnectionManager connMgr = new MultiThreadedHttpConnectionManager();
    connMgr.setParams(connMgrParams);

    HttpClient httpClient = new HttpClient(clientParams, connMgr);

    if (proxyHost != null) {
        HostConfiguration hostConfig = new HostConfiguration();
        hostConfig.setProxy(proxyHost, proxyPort);
        httpClient.setHostConfiguration(hostConfig);

        if (proxyUsername != null) {
            AuthScope proxyAuthScope = new AuthScope(proxyHost, proxyPort);
            UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUsername,
                    proxyPassword);
            httpClient.getState().setProxyCredentials(proxyAuthScope, proxyCredentials);
        }
    }

    return httpClient;
}
 

import org.apache.commons.httpclient.params.HttpConnectionManagerParams; //导入方法依赖的package包/类
/**
 * Configures the HTTP client
 */
private void configureClient() {

  // Set up an HTTPS socket factory that accepts self-signed certs.
  // ProtocolSocketFactory factory = new SSLProtocolSocketFactory();
  ProtocolSocketFactory factory = new DummySSLProtocolSocketFactory();
  Protocol https = new Protocol("https", factory, 443);
  Protocol.registerProtocol("https", https);

  HttpConnectionManagerParams params = connectionManager.getParams();
  params.setConnectionTimeout(timeout);
  params.setSoTimeout(timeout);
  params.setSendBufferSize(BUFFER_SIZE);
  params.setReceiveBufferSize(BUFFER_SIZE);

  // --------------------------------------------------------------------------------
  // NUTCH-1836: Modification to increase the number of available connections
  // for multi-threaded crawls.
  // --------------------------------------------------------------------------------
  params.setMaxTotalConnections(conf.getInt(
      "mapred.tasktracker.map.tasks.maximum", 5)
      * conf.getInt("fetcher.threads.fetch", maxThreadsTotal));

  // Also set max connections per host to maxThreadsTotal since all threads
  // might be used to fetch from the same host - otherwise timeout errors can
  // occur
  params.setDefaultMaxConnectionsPerHost(conf.getInt(
      "fetcher.threads.fetch", maxThreadsTotal));

  // executeMethod(HttpMethod) seems to ignore the connection timeout on the
  // connection manager.
  // set it explicitly on the HttpClient.
  client.getParams().setConnectionManagerTimeout(timeout);

  HostConfiguration hostConf = client.getHostConfiguration();
  ArrayList<Header> headers = new ArrayList<Header>();
  // Set the User Agent in the header
  // headers.add(new Header("User-Agent", userAgent)); //NUTCH-1941
  // prefer English
  headers.add(new Header("Accept-Language", acceptLanguage));
  // prefer UTF-8
  headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
  // prefer understandable formats
  headers
      .add(new Header(
          "Accept",
          "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
  // accept gzipped content
  headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate"));
  hostConf.getParams().setParameter("http.default-headers", headers);

  // HTTP proxy server details
  if (useProxy) {
    hostConf.setProxy(proxyHost, proxyPort);

    if (proxyUsername.length() > 0) {

      AuthScope proxyAuthScope = getAuthScope(this.proxyHost, this.proxyPort,
          this.proxyRealm);

      NTCredentials proxyCredentials = new NTCredentials(this.proxyUsername,
          this.proxyPassword, Http.agentHost, this.proxyRealm);

      client.getState().setProxyCredentials(proxyAuthScope, proxyCredentials);
    }
  }

}
 

import org.apache.commons.httpclient.params.HttpConnectionManagerParams; //导入方法依赖的package包/类
/**
 * Configures the HTTP client
 */
private void configureClient() {

  // Set up an HTTPS socket factory that accepts self-signed certs.
  Protocol https = new Protocol("https",
      new DummySSLProtocolSocketFactory(), 443);
  Protocol.registerProtocol("https", https);

  HttpConnectionManagerParams params = connectionManager.getParams();
  params.setConnectionTimeout(timeout);
  params.setSoTimeout(timeout);
  params.setSendBufferSize(BUFFER_SIZE);
  params.setReceiveBufferSize(BUFFER_SIZE);
  params.setMaxTotalConnections(maxThreadsTotal);

  // executeMethod(HttpMethod) seems to ignore the connection timeout on the connection manager.
  // set it explicitly on the HttpClient.
  client.getParams().setConnectionManagerTimeout(timeout);

  HostConfiguration hostConf = client.getHostConfiguration();
  ArrayList headers = new ArrayList();
  // Set the User Agent in the header
  headers.add(new Header("User-Agent", userAgent));
  // prefer English
  headers.add(new Header("Accept-Language", acceptLanguage));
  // prefer UTF-8
  headers.add(new Header("Accept-Charset", "utf-8,ISO-8859-1;q=0.7,*;q=0.7"));
  // prefer understandable formats
  headers.add(new Header("Accept",
          "text/html,application/xml;q=0.9,application/xhtml+xml,text/xml;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"));
  // accept gzipped content
  headers.add(new Header("Accept-Encoding", "x-gzip, gzip, deflate"));
  hostConf.getParams().setParameter("http.default-headers", headers);

  // HTTP proxy server details
  if (useProxy) {
    hostConf.setProxy(proxyHost, proxyPort);

    if (proxyUsername.length() > 0) {

      AuthScope proxyAuthScope = getAuthScope(
          this.proxyHost, this.proxyPort, this.proxyRealm);

      NTCredentials proxyCredentials = new NTCredentials(
          this.proxyUsername, this.proxyPassword,
          this.agentHost, this.proxyRealm);

      client.getState().setProxyCredentials(
          proxyAuthScope, proxyCredentials);
    }
  }

}