Python HTTPConnectionPool.maxPersistentPerHost方法代码示例

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
    def requestAvatarId(self, c):
        creds = credentials.IUsernamePassword(c, None)

        if creds is not None:
            locks = []
            pool = HTTPConnectionPool(reactor, persistent=False)
            pool.cachedConnectionTimeout = self.timeout
            if self.max_concurrency:
                pool.persistent = True
                pool.maxPersistentPerHost = self.max_concurrency
                locks.append(
                    defer.DeferredSemaphore(self.max_concurrency))

            if self.global_max_concurrency:
                locks.append(
                    defer.DeferredSemaphore(self.global_max_concurrency))

            conn = ThrottledSwiftConnection(
                locks, self.auth_url, creds.username, creds.password,
                pool=pool,
                extra_headers=self.extra_headers,
                verbose=self.verbose,
                ceph_compatible=self.ceph_compatible
            )
            conn.user_agent = USER_AGENT

            d = conn.authenticate()
            d.addCallback(self._after_auth, conn)
            d.addErrback(eb_failed_auth)
            return d
        return defer.fail(error.UnauthorizedLogin())

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
def makeService(options):
    """
    Makes a new swftp-ftp service. The only option is the config file
    location. The config file has the following options:
     - host
     - port
     - auth_url
     - num_persistent_connections
     - connection_timeout
     - welcome_message
    """
    from twisted.protocols.ftp import FTPFactory
    from twisted.web.client import HTTPConnectionPool
    from twisted.cred.portal import Portal

    from swftp.ftp.server import SwiftFTPRealm
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import print_runtime_info

    print('Starting SwFTP-ftp %s' % VERSION)

    c = get_config(options['config_file'], options)
    ftp_service = service.MultiService()

    # Add statsd service
    if c.get('ftp', 'log_statsd_host'):
        try:
            from swftp.statsd import makeService as makeStatsdService
            makeStatsdService(
                c.get('ftp', 'log_statsd_host'),
                c.getint('ftp', 'log_statsd_port'),
                sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'),
                prefix=c.get('ftp', 'log_statsd_metric_prefix')
            ).setServiceParent(ftp_service)
        except ImportError:
            log.err('Missing Statsd Module. Requires "txstatsd"')

    pool = HTTPConnectionPool(reactor, persistent=True)
    pool.maxPersistentPerHost = c.getint('ftp', 'num_persistent_connections')
    pool.cachedConnectionTimeout = c.getint('ftp', 'connection_timeout')

    authdb = SwiftBasedAuthDB(auth_url=c.get('ftp', 'auth_url'),
                              verbose=c.getboolean('ftp', 'verbose'))

    ftpportal = Portal(SwiftFTPRealm())
    ftpportal.registerChecker(authdb)
    ftpfactory = FTPFactory(ftpportal)
    ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message')
    ftpfactory.allowAnonymous = False

    signal.signal(signal.SIGUSR1, print_runtime_info)
    signal.signal(signal.SIGUSR2, print_runtime_info)

    internet.TCPServer(
        c.getint('ftp', 'port'),
        ftpfactory,
        interface=c.get('ftp', 'host')).setServiceParent(ftp_service)
    return ftp_service

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
    def _getConnectionPool(self):
        pool = HTTPConnectionPool(reactor, self._persistent)

        if self._persistent:
            pool.maxPersistentPerHost = self._maxPersistentPerHost
            pool.cachedConnectionTimeout = self._cachedConnectionTimeout
            pool.retryAutomatically = self._retryAutomatically

        return pool

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
 def __init__(self, hs):
     self.hs = hs
     self.signing_key = hs.config.signing_key[0]
     self.server_name = hs.hostname
     pool = HTTPConnectionPool(reactor)
     pool.maxPersistentPerHost = 10
     self.agent = MatrixFederationHttpAgent(reactor, pool=pool)
     self.clock = hs.get_clock()
     self.version_string = hs.version_string

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
 def __init__(self, hs):
     self.hs = hs
     # The default context factory in Twisted 14.0.0 (which we require) is
     # BrowserLikePolicyForHTTPS which will do regular cert validation
     # 'like a browser'
     pool = HTTPConnectionPool(reactor)
     pool.maxPersistentPerHost = 10
     self.agent = Agent(reactor, pool=pool)
     self.version_string = hs.version_string

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
 def __init__(self, hs):
     self.hs = hs
     self.signing_key = hs.config.signing_key[0]
     self.server_name = hs.hostname
     pool = HTTPConnectionPool(reactor)
     pool.maxPersistentPerHost = 10
     self.agent = Agent.usingEndpointFactory(reactor, MatrixFederationEndpointFactory(hs), pool=pool)
     self.clock = hs.get_clock()
     self.version_string = hs.version_string
     self._next_id = 1

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
def _get_agent():
    context_factory = MyWebClientContextFactory()
    try:
        # HTTPConnectionPool has been present since Twisted version 12.1
        from twisted.web.client import HTTPConnectionPool
        pool = HTTPConnectionPool(reactor, persistent=True)
        pool.maxPersistentPerHost = _MAX_PERSISTENT_PER_HOST
        pool.cachedConnectionTimeout = _CACHED_CONNECTION_TIMEOUT
        agent = Agent(reactor, context_factory,
                       connectTimeout=_CONNECT_TIMEOUT, pool=pool)
    except ImportError:
        from _zenclient import ZenAgent
        agent = ZenAgent(reactor, context_factory, persistent=True, maxConnectionsPerHostName=1)
    return agent

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
    def __init__(self, reactor, node=('localhost', 4001), ca=None, cert=None):
        self.reactor = reactor
        self.node = node
        self.scheme = 'http'
        self.ca = ca
        self.cert = cert
        context = None
        if ca:
            self.scheme = 'https'
            context = PolicyForHTTPS(ca, cert)

        quietPool = HTTPConnectionPool(reactor, persistent = True)
        quietPool.maxPersistentPerHost = 2
        quietPool._factory = QuietHTTP11ClientFactory

        self.agent = Agent(self.reactor, contextFactory=context, pool=quietPool)

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
def _get_agent():
    global _AGENT
    if _AGENT is None:
        try:
            # HTTPConnectionPool has been present since Twisted version 12.1
            from twisted.web.client import HTTPConnectionPool
            pool = HTTPConnectionPool(reactor, persistent=True)
            pool.maxPersistentPerHost = _MAX_PERSISTENT_PER_HOST
            pool.cachedConnectionTimeout = _CACHED_CONNECTION_TIMEOUT
            _AGENT = Agent(
                reactor, connectTimeout=_CONNECT_TIMEOUT, pool=pool)
        except ImportError:
            try:
                # connectTimeout first showed up in Twisted version 11.1
                _AGENT = Agent(reactor, connectTimeout=_CONNECT_TIMEOUT)
            except TypeError:
                _AGENT = Agent(reactor)
    return _AGENT

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
    def __init__(self, reactor):
        self.reactor = reactor
        pool = HTTPConnectionPool(reactor, persistent=True)
        pool.maxPersistentPerHost = 1
        pool.cachedConnectionTimeout = 600
        self.agent = RedirectAgent(Agent(reactor, pool=pool))
        self.reqQ = HttpReqQ(self.agent, self.reactor)
        self.clientPlaylist = HlsPlaylist()
        self.verbose = False
        self.download = False
        self.outDir = ""
        self.encryptionHandled=False

        # required for the dump durations functionality
        self.dur_dump_file = None
        self.dur_avproble_acc = 0
        self.dur_vt_acc = 0
        self.dur_playlist_acc = 0

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
	def parallelFetchAllProblems(self):
			pool = HTTPConnectionPool(reactor)
			pool.maxPersistentPerHost = self.concurrency
			agent = Agent(reactor, pool=pool)
			sem = DeferredSemaphore(self.concurrency)
			self.done = 0

			def assign():
				self.query.execute('BEGIN')
				for id in range(1001, self.getProblemMax()+1):
					sem.acquire().addCallback(requestFactory, id)
			
			def requestFactory(token, id):
				deferred = agent.request('GET', self.baseUrl + self.problemPath + str(id))
				deferred.addCallback(onHeader, id)
				deferred.addErrback(errorHandler, id)
				return deferred
			
			def onHeader(response, id):
				deferred = readBody(response)
				deferred.addCallback(onBody, id)
				deferred.addErrback(errorHandler, id)
				return deferred

			def onBody(html, id):
				sem.release()
				d = pyq(html)
				title = d('#content_body > center:nth-child(1) > span').text(),
				body = d('#content_body').text()
				print('Fetched ProblemID: %s, Title: %s, done: %s' % (id, title[0], self.done))
				self.storeProblem(id, title[0], body)
				self.done += 1
				if(self.done == self.problemCount):
					print('Fetch data used %s s' % (reactor.seconds() - startTimeStamp))
					print('Fetch data end, writing to database')
					self.query.execute('COMMIT')
					reactor.stop()

			def errorHandler(err, id):
				print('[%s] id %s: %s' % (reactor.seconds() - startTimeStamp, id, err))

			startTimeStamp = reactor.seconds()
			reactor.callWhenRunning(assign)
			reactor.run()

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
    def __init__(self, hs):
        self.hs = hs
        self.signing_key = hs.config.signing_key[0]
        self.server_name = hs.hostname
        reactor = hs.get_reactor()
        pool = HTTPConnectionPool(reactor)
        pool.retryAutomatically = False
        pool.maxPersistentPerHost = 5
        pool.cachedConnectionTimeout = 2 * 60
        self.agent = Agent.usingEndpointFactory(
            reactor, MatrixFederationEndpointFactory(hs), pool=pool
        )
        self.clock = hs.get_clock()
        self._store = hs.get_datastore()
        self.version_string_bytes = hs.version_string.encode('ascii')
        self.default_timeout = 60

        def schedule(x):
            reactor.callLater(_EPSILON, x)

        self._cooperator = Cooperator(scheduler=schedule)

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
    def agent(self, scheme='http'):
        if not self._agents:
            pool = HTTPConnectionPool(reactor)
            pool.maxPersistentPerHost = 10
            pool.cachedConnectionTimeout = 15

            contextFactory = PermissiveBrowserLikePolicyForHTTPS()
            proxies = getproxies()

            if 'http' in proxies or 'https' in proxies:
                # I've noticed some intermittent failures (ResponseNeverReceived) to
                # POST request through a proxy when persistent connections are enabled.
                pool.persistent = False

            if 'https' in proxies:
                proxy = urlparse(proxies.get('https'))
                if proxy:
                    # Note- this isn't going to work completely.  It's not being
                    # passed the modified contextFactory, and in fact it doesn't
                    # even work properly for other reasons (ZPS-2061)
                    log.info("Creating https proxy (%s:%s)" % (proxy.hostname, proxy.port))
                    endpoint = TCP4ClientEndpoint(reactor, proxy.hostname, proxy.port, timeout=CONNECT_TIMEOUT)
                    SessionManager._agents['https'] = \
                        ProxyAgent(endpoint, reactor, pool=pool)
            else:
                SessionManager._agents['https'] = \
                    Agent(reactor, pool=pool, connectTimeout=CONNECT_TIMEOUT, contextFactory=contextFactory)

            if 'http' in proxies:
                proxy = urlparse(proxies.get('http'))
                if proxy:
                    log.info("Creating http proxy (%s:%s)" % (proxy.hostname, proxy.port))
                    endpoint = TCP4ClientEndpoint(reactor, proxy.hostname, proxy.port, timeout=CONNECT_TIMEOUT)
                    SessionManager._agents['http'] = \
                        ProxyAgent(endpoint, reactor, pool=pool)
            else:
                SessionManager._agents['http'] = \
                    Agent(reactor, pool=pool, connectTimeout=CONNECT_TIMEOUT)

        return SessionManager._agents[scheme]

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
    def __init__(self, page_archiver, cookie_file=None):
        self._logger = logging.getLogger(__name__)
        self._page_archiver = page_archiver
        self._logger.debug('Using page archiver: %s. Cookie file: %s',
                           page_archiver is not None,
                           cookie_file)
        if cookie_file:
            umask = os.umask(077)
            self._cj = LWPCookieJar(cookie_file)
            try:
                self._cj.load()
            except LoadError:
                self._logger.warning('Cannot load cookies from %s' % (cookie_file, ))
            os.umask(umask)
        else:
            self._cj = CookieJar()

        pool = HTTPConnectionPool(reactor, persistent=True)
        pool.maxPersistentPerHost = 10
        self._agent = CookieAgent(ContentDecoderAgent(Agent(reactor, pool=pool),
                                                       [('gzip', GzipDecoder)]), self._cj)
        self._lock = Lock()

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import maxPersistentPerHost [as 别名]
from twisted.python import util, log
from twisted.web.http_headers import Headers

from smap import core
from smap.util import periodicSequentialCall, BufferProtocol
import smap.driver
import smap.contrib.dtutil as dtutil

TIMEFMT = "%Y-%m-%d %H:%M:%S"

# make a connection pool
try:
    connection_pool
except NameError:
    connection_pool = HTTPConnectionPool(reactor, persistent=True)
    connection_pool.maxPersistentPerHost = 3


def make_field_idxs(type, header, location=None):
    paths = [None]
    map_ = sensordb.get_map(type, header=header, location=location)
    for t in header[1:]:
        paths.append(None)
        for channel in map_['sensors'] + map_['meters']:
            if t.strip().startswith(channel[0]):
                paths[-1] = (channel[2], channel[3])
                break
    ddups = {}
    for elt in paths:
        if elt:
            name = '-'.join(elt)