Python client._makeGetterFactory函数代码示例

def downloadFile(url, file, statusCallback=None, bucketFilter=None, contextFactory=None, *args, **kwargs):
    factoryFactory = lambda url, *a, **kw: HTTPManagedDownloader(url, file, statusCallback=statusCallback, bucketFilter=bucketFilter, *a, **kw)
    return _makeGetterFactory(
        url,
        factoryFactory,
        contextFactory=contextFactory,
        *args, **kwargs).deferred

def quiet_get_page(url, contextFactory=None, *args, **kwargs):
    """A version of getPage that uses QuietHTTPClientFactory."""
    return _makeGetterFactory(
        url,
        QuietHTTPClientFactory,
        contextFactory=contextFactory,
        *args, **kwargs).deferred

def getPageAndHeaders(url, contextFactory=None, *args, **kwargs):
    factory = client._makeGetterFactory(
        url,
        client.HTTPClientFactory,
        contextFactory=contextFactory,
        *args, **kwargs)
    return factory.deferred.addCallback(lambda page: (page, factory.response_headers))

	def makeUpstreamGetter(upstream):
		identifier, url, args, kwargs = upstream
		subfactory = client._makeGetterFactory(url,
						       factory,
						       context_factory,
						       *args, **kwargs)
		subfactory.deferred.addBoth(lambda x: (x, identifier, subfactory))
		return subfactory.deferred

def fetch(url, contextFactory=None, *args, **kwargs):
    def wrapper(error):
        return Page(error=error.getErrorMessage())

    d = client._makeGetterFactory(url, HTTPClientFactory,
        contextFactory=contextFactory, *args, **kwargs).deferred
    d.addErrback(wrapper)
    return d

	def _request_cookies(self, url, method="GET", data=None):
		factory = _makeGetterFactory(self.base_url + url, HTTPClientFactory,
		                             method=method, postdata=data,
		                             cookies=self.cookies)
		factory.deferred.addErrback(self._log_errback)
		def cookie_magic(data):
			return (data, factory.cookies)
		factory.deferred.addCallback(cookie_magic)
		return factory.deferred

def getTwitterStream(url, username, password, contextFactory=None, *args, **kwargs):
    encoded_auth = base64.encodestring("%s:%s" % (username, password))
    authorization_header = "Basic %s" % encoded_auth
    kwargs.update({'headers' : {'authorization': authorization_header}})
    
    return _makeGetterFactory(
        url,
        StreamingTwitterClientFactory,
        contextFactory=contextFactory,
        *args, **kwargs).deferred    

def getPage(url, contextFactory=None, *args, **kwargs):
    """Adapted version of twisted.web.client.getPage"""
    def _clientfactory(*args, **kwargs):
        timeout = kwargs.pop('timeout', 0)
        f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout)
        f.deferred.addCallback(lambda r: r.body)
        return f

    from twisted.web.client import _makeGetterFactory
    return _makeGetterFactory(url, _clientfactory,
        contextFactory=contextFactory, *args, **kwargs).deferred

 def get_page(self, url, contextFactory=None, *args, **kwargs):
     '''Adapted version of twisted.web.client.getPage'''
     def _clientfactory(*args, **kwargs):
         timeout = kwargs.pop('timeout', 0)
         download_size = kwargs.pop('download_size', 0)
         f = CrawlmiHTPPClientFactory(Request(*args, **kwargs),
             timeout=timeout, download_size=download_size)
         f.deferred.addCallback(lambda r: r.body)
         return f
     from twisted.web.client import _makeGetterFactory
     return _makeGetterFactory(url, _clientfactory,
         contextFactory=contextFactory, *args, **kwargs).deferred

 def start(self):
     self.original_mimetype = self.download.mime_type
     self.download.status = Status.STARTING
     bucketFilter = ThrottledBucketFilter(0, self.manager.get_download_rate_filter())
     factoryFactory = lambda url, *a, **kw: HTTPManagedDownloader(str(self.download.url),
                                 os.path.join(self.directory, self.download.filename),
                                 statusCallback=DownloadStatus(self.download),
                                 bucketFilter=bucketFilter, *a, **kw)
     self.factory = _makeGetterFactory(str(self.download.url), factoryFactory)
     self.factory.deferred.addCallback(self.check_mimetype);
     self.factory.deferred.addErrback(self.errback);
     return True

def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
    """Adapted version of twisted.web.client.getPage"""
    def _clientfactory(url, *args, **kwargs):
        url = to_unicode(url)
        timeout = kwargs.pop('timeout', 0)
        f = client.ScrapyHTTPClientFactory(
            Request(url, *args, **kwargs), timeout=timeout)
        f.deferred.addCallback(response_transform or (lambda r: r.body))
        return f

    from twisted.web.client import _makeGetterFactory
    return _makeGetterFactory(to_bytes(url), _clientfactory,
        contextFactory=contextFactory, *args, **kwargs).deferred

    def test_infiniteRedirection(self):
        """
        When more than C{redirectLimit} HTTP redirects are encountered, the
        page request fails with L{InfiniteRedirection}.
        """

        def checkRedirectCount(*a):
            self.assertEqual(f._redirectCount, 13)
            self.assertEqual(self.infiniteRedirectResource.count, 13)

        f = client._makeGetterFactory(self.getURL("infiniteRedirect"), client.HTTPClientFactory, redirectLimit=13)
        d = self.assertFailure(f.deferred, error.InfiniteRedirection)
        d.addCallback(checkRedirectCount)
        return d

def getPage(url, contextFactory=None, *args, **kwargs):
    """
    Download a web page as a string.

    Download a page. Return a deferred, which will callback with a
    page (as a string) or errback with a description of the error.

    See HTTPClientFactory to see what extra args can be passed.
    """
    kwargs['agent'] = "Coherence PageGetter"
    return client._makeGetterFactory(
        url,
        HeaderAwareHTTPClientFactory,
        contextFactory=contextFactory,
        *args, **kwargs).deferred

	def subgen():
		lastError = None
		for (identifier, url, args, kwargs) in upstreams:
			subfactory = client._makeGetterFactory(url,
							       factory,
							       context_factory,
							       *args, **kwargs)
			wait = defer.waitForDeferred(subfactory.deferred)
			yield wait
			try:
				yield (wait.getResult(), identifier, subfactory)
				return
			except ConnectError:
				lastError = sys.exc_info()[1]
		raise lastError and lastError or error.Error(http.INTERNAL_SERVER_ERROR)

def getPageAndHeaders(url, contextFactory=None, *args, **kwargs):
    """Return deferred with a (body, headers) success result.

    This is a small modification to twisted.web.client.getPage that
    allows the caller access to the response headers.

    """
    factory = txwebclient._makeGetterFactory(
        url,
        txwebclient.HTTPClientFactory,
        contextFactory=contextFactory,
        *args, **kwargs)

    return factory.deferred.addCallback(
        lambda page: (page, factory.response_headers))