Python _newclient.ResponseNeverReceived方法代碼示例

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def test_onlyRetryIfNoResponseReceived(self):
        """
        Only L{RequestNotSent}, L{RequestTransmissionFailed} and
        L{ResponseNeverReceived} exceptions cause a retry.
        """
        pool = client.HTTPConnectionPool(None)
        connection = client._RetryingHTTP11ClientProtocol(None, pool)
        self.assertTrue(connection._shouldRetry(
            b"GET", RequestNotSent(), None))
        self.assertTrue(connection._shouldRetry(
            b"GET", RequestTransmissionFailed([]), None))
        self.assertTrue(connection._shouldRetry(
            b"GET", ResponseNeverReceived([]),None))
        self.assertFalse(connection._shouldRetry(
            b"GET", ResponseFailed([]), None))
        self.assertFalse(connection._shouldRetry(
            b"GET", ConnectionRefusedError(), None)) 

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def test_someResponseButNotAll(self):
        """
        If a partial response was received and the connection is lost, the
        resulting error is L{ResponseFailed}, but not
        L{ResponseNeverReceived}.
        """
        protocol = HTTPClientParser(
            Request(b'HEAD', b'/', _boringHeaders, None),
            lambda ign: None)
        d = protocol._responseDeferred

        protocol.makeConnection(StringTransport())
        protocol.dataReceived(b'2')
        protocol.connectionLost(ConnectionLost())
        return self.assertFailure(d, ResponseFailed).addCallback(
            self.assertIsInstance, ResponseFailed) 

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def test_dontRetryIfFailedDueToCancel(self):
        """
        If a request failed due to the operation being cancelled,
        C{_shouldRetry} returns C{False} to indicate the request should not be
        retried.
        """
        pool = client.HTTPConnectionPool(None)
        connection = client._RetryingHTTP11ClientProtocol(None, pool)
        exception = ResponseNeverReceived([Failure(defer.CancelledError())])
        self.assertFalse(connection._shouldRetry(b"GET", exception, None)) 

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def test_retryIfFailedDueToNonCancelException(self):
        """
        If a request failed with L{ResponseNeverReceived} due to some
        arbitrary exception, C{_shouldRetry} returns C{True} to indicate the
        request should be retried.
        """
        pool = client.HTTPConnectionPool(None)
        connection = client._RetryingHTTP11ClientProtocol(None, pool)
        self.assertTrue(connection._shouldRetry(
            b"GET", ResponseNeverReceived([Failure(Exception())]), None)) 

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def retryAttempt(self, willWeRetry):
        """
        Fail a first request, possibly retrying depending on argument.
        """
        protocols = []
        def newProtocol():
            protocol = StubHTTPProtocol()
            protocols.append(protocol)
            return defer.succeed(protocol)

        bodyProducer = object()
        request = client.Request(b"FOO", b"/", client.Headers(), bodyProducer,
                                 persistent=True)
        newProtocol()
        protocol = protocols[0]
        retrier = client._RetryingHTTP11ClientProtocol(protocol, newProtocol)

        def _shouldRetry(m, e, bp):
            self.assertEqual(m, b"FOO")
            self.assertIdentical(bp, bodyProducer)
            self.assertIsInstance(e, (RequestNotSent, ResponseNeverReceived))
            return willWeRetry
        retrier._shouldRetry = _shouldRetry

        d = retrier.request(request)

        # So far, one request made:
        self.assertEqual(len(protocols), 1)
        self.assertEqual(len(protocols[0].requests), 1)

        # Fail the first request:
        protocol.requests[0][1].errback(RequestNotSent())
        return d, protocols 

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def test_onlyRetryOnce(self):
        """
        If a L{client._RetryingHTTP11ClientProtocol} fails more than once on
        an idempotent query before a response is received, it will not retry.
        """
        d, protocols = self.retryAttempt(True)
        self.assertEqual(len(protocols), 2)
        # Fail the second request too:
        protocols[1].requests[0][1].errback(ResponseNeverReceived([]))
        # We didn't retry again:
        self.assertEqual(len(protocols), 2)
        return self.assertFailure(d, ResponseNeverReceived) 

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def test_cancelBeforeResponse(self):
        """
        The L{Deferred} returned by L{HTTP11ClientProtocol.request} will fire
        with a L{ResponseNeverReceived} failure containing a L{CancelledError}
        exception if the request was cancelled before any response headers were
        received.
        """
        transport = StringTransport()
        protocol = HTTP11ClientProtocol()
        protocol.makeConnection(transport)
        result = protocol.request(Request(b'GET', b'/', _boringHeaders, None))
        result.cancel()
        self.assertTrue(transport.aborting)
        return assertWrapperExceptionTypes(
            self, result, ResponseNeverReceived, [CancelledError]) 

# 需要導入模塊: from twisted.web import _newclient [as 別名]
# 或者: from twisted.web._newclient import ResponseNeverReceived [as 別名]
def errback_catcher(self, failure):
            # catch all errback failures,
            self.logger.error(repr(failure))

            if failure.check(ResponseNeverReceived):
                request = failure.request
                url= request.meta['current_url']
                father = request.meta['father']

                self.logger.error('Splash, ResponseNeverReceived for %s, retry in 10s ...', url)
                time.sleep(10)
                if response:
                    response_root_key = response.meta['root_key']
                else:
                    response_root_key = None
                yield SplashRequest(
                    url,
                    self.parse,
                    errback=self.errback_catcher,
                    endpoint='execute',
                    cache_args=['lua_source'],
                    meta={'father': father, 'current_url': url},
                    args=self.build_request_arg(response.cookiejar)
                )

            else:
                print('failure')
                #print(failure)
                print(failure.type)