Python Failure.check方法代码示例

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
 def _start_stop_common(self, label, action):
     result = {}
     try:
         function = getattr(self.model, action)
         d = defer.maybeDeferred(function, label)
         wfd = defer.waitForDeferred(d)
         yield wfd
         result = wfd.getResult()
         #take this time to update the instance
         if isinstance(result, dict):
             thisInst = self.model.getInstance(label)
             thisInst.updateInfo(result) 
     except:
         failure = Failure()
         if failure.check(DroneCommandFailed):
             result = failure.value.resultContext
         else:
             #log the error, allowing for debugging
             self.debugReport()
             #be nice and return something to the end user
             template = "%s: %s" % (getException(failure), failure.getErrorMessage())
             context = {'error':failure,'code':-2}
             result = self.resultContext(template, None, **context)
         #finally wrap the failure into a known type
         result = Failure(DroneCommandFailed(result))
     #AppInstances need a moment to be updated
     d = defer.Deferred()
     reactor.callLater(1.0, d.callback, None)
     wfd = defer.waitForDeferred(d)
     yield wfd
     wfd.getResult()
     yield result

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
 def __call__(self, argstr):
     args = argstr.split()
     resultContext = None
     if not args: #return command usage
         methods = {}
         for name,args,doc in self.exposedMethodInfo:
             methods[name] = {'args' : args, 'doc' : doc}
         resultContext = dict(description=self.__doc__, methods=methods)
         yield resultContext
     else:
         method = args.pop(0)
         try:
             wfd = defer.waitForDeferred(
                     self.invoke(method,args)
             )
             yield wfd
             resultContext = wfd.getResult()
         except:
             failure = Failure()
             if failure.check(DroneCommandFailed):
                 resultContext = failure.value.resultContext
             else:
                 #be nice and return something to the end user
                 template = "[%(application)s] "
                 template += "%s: %s" % (getException(failure), failure.getErrorMessage())
                 context = {'error': True, 'code': -2, 'stacktrace': failure.getTraceback()}
                 resultContext = self.resultContext(template, None, 
                     **context
                 )
                 
         yield resultContext

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
 def newfunc(*args,**kwargs):
     try:
         return func(*args,**kwargs)
     except:
         failure = Failure()
         caught_exc = failure.value
         err_msg = failure.getErrorMessage()
         if failure.check(exc): raise caught_exc #1
         exc_inst = exc(err_msg)
         exc_inst.inner_exception = caught_exc
         raise exc_inst

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
 def handleDeferreds(labels):
     """Remember last yield is the return value, don't use return"""
     results = {}
     descriptions = []
     ret = {}
     code = 0
     for l in labels:
         try:
             d = defer.maybeDeferred(func, l, *args[1:], **kwargs)
             wfd = defer.waitForDeferred(d)
             yield wfd
             ret = wfd.getResult()
         except:
             failure = Failure()
             des = "%s: %s" % \
                     (getException(failure),failure.getErrorMessage())
             if failure.check(DroneCommandFailed):
                 result[l] = failure.value.resultContext
                 if 'description' not in result[l]:
                     result[l]['description'] = des
                 result[l]['stacktrace'] = failure.getTraceback()
                 result[l]['error'] = True
                 if 'code' not in result[l]:
                     result[l]['code'] = 1
             else:
                 ret = {
                     'description': des,
                     'code': 1,
                     'error': True,
                     'stacktrace': failure.getTraceback()
                 }
         if not ret: #NoneType detection
             ret = {'description' : str(ret), 'code' : 0}
         if 'code' in ret:
             code += abs(ret['code'])
         results[l] = ret
         try:
             descriptions.append(results[l]['description'])
         except:
            self.debugReport()
     results['code'] = code
     try:
         results['description'] = '\n'.join(descriptions)
     except:
         results['description'] = None
     if len(labels) == 0:
         Label = labels[0]
     else:
         Label = None
     ret = self.resultContext('%(description)s',label=Label,**results) 
     yield ret

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
    def invoke(self, name, args):
        """Invoke Exposed Methods
           @param name (str) - name of method to invoke
           @param args (tuple) - arguments to pass to invoked method

           @return (defer.Deferred)
        """
        if name not in self.exposedMethods:
            return defer.fail(DroneCommandFailed(self.resultContext(
                "[%(application)s] Unknown method '%(method)s'", method=name, 
                error='unknown method'))
            )
        try:
            #our own form of maybeDeferred
            d = self.exposedMethods[name](*args)
            if isinstance(d, defer.Deferred):
                action = Action(' '.join([str(i) for i in \
                        (self.action, name) + tuple(args)]), d)
                return action.deferred
            elif isinstance(d, DroneCommandFailed):
                return defer.fail(d)
            elif isinstance(d, dict):
                return defer.succeed(d)
            elif isinstance(d, type(None)):
                #this just feels dirty
                return defer.succeed(d)
            elif isinstance(d, Failure):
                d.raiseException() #sigh
            #probably from a triggerred Event callback
            elif type(d) == types.InstanceType:
                return defer.succeed(None)
            return defer.fail(FormatError("Result is not formatted correctly you " + \
                 "must return self.resultContext or DroneCommandFailed." + \
                 "\nResult: <%s>" % (str(d),)))
        except:
            failure = Failure()
            if failure.check(DroneCommandFailed):
                template = "[%(application)s] %(description)s"
                context = failure.value.resultContext
                if not 'description' in context:
                    context['description'] = failure.getErrorMessage()
            else:
                template = "[%(application)s] " + "%s: %s" % (getException(failure),
                        failure.getErrorMessage())
                context = {'error': True, 'code':-2, 'stacktrace': failure.getTraceback()}
            return defer.fail(DroneCommandFailed(self.resultContext(template, 
                None, **context))
            )

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
def droneblast(conversation, command):
    server = conversation.context.get('server')
    response = None
    d = None
    try:
        if not isinstance(server, Server):
            conversation.say('On what <b>server</b>?')
            raise AssertionError('incomplete converation context')

        options = {}
        if 'timeout' in conversation.context:
            options['timeout'] = conversation.context['timeout']

        try:
            conversation.say("Running droned command...")
            d = server.manager.run(command, **options)
            deferreds = conversation.context.get('deferreds', [])
            deferreds.append(d)
            conversation.context.update({'deferreds': deferreds})
            wfd = defer.waitForDeferred(d)
            yield wfd
            result = wfd.getResult()
        except:
            failure = Failure()
            if failure.check(DroneCommandFailed):
                rc = failure.value.resultContext
                conversation.say(rc.get('description') or str(rc), useHTML=False)
            else:
                conversation.say(failure.getTraceback(), useHTML=False)
        else:
            if isinstance(result, dict):
                output = result.values()[0].get('description', str(result))
            else:
                output = str(result)
            deferreds = conversation.context.get('deferreds', [])
            try: deferreds.remove(d)
            except: pass
            conversation.context.update({'deferreds': deferreds})
            conversation.say("Command completed\n%s" % output, useHTML=False)
    except AssertionError: pass
    except:
        result = Failure()
    yield result

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
   def startService(self):
       """Start All AppManager Services"""
       if self.scanning.called: #need to pre-populate values
           self.scanning = defer.maybeDeferred(self._first_scan)
       self.first_run = True
       self._task = task.LoopingCall(self.scan_app_instances)

       #plugins will be created and loaded when needed
       for shortname in config.APPLICATIONS.keys():
           manager = None
           try:
               applog = logWithContext(type=shortname,route=SERVICENAME)
               applog('Loading Application Plugin')
               applog('Creating Application Manager')
               manager = AppManager(shortname)
               manager.parentService = self
               #check and see if the model is bound
               if not AppManager(shortname).running:
                   applog('Starting Application Manager')
                   manager.start()
           except:
               failure = Failure()
               #bad plugin, not adaptable
               failures = (InvalidPlugin, TypeError)
               if failure.check(*failures) and manager:
                   log('plugin for %s is invalid' % (manager.name,))
                   manager.action.__class__.delete(manager.action)
                   try: pluginFactory.delete_plugin(manager.model)
                   except: pass #silence
                   AppManager.delete(manager)
                   if not config.EXCESSIVE_LOGGING: continue #avoid extra logging
               try: failure.raiseException()
               except: crashReport('ApplicationLoader', self)
       Service.startService(self)
       Event('instance-started').subscribe(self.reset_tracking)
       #wire allapps action into the server
       drone.builtins.update({
           'allapps': self.allapps_action,
           'applist': self.applist_action,
       })
       #delay scanning by some interval
       config.reactor.callLater(SERVICECONFIG.initial_delay, self._start_all_tasks)

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
    def startBuild(self, build_status, workerforbuilder):
        """This method sets up the build, then starts it by invoking the
        first Step. It returns a Deferred which will fire when the build
        finishes. This Deferred is guaranteed to never errback."""
        self.workerforbuilder = workerforbuilder
        self.conn = None

        worker = workerforbuilder.worker

        log.msg("%s.startBuild" % self)

        self.build_status = build_status
        # TODO: this will go away when build collapsing is implemented; until
        # then we just assign the build to the first buildrequest
        brid = self.requests[0].id
        builderid = yield self.getBuilderId()
        self.buildid, self.number = \
            yield self.master.data.updates.addBuild(
                builderid=builderid,
                buildrequestid=brid,
                workerid=worker.workerid)

        self.stopBuildConsumer = yield self.master.mq.startConsuming(self.controlStopBuild,
                                                                     ("control", "builds",
                                                                      str(self.buildid),
                                                                      "stop"))
        self.setupOwnProperties()

        # then narrow WorkerLocks down to the right worker
        self.locks = [(l.getLock(workerforbuilder.worker), a)
                      for l, a in self.locks]
        metrics.MetricCountEvent.log('active_builds', 1)

        # make sure properties are available to people listening on 'new'
        # events
        yield self._flushProperties(None)
        self.build_status.buildStarted(self)
        yield self.master.data.updates.setBuildStateString(self.buildid, u'starting')
        yield self.master.data.updates.generateNewBuildEvent(self.buildid)

        try:
            self.setupBuild()  # create .steps
        except Exception:
            yield self.buildPreparationFailure(Failure(), "worker_prepare")
            self.buildFinished(['Build.setupBuild', 'failed'], EXCEPTION)
            return

        # flush properties in the beginning of the build
        yield self._flushProperties(None)

        yield self.master.data.updates.setBuildStateString(self.buildid,
                                                           u'preparing worker')
        try:
            ready_or_failure = yield workerforbuilder.prepare(self)
        except Exception:
            ready_or_failure = Failure()

        # If prepare returns True then it is ready and we start a build
        # If it returns failure then we don't start a new build.
        if ready_or_failure is not True:
            yield self.buildPreparationFailure(ready_or_failure, "worker_prepare")
            if self.stopped:
                self.buildFinished(["worker", "cancelled"], self.results)
            elif isinstance(ready_or_failure, Failure) and ready_or_failure.check(interfaces.LatentWorkerCannotSubstantiate):
                self.buildFinished(["worker", "cannot", "substantiate"], EXCEPTION)
            else:
                self.buildFinished(["worker", "not", "available"], RETRY)
            return

        # ping the worker to make sure they're still there. If they've
        # fallen off the map (due to a NAT timeout or something), this
        # will fail in a couple of minutes, depending upon the TCP
        # timeout.
        #
        # TODO: This can unnecessarily suspend the starting of a build, in
        # situations where the worker is live but is pushing lots of data to
        # us in a build.
        yield self.master.data.updates.setBuildStateString(self.buildid,
                                                           u'pinging worker')
        log.msg("starting build %s.. pinging the worker %s"
                % (self, workerforbuilder))
        try:
            ping_success_or_failure = yield workerforbuilder.ping()
        except Exception:
            ping_success_or_failure = Failure()

        if ping_success_or_failure is not True:
            yield self.buildPreparationFailure(ping_success_or_failure, "worker_ping")
            self.buildFinished(["worker", "not", "pinged"], RETRY)
            return

        self.conn = workerforbuilder.worker.conn

        # To retrieve the builddir property, the worker must be attached as we
        # depend on its path_module. Latent workers become attached only after
        # preparing them, so we can't setup the builddir property earlier like
        # the rest of properties
        self.setupWorkerBuildirProperty(workerforbuilder)
        self.setupWorkerForBuilder(workerforbuilder)
        self.subs = self.conn.notifyOnDisconnect(self.lostRemote)
#.........这里部分代码省略.........

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
class TubConnector:
    """I am used to make an outbound connection. I am given a target TubID
    and a list of locationHints, and I try all of them until I establish a
    Broker connected to the target. I will consider redirections returned
    along the way. The first hint that yields a connected Broker will stop
    the search. If targetTubID is None, we are going to make an unencrypted
    connection.

    This is a single-use object. The connection attempt begins as soon as my
    connect() method is called.

    @param locationHints: the list of 'host:port' hints where the remote tub
                          can be found.
    """

    failureReason = None
    CONNECTION_TIMEOUT = 60
    timer = None

    def __init__(self, parent, tubref):
        self.tub = parent
        self.target = tubref
        self.remainingLocations = self.target.getLocations()
        # attemptedLocations keeps track of where we've already try to
        # connect, so we don't try them twice.
        self.attemptedLocations = []

        # pendingConnections contains a (PBClientFactory -> Connector) map
        # for pairs where connectTCP has started, but negotiation has not yet
        # completed. We keep track of these so we can shut them down when we
        # stop connecting (either because one of the connections succeeded,
        # or because someone told us to give up).
        self.pendingConnections = {}

    def connect(self):
        """Begin the connection process. This should only be called once.
        This will either result in the successful Negotiation object invoking
        the parent Tub's brokerAttached() method, our us calling the Tub's
        connectionFailed() method."""
        timeout = self.tub.options.get('connect_timeout',
                                       self.CONNECTION_TIMEOUT)
        self.timer = reactor.callLater(timeout, self.connectionTimedOut)
        self.active = True
        self.connectToAll()

    def stopConnectionTimer(self):
        if self.timer:
            self.timer.cancel()
            del self.timer

    def shutdown(self):
        self.active = False
        self.stopConnectionTimer()
        for c in self.pendingConnections.values():
            c.disconnect()

    def connectToAll(self):
        while self.remainingLocations:
            location = self.remainingLocations.pop()
            if location in self.attemptedLocations:
                continue
            self.attemptedLocations.append(location)
            host, port = location.split(":")
            port = int(port)
            f = TubConnectorClientFactory(self, host)
            c = reactor.connectTCP(host, port, f)
            self.pendingConnections[f] = c
            if self.tub.options.get("debug_stall_second_connection"):
                # for unit tests, hold off on making the second connection
                # for a moment. This allows the first connection to get to a
                # known state.
                reactor.callLater(0.1, self.connectToAll)
                return
        self.checkForFailure()

    def connectionTimedOut(self):
        self.timer = None
        why = "no connection established within client timeout"
        self.failureReason = Failure(NegotiationError(why))
        self.shutdown()
        self.failed()

    def clientConnectionFailed(self, factory, reason):
        # this is called if a TCP connection cannot be established
        if not self.failureReason:
            self.failureReason = reason
        del self.pendingConnections[factory]
        self.checkForFailure()

    def redirectReceived(self, newLocation):
        # the redirected connection will disconnect soon, which will trigger
        # negotiationFailed(), so we don't have to do a
        # del self.pendingConnections[factory]
        self.remainingLocations.append(newLocation)
        self.connectToAll()

    def negotiationFailed(self, factory, reason):
        # this is called if protocol negotiation cannot be established, or if
        # the connection is closed for any reason prior to switching to the
        # Banana protocol
#.........这里部分代码省略.........

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
    def stopInstance(self, label):
        """Stops an application instance by label

           @param label: (string)

           @fires Event('instance-stopped')

           return defer.Deferred()
        """
        result = {}
        template = '[%(application)s,%(label)s] %(description)s'
        context = {'code': 254}
        thisInst = None
        try:
            thisInst = self.model.getInstance(label)
            thisInst.shouldBeRunning = False
            if not thisInst.running:
                context.update(self.model.statusInstance(label))
                raise DroneCommandFailed(context)
            pid = thisInst.process.pid 
            self.log("Trying to shutdown %d gracefully" % (pid,))

            def failed(result):
                """attempting to be consistant"""
                self.log("Failed to shutdown process gracefully")
                return result

            def success(result):
                """attempting to be consistant"""
                self.log("process %d gracefully shutdown" % (pid,))
                return result

            d = self._start_stop_common(label, 'stopInstance')
            d.addCallback(success)
            d.addErrback(failed)
            d.addErrback(self._killInstance, thisInst)
            wfd = defer.waitForDeferred(d)
            yield wfd
            #refresh the instance as it can change
            thisInst = self.model.getInstance(label)
            result = wfd.getResult()
            if isinstance(result, dict):
                context.update(result)
            elif isinstance(result, DroneCommandFailed):
                context.update(result.resultContext)
            if not thisInst.running:
                context['code'] = 0
                Event('instance-stopped').fire(instance=thisInst)
                raise AssertionError('ignore me')
            raise DroneCommandFailed(context)
        except AssertionError:
            #update the instance model
            wfd = defer.waitForDeferred(self.statusInstance(label))
            yield wfd
            result = wfd.getResult()
            result['code'] = context['code']
        except:
            failure = Failure()
            if failure.check(DroneCommandFailed):
                context = failure.value.resultContext
                template = '%(description)s'
            else:
                temp = "%s: %s" % (getException(failure), failure.getErrorMessage())
                context = {'error': failure, 'code': 253, 'description': temp}
            result = self.resultContext(template, thisInst, **context)
        try:
            thisInst = self.model.getInstance(label)
            thisInst.shouldBeRunning = False
        except: pass
        yield result

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
    def startInstance(self, label):
        """Starts an application instance by label

           @param label: (string)

           @fires Event('instance-started')

           return defer.Deferred()
        """
        template = '[%(application)s,%(label)s] %(description)s'
        context = {
            'description': 'Failed to Start',
            'code': 254
        }
        result = {}
        thisInst = None
        try:
            if self.model.getInstance(label).running:
                context.update(self.model.statusInstance(label))
                raise DroneCommandFailed(context)

            d = self._start_stop_common(label, 'startInstance')
            wfd = defer.waitForDeferred(d)
            yield wfd
            result = wfd.getResult()

            d = self.statusInstance(label)
            wfd = defer.waitForDeferred(d)
            yield wfd
            result.update(wfd.getResult())
            #refresh the instance as it can change
            thisInst = self.model.getInstance(label)
            if isinstance(result, dict):
                context.update(result)
            elif isinstance(result, DroneCommandFailed):
                context.update(result.resultContext)
            if thisInst.running:
                Event('instance-started').fire(instance=thisInst)
                context['code'] = 0
                raise AssertionError('ignore')
            raise DroneCommandFailed(context)
        except AssertionError:
            #update the instance model
            wfd = defer.waitForDeferred(self.statusInstance(label))
            yield wfd
            result = wfd.getResult()
        except:
            thisInst = self.model.getInstance(label)
            failure = Failure()
            if failure.check(DroneCommandFailed):
                template = '%(description)s'
                context = failure.value.resultContext
            else:
                #log the error, allowing for debugging
                self.debugReport()
                #be nice and return something to the end user
                temp = "%s: %s" % (getException(failure), failure.getErrorMessage())
                context = {'error': failure, 'code': 253, 'description': temp}
            result = self.resultContext(template, thisInst, **context)
        try:
            thisInst = self.model.getInstance(label)
            thisInst.shouldBeRunning = True
        except: pass
        yield result

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
class CommandTestMixin(ConnectionTestMixin):
    """A subclass of `ConnectionTestMixin` that also sets up a command
    plugin in addition to a connection and transport."""

    #: The command plugin class to test.
    command_class = None

    #: Any additional help arguments to test in `test_help`.
    help_arguments = tuple()

    def setUp(self):
        super(CommandTestMixin, self).setUp()
        self.default_venue = self.connection.nickname
        name = self.command_class.name
        self.keyword = name.rsplit('/', 1)[-1].rsplit('.', 1)[-1].lower()
        self.command = self.connection.settings.enable(name, [self.keyword])
        self.reply_buffer = iter([])
        self.failure = None

    def command_message(self, content, **kwargs):
        action = kwargs.pop('action', 'command')
        kwargs.setdefault('actor', self.other_users[0])
        kwargs.setdefault('venue', self.default_venue)
        kwargs.setdefault('subaction', self.keyword)
        return Message(self.connection, False, action,
                       content=content, **kwargs)

    @inlineCallbacks
    def send_command(self, content, **kwargs):
        if isinstance(content, unicode):
            content = content.encode(DEFAULT_ENCODING)
        request = self.command_message(content, **kwargs)
        try:
            response = yield self.command.respond_to(request)
        except UserVisibleError:
            self.failure = Failure()
        else:
            if response is not None:
                self.reply_buffer = ReplyBuffer(response, request)

    def assert_reply(self, expected):
        finished = maybeDeferred(next, self.reply_buffer, None)
        finished.addCallback(self.assertEqual, expected)
        return finished

    def assert_no_replies(self):
        finished = maybeDeferred(next, self.reply_buffer, None)
        finished.addCallback(self.assertIsNone)
        return finished

    def assert_error(self, expected):
        self.assertIsNotNone(self.failure)
        self.assertIsNotNone(self.failure.check(UserVisibleError))
        self.assertEqual(self.failure.getErrorMessage(), expected)
        self.failure = None

    @staticmethod
    def use_cassette(cassette_name):
        cassette_path = os.path.join(CASSETTE_LIBRARY, cassette_name + '.json')
        cassette_agent = CassetteAgent(Agent(reactor), cassette_path)
        def decorator(func):
            @wraps(func)
            def wrapper(self, *args, **kwargs):
                self.command.agent = IdentifyingAgent(ContentDecoderAgent(
                    RedirectAgent(cassette_agent), [('gzip', GzipDecoder)]))
                finished = maybeDeferred(func, self, *args, **kwargs)
                finished.addCallback(cassette_agent.save)
                return finished
            return wrapper
        return decorator

    def test_help(self):
        """Ensure that command help doesn't cause errors."""
        for content in ('',) + self.help_arguments:
            self.send_command(content, action='cmdhelp')

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
            else:
                # there's no regex in cmdinfo to extract info
                log.msg("%s: no callback registered" % self.state)
                self.notify_success(self.waitbuf)
                self.waitbuf = self.waitbuf.replace(match.group(), '', 1)

            self.transition_to_idle()
        else:
            # there is no end of response detected, so we have either an error
            # or a split command (like send_sms, save_sms, etc.)
            match = E.extract_error(self.waitbuf)
            if match:
                exception, error, m = match
                e = exception(error)
                f = Failure(e)
                if not f.check(*self.cmd.nolog):
                    log.err(e, "waiting")
                # send the failure back
                self.notify_failure(f)
                # remove the exception string from the waitbuf
                self.waitbuf = self.waitbuf.replace(m.group(), '', 1)
                self.transition_to_idle()
            else:
                match = SPLIT_PROMPT.search(data)
                if match:
                    log.msg("waiting: split command prompt detected")
                    self.send_splitcmd()
                    self.waitbuf = self.waitbuf.replace(match.group(), '', 1)
                else:
                    log.msg("waiting: unmatched data %r" % data)

# 需要导入模块: from twisted.python.failure import Failure [as 别名]
# 或者: from twisted.python.failure.Failure import check [as 别名]
    def _handle_send_response(self, result, payloadsByTopicPart,
                              deferredsByTopicPart):
        """Handle the response from our client to our send_produce_request

        This is a bit complex. Failures can happen in a few ways:

          1. The client sent an empty list, False, None or some similar thing
             as the result, but we were expecting real responses.
          2. The client had a failure before it even tried sending any requests
             to any brokers.

               a. Kafka error: See if we can retry the whole request
               b. Non-kafka: Figure it's a programming error, fail all deferreds

          3. The client sent all the requests (it's all or none) to the brokers
             but one or more request failed (timed out before receiving a
             response, or the brokerclient threw some sort of exception on send
             In this case, the client throws FailedPayloadsError, and attaches
             the responses (NOTE: some can have errors!), and the payloads
             where the send itself failed to the exception.
          4. The client sent all the requests, all responses were received, but
             the Kafka broker indicated an error with servicing the request on
             some of the responses.
        """

        def _deliver_result(d_list, result=None):
            """Possibly callback each deferred in a list with single result"""
            for d in d_list:
                if not isinstance(d, Deferred):
                    # nested list...
                    _deliver_result(d, result)
                else:
                    # We check d.called since the request could have been
                    # cancelled while we waited for the response
                    if not d.called:
                        d.callback(result)

        def _do_retry(payloads):
            # We use 'fail_on_error=False' because we want our client to
            # process every response that comes back from the brokers so
            # we can determine which requests were successful, and which
            # failed for retry
            d = self.client.send_produce_request(
                payloads, acks=self.req_acks, timeout=self.ack_timeout,
                fail_on_error=False)
            self._req_attempts += 1
            # add our handlers
            d.addBoth(self._handle_send_response, payloadsByTopicPart,
                      deferredsByTopicPart)
            return d

        def _cancel_retry(failure, dc):
            # Cancel the retry callLater and pass-thru the failure
            dc.cancel()
            # cancel all the top-level deferreds associated with the request
            _deliver_result(deferredsByTopicPart.values(), failure)
            return failure

        def _check_retry_payloads(failed_payloads_with_errs):
            """Check our retry count and retry after a delay or errback

            If we have more retries to try, create a deferred that will fire
            with the result of delayed retry. If not, errback the remaining
            deferreds with failure

            Params:
            failed_payloads - list of (payload, failure) tuples
            """
            # Do we have retries left?
            if self._req_attempts >= self._max_attempts:
                # No, no retries left, fail each failed_payload with its
                # associated failure
                for p, f in failed_payloads_with_errs:
                    t_and_p = TopicAndPartition(p.topic, p.partition)
                    _deliver_result(deferredsByTopicPart[t_and_p], f)
                return
            # Retries remain!  Schedule one...
            d = Deferred()
            dc = self.client.reactor.callLater(
                self._retry_interval, d.callback, [p for p, f in
                                                   failed_payloads])
            self._retry_interval *= self.RETRY_INTERVAL_FACTOR
            # Cancel the callLater when request is cancelled before it fires
            d.addErrback(_cancel_retry, dc)

            # Reset the topic metadata for all topics which had failed_requests
            # where the failures were of the kind UnknownTopicOrPartitionError
            # or NotLeaderForPartitionError, since those indicate our client's
            # metadata is out of date.
            reset_topics = set()
            for payload, e in failed_payloads:
                if (isinstance(e, NotLeaderForPartitionError) or
                        isinstance(e, UnknownTopicOrPartitionError)):
                    reset_topics.add(payload.topic)
            if reset_topics:
                self.client.reset_topic_metadata(*reset_topics)

            d.addCallback(_do_retry)
            return d

#.........这里部分代码省略.........