Python DeferredList.addErrback方法代码示例

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
 def _storeData(self,
         data,
         request_hash,
         confirm_cache_write,
         http_history=None):
     if len(data["response"]) == 0:
         return self._storeDataErrback(Failure(exc_value=Exception("Response data is of length 0")), data, request_hash)
     #data["content-sha1"] = sha1(data["response"]).hexdigest()
     if http_history is None:
         http_history = {}
     if "content-sha1" not in http_history:
         http_history["content-sha1"] = data["content-sha1"]
     if "content-changes" not in http_history:
         http_history["content-changes"] = []
     if data["content-sha1"] != http_history["content-sha1"]:
         http_history["content-changes"].append(str(int(self.time_offset + time.time())))
     http_history["content-changes"] = http_history["content-changes"][-10:]
     headers = {}
     http_history["content-changes"] = filter(lambda x:len(x) > 0, http_history["content-changes"])
     headers["content-changes"] = ",".join(http_history["content-changes"])
     headers["content-sha1"] = data["content-sha1"]
     if "cache-control" in data["headers"]:
         if isinstance(data["headers"]["cache-control"], (list, tuple)):
             if "no-cache" in data["headers"]["cache-control"][0]:
                 return data
         else:
             if "no-cache" in data["headers"]["cache-control"]:
                 return data
     if "expires" in data["headers"]:
         if isinstance(data["headers"]["expires"], (list, tuple)):
             headers["cache-expires"] = data["headers"]["expires"][0]
         else:
             headers["cache-expires"] = data["headers"]["expires"]
     if "etag" in data["headers"]:
         if isinstance(data["headers"]["etag"], (list, tuple)):
             headers["cache-etag"] = data["headers"]["etag"][0]
         else:
             headers["cache-etag"] = data["headers"]["etag"]
     if "last-modified" in data["headers"]:
         if isinstance(data["headers"]["last-modified"], (list, tuple)):
             headers["cache-last-modified"] = data["headers"]["last-modified"][0]
         else:
             headers["cache-last-modified"] = data["headers"]["last-modified"]
     if "content-type" in data["headers"]:
         if isinstance(data["headers"]["content-type"], (list, tuple)):
             headers["content_type"] = data["headers"]["content-type"][0]
         else:
             headers["content_type"] = data["headers"]["content-type"]
     headers_key = 'headers:%s' % request_hash
     http_key = 'http:%s' % request_hash
     logger.debug("Writing data for request %s to redis." % request_hash)
     deferreds = []
     deferreds.append(self.redis_client.set(headers_key, compress(json.dumps(headers), 1)))
     deferreds.append(self.redis_client.set(http_key, compress(json.dumps(data["response"]), 1)))
     d = DeferredList(deferreds, consumeErrors=True)
     if confirm_cache_write:
         d.addCallback(self._storeDataCallback, data)
         d.addErrback(self._storeDataErrback, data, request_hash)
         return d
     return data

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
 def send_payment_request(self, readTokens, writeTokens):
   """Called by a Circuit object when it wants to actually make a payment
   @param readTokens:  the number of read tokens to pay for at each hop in the circuit
   @type  readTokens:  int
   @param writeTokens:  the number of read tokens to pay for at each hop in the circuit
   @type  writeTokens:  int"""
   assert (readTokens + writeTokens) / Globals.CELLS_PER_PAYMENT, "tried to pay for bad number of cells"
   #make sure our setup is done:
   if not self.setupDone:
     #have we even started?
     if not self.setupStarted:
       self.send_setup_message()
     self.queuedReadTokens += readTokens
     self.queuedWriteTokens += writeTokens
     return
   #dont bother trying to send payments for circuits that are already closed
   if self.circ.is_done():
     return
   #send the payments
   deferreds = []
   for paymentStream in self.paymentStreams.values():
     deferreds.append(paymentStream.send_payment(readTokens, writeTokens))
   paymentsDoneDeferred = DeferredList(deferreds)
   paymentsDoneDeferred.addErrback(self.generic_error_handler)
   addTokensDeferred = Deferred()
   self.inflightReadTokens += readTokens
   self.inflightWriteTokens += writeTokens
   #timeout in case the payment fails.  We will close the circuit in this case.
   event = Scheduler.schedule_once(PaymentStream.PAR_TIMEOUT, self.all_receipts_received, None, addTokensDeferred, readTokens, writeTokens, None)
   paymentsDoneDeferred.addCallback(self.all_receipts_received, addTokensDeferred, readTokens, writeTokens, event)
   addTokensDeferred.addCallback(self._add_tokens_callback, readTokens, writeTokens)
   addTokensDeferred.addErrback(self.generic_error_handler)

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
    def _configure_ssh(self, deployment):
        """
        :return: A ``Deferred`` which fires when all nodes have been configured
            with ssh keys.
        """
        self.ssh_configuration.create_keypair()
        results = []
        for node in deployment.nodes:
            results.append(
                deferToThread(
                    self.ssh_configuration.configure_ssh,
                    node.hostname, self.ssh_port
                )
            )
        d = DeferredList(results, fireOnOneErrback=True, consumeErrors=True)

        # Exit with ssh's output if it failed for some reason:
        def got_failure(failure):
            if failure.value.subFailure.check(CalledProcessError):
                raise SystemExit(
                    b"Error connecting to cluster node: " +
                    failure.value.subFailure.value.output)
            else:
                return failure

        d.addErrback(got_failure)
        return d

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
 def test_complex2(self, runtime):
     def check(ls):
         if (2 == runtime.id) or (1 == runtime.id):
             self.assertEquals(ls[0][1], "V1")
             self.assertEquals(ls[1][1], "V1")
             self.assertEquals(ls[2][1], "V1")
             self.assertEquals(ls[3][1], "V2")
         else:
             self.assertEquals(ls[0][1], "V1")
             self.assertEquals(ls[1][1], "V1")
             self.assertEquals(ls[2][1], "V1")
             self.assertEquals(ls[3][1], "V2")
             self.assertEquals(ls[4][1], "V2")
     field = self.Zp
     results = []
     results += runtime.broadcast(runtime.players.keys(), runtime.players.keys(), "V1")
     if runtime.id in [1, 2]:
         v = runtime.broadcast([1, 2], [3], "V2")
         if isinstance(v, list):
             results += v
         else:
             results.append(v)
     else:
         results += runtime.broadcast([1, 2], [3])
     if 3 == runtime.id:
         results += [runtime.broadcast([3], runtime.players.keys(), str(7))]
     else:
         results += [runtime.broadcast([3], runtime.players.keys())]
     dls = DeferredList(results)
     runtime.schedule_callback(dls, check)
     dls.addErrback(runtime.error_handler)
     return dls

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
 def render(self, request):
     
     request.setHeader('Content-type', 'text/javascript; charset=UTF-8')
     
     if len(request.postpath) > 0:
         if request.postpath[0] == "server":
             return simplejson.dumps(self.spider.getServerData())
         elif request.postpath[0] == "exposed_function_details":
             return simplejson.dumps(self.spider.getExposedFunctionDetails())
         elif request.postpath[0] == "get":
             if "uuid" in request.args:
                 get_deferreds = []
                 for uuid in request.args["uuid"]:
                     get_deferreds.append(self.spider.get(uuid))
                 d = DeferredList(get_deferreds, consumeErrors = True)
                 d.addCallback(self._getCallback, request.args["uuid"])
                 d.addCallback(self._successResponse)
                 d.addErrback(self._errorResponse)
                 d.addCallback(self._immediateResponse, request)
                 return server.NOT_DONE_YET
             else:
                 return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
         elif request.postpath[0] == "show_reservation":
             if "uuid" in request.args:
                 d = self.spider.showReservation(request.args["uuid"][0])
                 d.addCallback(self._successResponse)
                 d.addErrback(self._errorResponse)
                 d.addCallback(self._immediateResponse, request)
                 return server.NOT_DONE_YET
             else:
                 return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
         elif request.postpath[0] == "delete_reservation":
             if "uuid" in request.args:
                 d = self.spider.deleteReservation(request.args["uuid"][0])
                 d.addCallback(self._successResponse)
                 d.addErrback(self._errorResponse)
                 d.addCallback(self._immediateResponse, request)
                 return server.NOT_DONE_YET
             else:
                 return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))
         elif request.postpath[0] == "delete_function_reservations": 
             if "function_name" in request.args:
                 d = self.spider.deleteFunctionReservations(request.args["function_name"][0])
                 d.addCallback(self._successResponse)
                 d.addErrback(self._errorResponse)
                 d.addCallback(self._immediateResponse, request)
                 return server.NOT_DONE_YET           
             else:
                 return self._errorResponse(Failure(exc_value=Exception("Parameter function_name is required."))) 
         elif request.postpath[0] == "execute_reservation":
             if "uuid" in request.args:
                 d = self.spider.queryByUUID(request.args["uuid"][0])
                 d.addCallback(self._successResponse)
                 d.addErrback(self._errorResponse)
                 d.addCallback(self._immediateResponse, request)
                 return server.NOT_DONE_YET
             else:
                 return self._errorResponse(Failure(exc_value=Exception("Parameter UUID is required.")))                                                            

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
 def deleteReservation( self, uuid, function_name="Unknown" ):
     logger.info("Deleting reservation %s, %s." % (function_name, uuid))
     deferreds = []
     deferreds.append(self.sdb.delete(self.aws_sdb_reservation_domain, uuid))
     deferreds.append(self.s3.deleteObject(self.aws_s3_storage_bucket, uuid))
     d = DeferredList(deferreds)
     d.addCallback(self._deleteReservationCallback, function_name, uuid)
     d.addErrback(self._deleteReservationErrback, function_name, uuid)
     return d

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
    def createReservation( self, function_name, **kwargs ):
        
        if not isinstance( function_name, str ):
            for key in self.functions:
                if self.functions[key]["function"] == function_name:
                    function_name = key
                    break            
        
        if function_name not in self.functions:
            raise Exception("Function %s does not exist." % function_name )
    
        function = self.functions[ function_name ]
    
        filtered_kwargs = {}
        for key in function["required_arguments"]:
            if key in kwargs:
                filtered_kwargs[key] = convertToUTF8( kwargs[key] )
            else:
                raise Exception("Required parameter '%s' not found. Required parameters are %s. Optional parameters are %s." % (key, function["required_arguments"], function["optional_arguments"] ))

        for key in function["optional_arguments"]:
            if key in kwargs:
                filtered_kwargs[key] = convertToUTF8( kwargs[key] )
        
        if function["interval"] > 0:
            reserved_arguments = {}
            reserved_arguments["reservation_function_name"] = function_name
            reserved_arguments["reservation_created"] = sdb_now(offset=self.time_offset)
            reserved_arguments["reservation_next_request"] = reserved_arguments["reservation_created"]
            reserved_arguments["reservation_error"] = "0"
            
            all_arguments = {}
            all_arguments.update( reserved_arguments )
            all_arguments.update( filtered_kwargs )
            
            uuid = uuid4().hex
            logger.debug( "Creating reservation on SimpleDB for %s, %s." % (function_name, uuid))

            a = self.sdb.putAttributes( self.aws_sdb_reservation_domain, uuid, all_arguments )
            a.addCallback( self._createReservationCallback, function_name, uuid )
            a.addErrback( self._createReservationErrback, function_name, uuid )
                        
            if "call_immediately" in kwargs and not evaluateBoolean( kwargs["call_immediately"] ):    
                d = DeferredList([a], consumeErrors=True)
            else:
                logger.debug( "Calling %s immediately with arguments:\n%s" % (function_name, PrettyPrinter.pformat(filtered_kwargs) ) )
                b = self.callExposedFunctionImmediately( function["function"], filtered_kwargs, function_name )
                d = DeferredList([a,b], consumeErrors=True)
            
            d.addCallback( self._createReservationCallback2, function_name, uuid )
            d.addErrback( self._createReservationErrback2, function_name, uuid )
            return d
            
        else:
            logger.debug( "Calling %s immediately with arguments:\n%s" % (function_name, PrettyPrinter.pformat(filtered_kwargs) ) )
            d = self.callExposedFunctionImmediately( function["function"], filtered_kwargs, function_name )
            return d

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
 def testStoredReturn(self):
     deferreds = []
     for filename in self.filenames:
         d = self.spider.rq.getPage("http://127.0.0.1:5000/function/load_data_stored", method="POST", postdata={"filename":filename})
         d.addCallback( self._processStoredRequestCallback, filename )
         deferreds.append(d)
     d = DeferredList(deferreds, consumeErrors=True )
     d.addCallback(self._testStoredReturnCallback)
     d.addErrback( self._testStoredReturnErrback )
     return d        

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
 def deleteReservation(self, uuid, function_name="Unknown"):
     LOGGER.info("Deleting reservation %s, %s." % (function_name, uuid))
     parameters = {'uuid': uuid}
     query_string = urllib.urlencode(parameters)
     url = 'http://%s:%s/function/schedulerserver/remoteremovefromheap?%s' % (self.scheduler_server, self.schedulerserver_port, query_string)
     deferreds = []
     deferreds.append(self.getPage(url=url))
     deferreds.append(self.s3.deleteObject(self.aws_s3_storage_bucket, uuid))
     d = DeferredList(deferreds)
     d.addCallback(self._deleteReservationCallback, function_name, uuid)
     d.addErrback(self._deleteReservationErrback, function_name, uuid)
     return d

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
    def publish(self):
        """Publish all added charms to provider storage and zookeeper.

        Returns the charm_state of all scheduled charms.
        """
        publish_deferreds = []
        for charm_id, charm in self._charm_add_queue:
            publish_deferreds.append(self._publish_one(charm_id, charm))

        publish_deferred = DeferredList(publish_deferreds,
                                        fireOnOneErrback=1,
                                        consumeErrors=1)
        # callbacks and deferreds to unwind the dlist
        publish_deferred.addCallback(self._cb_extract_charm_state)
        publish_deferred.addErrback(self._eb_extract_error)
        return publish_deferred

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
	def multi_set_attr(self, owner_userid, image_ids, attr_dict):
		"""
		Sets attributes on a list of images to be whatever is in the attr_dict.
		attr_dict must have keys that are valid in images.set_attr

		@param owner_username: Username
		@type owner_username: String

		@param media_id_list: set of media_ids to change
		@type media_id_list: List

		@param attr_dict: dictionary of attribute->values to apply to the list of images
		@type attr_dict: Dictionary

		@return: image attributes
		@rtype: List
		"""
		owner_userid = validation.cast_integer(owner_userid, 'owner_userid')
		if not isinstance(image_ids, (list, tuple)):
			raise errors.ValidationError, 'media_id_list must be a list or tuple'
		if not isinstance(attr_dict, dict):
			raise errors.ValidationError, 'attr_dict must be a dictionary'
		for x in image_ids:
			self.log.debug("validating image_id [%s]" % x)
			validation.cast_integer(x, 'x')
		for x in attr_dict.keys():
			if x not in self.attr_fields:
				raise errors.ValidationError, 'attr_dict key [%s] is not valid!' % x
		self.log.debug("all args look OK in multi_set_attr")

		dl = []
		for image_id in image_ids:
			image_id = validation.cast_integer(image_id, 'image_id')
			d_temp = Deferred()
			self.log.debug("adding deferreds for image_id [%s]" % image_id)
			for key, value in attr_dict.items():
				self.log.debug("adding defferred for image_id [%s] key [%s]" % (image_id, key))
				d_temp.addCallback(self.set_attr, image_id, key, value)
				d_temp.addCallback(lambda _: owner_userid)
			d_temp.callback(owner_userid)
			dl.append(d_temp)
		d_list = DeferredList(dl, fireOnOneErrback=True)
		d_list.addCallback(lambda _: 0);
		def handle_it(failure):
			self.log.warning("failure on multi_set_attr [%s]" % failure.getErrorMessage())
		d_list.addErrback(handle_it)
		return d_list

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
	def xmlrpc_del_albums(self, info, set_ids, album_ids):
		@stack
		def process_set(set_id):
			dl2 = []
			for album_id in album_ids:
				dl2.append(process_album(set_id, album_id))
			dList2 = DeferredList(dl2, fireOnOneErrback=True)
			return dList2

		@stack
		def process_album(set_id, album_id):
			return self.del_album(info['userid'], set_id, album_id)
		dl = []
		for set_id in set_ids:
			dl.append(process_set(set_id))
		dList = DeferredList(dl, fireOnOneErrback=True)
		dList.addCallback(lambda _: (0, "success"))
		dList.addErrback(lambda _: (-1, _.getErrorMessage))
		return dList

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
    def moveFiles(self, virtSrcBase, virtDestBase, relFiles):
        self.debug("MOVING: %r, %r, %r", virtSrcBase, virtDestBase, relFiles)
        if not self._local:
            raise TranscoderError("Component not properly setup yet")

        def move_failed(failure, src, dest):
            msg = ("Fail to move file '%s' to '%s': %s"
                   % (src, dest, log.getFailureMessage(failure)))
            self.warning("%s", msg)
            raise TranscoderError(msg, cause=failure)

        def move_file(src, dest, attr=None):
            self.debug("Moving file '%s' to '%s'", src, dest)
            dest_dir = os.path.dirname(dest)
            safe_mkdirs(dest_dir, "input file destination", attr)
            d = deferToThread(shutil.move, src, dest)
            d.addErrback(move_failed, src, dest)
            return d

        def move_files_failed(results):
            first_failure = None
            for ok, result in results:
                if not ok:
                    if not first_failure:
                        first_failure = result
            return first_failure

        sem = DeferredSemaphore(1)
        move_tasks = []

        for file in relFiles:
            source_path = virtSrcBase.append(file).localize(self._local)
            dest_path = virtDestBase.append(file).localize(self._local)
            source_path = os.path.realpath(source_path)
            dest_path = os.path.realpath(dest_path)
            d = sem.run(move_file, source_path, dest_path, self._pathAttr)
            move_tasks.append(d)

        dl = DeferredList(move_tasks, consumeErrors=True)
        dl.addErrback(move_files_failed)
        return d

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
def main():
    """Main command line entry point."""
    parser = argparse.ArgumentParser(
        description='Make requests to one or more HTTP or HTTPS URIs, '
                    'and record the interactions in a cassette.',
        epilog='If no URIs are passed on the command line, they are '
               'read from standard input, one per line.')
    parser.add_argument(
        'uris', metavar='URI', nargs='*', help='URI to fetch')
    parser.add_argument(
        'cassette_path', metavar='CASSETTE',
        help='path to output cassette')
    args = parser.parse_args()
    uris = args.uris or imap(lambda x: x.strip(), sys.stdin)
    cassette_agent = CassetteAgent(Agent(reactor), args.cassette_path)
    agent = ContentDecoderAgent(
        RedirectAgent(cassette_agent), [('gzip', GzipDecoder)])
    finished = DeferredList([agent.request('GET', uri) for uri in uris])
    finished.addCallback(save_and_exit, cassette_agent)
    finished.addErrback(fail_and_exit)
    reactor.run()

# 需要导入模块: from twisted.internet.defer import DeferredList [as 别名]
# 或者: from twisted.internet.defer.DeferredList import addErrback [as 别名]
def oneBatch(prev, ipList):
	done = 0
	deferreds = []
	for toDo in ipList:
		deferreds.append(makeRequest(toDo))
		done += 1
		if done == concurrentRequests: break
	else:
		print "\n%d requests ended in empty match" % prev
		d = getPage("%s/stats/avg" % (baseUrl, ))
		@d.addCallback
		def c2(time):
			print "Average matching time in server: %s" % time
			reactor.stop()
		return
	d = DeferredList(deferreds)
	@d.addCallback
	def c(results):
		return prev + len([x for x in results if x[1] in ("[]", "ok\n")])
	d.addErrback(log.err)
	d.addBoth(oneBatch, ipList)