Python defer.DeferredSemaphore类代码示例

def trigger_convergence_groups(authenticator, region, groups,
                               concurrency_limit, no_error_group):
    """
    Trigger convergence on given groups

    :param IAuthenticator authenticator: Otter authenticator
    :param str region: Region where this is running
    :param list groups: List of group dicts
    :param int concurrency_limit: Concurrency limit
    :param bool no_error_group: If true then do not converge ERROR groups

    :return: Deferred fired with None
    """
    sem = DeferredSemaphore(concurrency_limit)
    d = DeferredList(
        [sem.run(trigger_convergence, authenticator, region, group,
                 no_error_group)
         for group in groups],
        fireOnOneCallback=False,
        fireOnOneErrback=False,
        consumeErrors=True)
    d.addCallback(
        lambda results: [(g["tenantId"], g["groupId"], f.value)
                         for g, (s, f) in zip(groups, results) if not s])
    return d

	def recordVideoProcess(self, resW, resH, totalTimeSec, framerate, serverIP, piName, recordTimesList, file):
		semi = DeferredSemaphore(1)

		jobs = []
		for runs in range(len(recordTimesList)/2):
			print  "recordTimes recordVideoProcess:", recordTimesList
			self.writeFile("recordTimes recordVideoProcess:")
			try:
				startAtTime = self.calculateTimeDifference(recordTimesList.pop(0), recordTimesList.pop(0))
				jobs.append(semi.run(tv.takeVideo, int(resW), int(resH), int(totalTimeSec),\
						int(framerate), startAtTime, serverIP, piName, file))
			except:
				self.writeFile("That time was not valid. Calling next time.")
				self.writeFile("len recordTimesList: " + str(len(recordTimesList)))
				if len(recordTimesList)%2>0:
					self.writeFile("odd number")
					recordTimesList.pop(0)
					self.writeFile("new len: " + str(len(recordTimesList)))
					reactor.callLater(0.5, self.transport.write, "TIMEINPUTERROR {0}\n".format(piName))
				continue
			
		jobs = DeferredList(jobs)

		print  "Results: ", jobs.addCallback(self.getResults, piName)
		# self.writeFile("Results: ", jobs.addCallback(self.getResults, piName))
		jobs.addCallback(lambda _: reactor.callLater(5, reactor.stop))

class ThreadedUrllib2TestMixin(object):

    def setUp(self):
        self._semaphore = DeferredSemaphore(2)

    def tearDown(self):
        pass


    def getPages(self, count, url):
        return gatherResults([self.getPage(url) for i in xrange(0, count)])

    @inlineCallbacks
    def getPage(self, url):
        yield self._semaphore.acquire()
        page = yield deferToThread(self._openPage, url)
        self._semaphore.release()
        returnValue(page)

    def _openPage(self, url):
        log.msg("Opening url: %r" % url)
        return urlopen(url).read()

    @inlineCallbacks
    def getPageLength(self, url):
        response = yield self.getPage(url)
        returnValue(len(response))

 def __init__(self, buffer, start_callback=None):
     self.buffer = buffer
     self._done = False
     self._flush = DeferredSemaphore(1)
     self._waiter = DeferredSemaphore(1)
     self._flush.acquire()
     self._started = start_callback
     self._keepalive = LoopingCall(self._send_keepalive)

def parallel_map(iterable, fn, *args, **kwargs):
    deferreds = []
    parallelism_limiter = DeferredSemaphore(MAX_PARALLELISM)
    for item in iterable:
        d = parallelism_limiter.run(fn, item, *args, **kwargs)
        deferreds.append(d)
    results = yield gatherResults(deferreds)
    returnValue(results)

 def __init__(self, tokens = 5):
     if tokens < 1:
         raise ValueError("tokens must be > 0")
     
     if tokens == 1:
         self.lock = DeferredLock()
     else:
         self.lock = DeferredSemaphore(tokens)

def trigger_convergence_groups(authenticator, region, groups,
                               concurrency_limit):
    """
    Trigger convergence on given groups
    """
    sem = DeferredSemaphore(concurrency_limit)
    return gatherResults(
        [sem.run(trigger_convergence, authenticator, region, group)
         for group in groups],
        consumeErrors=True)

 def __init__(self, mapsPath, fetchURL, deleteIfNotPresent, tfLevelSounds):
     assert isinstance(mapsPath, str) and len(mapsPath)
     assert isinstance(fetchURL, str) and len(fetchURL)
     self.mapsPath = FilePath(mapsPath)
     self.downloadTempPath = self.mapsPath.child('mapupdater')
     self.fetchURL = URLPath.fromString(fetchURL)
     self.deleteIfNotPresent = deleteIfNotPresent
     self.tfLevelSounds = tfLevelSounds
     self.semaphore = DeferredSemaphore(1)
     self.downloadSemaphore = DeferredSemaphore(4)
     for fp in self.downloadTempPath.globChildren('*.bsp.bz2'):
         fp.remove()

	def runFiles():
	    semi = DeferredSemaphore(1)

	    jobs = []
	    for runs in range(5):
	        jobs.append(semi.run(collectFiles))

	    jobs = DeferredList(jobs)
	    def cbFinished(ignored):
	        print 'Finishing job'
	    jobs.addCallback(cbFinished)
	    return jobs

class Processor():
    """
    Whenever mixtapeReceived is called, deferToThread is scheduled to be run as
    soon as a "slot" for being run is available. There is currently 1 slot
    deferToThread runs process_mixtape in another thread, and releases the
    slot when its that process is done
    """
    def __init__(self):
        self.sem = DeferredSemaphore(1) #do one thing at a time

    def mixtapeReceived(self, mixtape):
        debug("Adding %s to be processed" % mixtape)
        self.sem.run(deferToThread, process.process_mixtape, *mixtape)

    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Issue a new request.
        @param method: The request method to send.
        @type method: C{str}
        @param uri: The request URI send.
        @type uri: C{str}
        @param scheme: A string like C{'http'} or C{'https'} (the only two
            supported values) to use to determine how to establish the
            connection.
 
        @param host: A C{str} giving the hostname which will be connected to in
            order to issue a request.

        @param port: An C{int} giving the port number the connection will be on.

        @param path: A C{str} giving the path portion of the request URL.
        @param headers: The request headers to send.  If no I{Host} header is
            included, one will be added based on the request URI.
        @type headers: L{Headers}
        @param bodyProducer: An object which will produce the request body or,
            if the request body is to be empty, L{None}.
        @type bodyProducer: L{IBodyProducer} provider
        @return: A L{Deferred} which fires with the result of the request (a
            L{Response} instance), or fails if there is a problem setting up a
            connection over which to issue the request.  It may also fail with
            L{SchemeNotSupported} if the scheme of the given URI is not
            supported.
        @rtype: L{Deferred}
        """
        scheme, host, port, path = _parse(uri)
        if headers is None:
            headers = Headers()
        if not headers.hasHeader('host'):
            # This is a lot of copying.  It might be nice if there were a bit
            # less.
            headers = Headers(dict(headers.getAllRawHeaders()))
            headers.addRawHeader(
                'host', self._computeHostValue(scheme, host, port))
        if self.persistent:
            sem = self._semaphores.get((scheme, host, port))
            if sem is None:
                sem = DeferredSemaphore(self.maxConnectionsPerHostName)
                self._semaphores[scheme, host, port] = sem
            return sem.run(self._request, method, scheme, host, port, path,
                           headers, bodyProducer)
        else:
            return self._request(
                method, scheme, host, port, path, headers, bodyProducer)

	def collectFiles():
	    semaphore = DeferredSemaphore(1)
	    files = glob.glob('*.py')
	    dl = list()

	    for item in range(len(files)):
	        #Queues list of things to be sent and runs it
	        dl.append(semaphore.run(sendFiles, files[item]))

	    # convert to a DefferedList. Allows for callback call
	    dl = DeferredList(dl)
	    def cbFinished(ignored):
	        print 'Finishing job'
	    dl.addCallback(cbFinished)
	    return dl

def main():
    agent = Agent(reactor)
    sem = DeferredSemaphore(5)
    print "Loading IDs"
    ids = getBeermeIds()
    ids = ids[:100]
    print "Done Loading %s IDs" % str(len(ids))
    jobs = []
    for id in ids:
        jobs.append(sem.run(beerme_request,id,agent))
    d = gatherResults(jobs)
    d.addBoth(cbShutdown)

    print "Starting reactor..."
    reactor.run()

    def __init__(self, conn_str):
        self.waiting = []
        self.inuse = []
        self.free = []

        self.semaphore = DeferredSemaphore(1)
        self.updateTime()

def main():
    agent = Agent(reactor)
    sem = DeferredSemaphore(10)
    print "Loading breweries..."
    mongo = MongoClient().entities.breweries
    breweries = loadBreweries(mongo)
    print "Done loading breweries."
    jobs = []
    for brewery in breweries:
        jobs.append(sem.run(socialRequest,brewery,agent,mongo))
    #    if len(jobs) % 50 == 0:
    #        print "Brewery Jobs started: %d" % len(jobs) 
    d = gatherResults(jobs)
    d.addBoth(cbShutdown)
    print "Let the Reactor BEGIN!"
    reactor.run()