Python DeferredSemaphore.run方法代码示例

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
class Processor():
    """
    Whenever mixtapeReceived is called, deferToThread is scheduled to be run as
    soon as a "slot" for being run is available. There is currently 1 slot
    deferToThread runs process_mixtape in another thread, and releases the
    slot when its that process is done
    """
    def __init__(self):
        self.sem = DeferredSemaphore(1) #do one thing at a time

    def mixtapeReceived(self, mixtape):
        debug("Adding %s to be processed" % mixtape)
        self.sem.run(deferToThread, process.process_mixtape, *mixtape)

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
def trigger_convergence_groups(authenticator, region, groups,
                               concurrency_limit, no_error_group):
    """
    Trigger convergence on given groups

    :param IAuthenticator authenticator: Otter authenticator
    :param str region: Region where this is running
    :param list groups: List of group dicts
    :param int concurrency_limit: Concurrency limit
    :param bool no_error_group: If true then do not converge ERROR groups

    :return: Deferred fired with None
    """
    sem = DeferredSemaphore(concurrency_limit)
    d = DeferredList(
        [sem.run(trigger_convergence, authenticator, region, group,
                 no_error_group)
         for group in groups],
        fireOnOneCallback=False,
        fireOnOneErrback=False,
        consumeErrors=True)
    d.addCallback(
        lambda results: [(g["tenantId"], g["groupId"], f.value)
                         for g, (s, f) in zip(groups, results) if not s])
    return d

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
	def recordVideoProcess(self, resW, resH, totalTimeSec, framerate, serverIP, piName, recordTimesList, file):
		semi = DeferredSemaphore(1)

		jobs = []
		for runs in range(len(recordTimesList)/2):
			print  "recordTimes recordVideoProcess:", recordTimesList
			self.writeFile("recordTimes recordVideoProcess:")
			try:
				startAtTime = self.calculateTimeDifference(recordTimesList.pop(0), recordTimesList.pop(0))
				jobs.append(semi.run(tv.takeVideo, int(resW), int(resH), int(totalTimeSec),\
						int(framerate), startAtTime, serverIP, piName, file))
			except:
				self.writeFile("That time was not valid. Calling next time.")
				self.writeFile("len recordTimesList: " + str(len(recordTimesList)))
				if len(recordTimesList)%2>0:
					self.writeFile("odd number")
					recordTimesList.pop(0)
					self.writeFile("new len: " + str(len(recordTimesList)))
					reactor.callLater(0.5, self.transport.write, "TIMEINPUTERROR {0}\n".format(piName))
				continue
			
		jobs = DeferredList(jobs)

		print  "Results: ", jobs.addCallback(self.getResults, piName)
		# self.writeFile("Results: ", jobs.addCallback(self.getResults, piName))
		jobs.addCallback(lambda _: reactor.callLater(5, reactor.stop))

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
def parallel_map(iterable, fn, *args, **kwargs):
    deferreds = []
    parallelism_limiter = DeferredSemaphore(MAX_PARALLELISM)
    for item in iterable:
        d = parallelism_limiter.run(fn, item, *args, **kwargs)
        deferreds.append(d)
    results = yield gatherResults(deferreds)
    returnValue(results)

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
def trigger_convergence_groups(authenticator, region, groups,
                               concurrency_limit):
    """
    Trigger convergence on given groups
    """
    sem = DeferredSemaphore(concurrency_limit)
    return gatherResults(
        [sem.run(trigger_convergence, authenticator, region, group)
         for group in groups],
        consumeErrors=True)

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
	def runFiles():
	    semi = DeferredSemaphore(1)

	    jobs = []
	    for runs in range(5):
	        jobs.append(semi.run(collectFiles))

	    jobs = DeferredList(jobs)
	    def cbFinished(ignored):
	        print 'Finishing job'
	    jobs.addCallback(cbFinished)
	    return jobs

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Issue a new request.
        @param method: The request method to send.
        @type method: C{str}
        @param uri: The request URI send.
        @type uri: C{str}
        @param scheme: A string like C{'http'} or C{'https'} (the only two
            supported values) to use to determine how to establish the
            connection.
 
        @param host: A C{str} giving the hostname which will be connected to in
            order to issue a request.

        @param port: An C{int} giving the port number the connection will be on.

        @param path: A C{str} giving the path portion of the request URL.
        @param headers: The request headers to send.  If no I{Host} header is
            included, one will be added based on the request URI.
        @type headers: L{Headers}
        @param bodyProducer: An object which will produce the request body or,
            if the request body is to be empty, L{None}.
        @type bodyProducer: L{IBodyProducer} provider
        @return: A L{Deferred} which fires with the result of the request (a
            L{Response} instance), or fails if there is a problem setting up a
            connection over which to issue the request.  It may also fail with
            L{SchemeNotSupported} if the scheme of the given URI is not
            supported.
        @rtype: L{Deferred}
        """
        scheme, host, port, path = _parse(uri)
        if headers is None:
            headers = Headers()
        if not headers.hasHeader('host'):
            # This is a lot of copying.  It might be nice if there were a bit
            # less.
            headers = Headers(dict(headers.getAllRawHeaders()))
            headers.addRawHeader(
                'host', self._computeHostValue(scheme, host, port))
        if self.persistent:
            sem = self._semaphores.get((scheme, host, port))
            if sem is None:
                sem = DeferredSemaphore(self.maxConnectionsPerHostName)
                self._semaphores[scheme, host, port] = sem
            return sem.run(self._request, method, scheme, host, port, path,
                           headers, bodyProducer)
        else:
            return self._request(
                method, scheme, host, port, path, headers, bodyProducer)

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
	def collectFiles():
	    semaphore = DeferredSemaphore(1)
	    files = glob.glob('*.py')
	    dl = list()

	    for item in range(len(files)):
	        #Queues list of things to be sent and runs it
	        dl.append(semaphore.run(sendFiles, files[item]))

	    # convert to a DefferedList. Allows for callback call
	    dl = DeferredList(dl)
	    def cbFinished(ignored):
	        print 'Finishing job'
	    dl.addCallback(cbFinished)
	    return dl

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
def main():
    agent = Agent(reactor)
    sem = DeferredSemaphore(5)
    print "Loading IDs"
    ids = getBeermeIds()
    ids = ids[:100]
    print "Done Loading %s IDs" % str(len(ids))
    jobs = []
    for id in ids:
        jobs.append(sem.run(beerme_request,id,agent))
    d = gatherResults(jobs)
    d.addBoth(cbShutdown)

    print "Starting reactor..."
    reactor.run()

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
def main():
    agent = Agent(reactor)
    sem = DeferredSemaphore(10)
    print "Loading breweries..."
    mongo = MongoClient().entities.breweries
    breweries = loadBreweries(mongo)
    print "Done loading breweries."
    jobs = []
    for brewery in breweries:
        jobs.append(sem.run(socialRequest,brewery,agent,mongo))
    #    if len(jobs) % 50 == 0:
    #        print "Brewery Jobs started: %d" % len(jobs) 
    d = gatherResults(jobs)
    d.addBoth(cbShutdown)
    print "Let the Reactor BEGIN!"
    reactor.run()

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
    def start(self):
        """ get each page """
        deferreds = []
        sem = DeferredSemaphore(self.MAX_RUN)
        
        for key in self.book.keys():

            sleep(self.RATE_LIMIT)
            d =  sem.run(getPage, self.book[key])
            d.addCallback(self.pageCallback, key)
            d.addErrback(self.errorHandler, key)
            deferreds.append(d)

        dl = DeferredList(deferreds)
        dl.addCallback(self.listCallback)
        dl.addCallback(self.finish)

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
def trigger_convergence_groups(authenticator, region, groups, concurrency_limit, no_error_group):
    """
    Trigger convergence on given groups

    :param IAuthenticator authenticator: Otter authenticator
    :param str region: Region where this is running
    :param list groups: List of group dicts
    :param int concurrency_limit: Concurrency limit
    :param bool no_error_group: If true then do not converge ERROR groups

    :return: Deferred fired with None
    """
    sem = DeferredSemaphore(concurrency_limit)
    return gatherResults(
        [sem.run(trigger_convergence, authenticator, region, group, no_error_group) for group in groups],
        consumeErrors=True,
    ).addCallback(lambda _: None)

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
def run(packages, modules, other_args, reactor, limit, excludes):
    sem = DeferredSemaphore(limit)
    proc_argss = get_cafe_args(packages, modules, excludes)
    deferreds = [
        sem.run(getProcessOutputAndValue, 'cafe-runner',
                other_args + proc_args, env=os.environ, reactor=reactor)
        for proc_args in proc_argss]
    results = yield gatherResults(deferreds, consumeErrors=True)

    failed = False
    for proc_args, (stdout, stderr, code) in zip(proc_argss, results):
        if code == 0:
            continue
        failed = True
        print('Error when running ', ' '.join(proc_args))
        print('Stdout\n', stdout, 'Stderr\n', stderr)

    if failed:
        raise SystemExit('Some tests failed')

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
    def moveFiles(self, virtSrcBase, virtDestBase, relFiles):
        self.debug("MOVING: %r, %r, %r", virtSrcBase, virtDestBase, relFiles)
        if not self._local:
            raise TranscoderError("Component not properly setup yet")

        def move_failed(failure, src, dest):
            msg = ("Fail to move file '%s' to '%s': %s"
                   % (src, dest, log.getFailureMessage(failure)))
            self.warning("%s", msg)
            raise TranscoderError(msg, cause=failure)

        def move_file(src, dest, attr=None):
            self.debug("Moving file '%s' to '%s'", src, dest)
            dest_dir = os.path.dirname(dest)
            safe_mkdirs(dest_dir, "input file destination", attr)
            d = deferToThread(shutil.move, src, dest)
            d.addErrback(move_failed, src, dest)
            return d

        def move_files_failed(results):
            first_failure = None
            for ok, result in results:
                if not ok:
                    if not first_failure:
                        first_failure = result
            return first_failure

        sem = DeferredSemaphore(1)
        move_tasks = []

        for file in relFiles:
            source_path = virtSrcBase.append(file).localize(self._local)
            dest_path = virtDestBase.append(file).localize(self._local)
            source_path = os.path.realpath(source_path)
            dest_path = os.path.realpath(dest_path)
            d = sem.run(move_file, source_path, dest_path, self._pathAttr)
            move_tasks.append(d)

        dl = DeferredList(move_tasks, consumeErrors=True)
        dl.addErrback(move_files_failed)
        return d

# 需要导入模块: from twisted.internet.defer import DeferredSemaphore [as 别名]
# 或者: from twisted.internet.defer.DeferredSemaphore import run [as 别名]
class VBMSClient(object):
    def __init__(self, reactor, connect_vbms_path, bundle_path, endpoint_url,
                 keyfile, samlfile, key, keypass, ca_cert, client_cert):
        self._reactor = reactor

        self._connect_vbms_path = connect_vbms_path
        self._bundle_path = bundle_path
        self._endpoint_url = endpoint_url
        self._keyfile = keyfile
        self._samlfile = samlfile
        self._key = key
        self._keypass = keypass
        self._ca_cert = ca_cert
        self._client_cert = client_cert

        self._connect_vbms_semaphore = DeferredSemaphore(tokens=8)

    def _path_to_ruby(self, path):
        if path is None:
            return "nil"
        else:
            return repr(path)

    def _execute_connect_vbms(self, logger, request, formatter, args):
        ruby_code = """#!/usr/bin/env ruby

$LOAD_PATH << '{connect_vbms_path}/src/'

require 'json'

require 'vbms'


client = VBMS::Client.new(
    {endpoint_url!r},
    {keyfile},
    {samlfile},
    {key},
    {keypass!r},
    {ca_cert},
    {client_cert},
)
request = {request}
result = client.send(request)
STDOUT.write({formatter})
STDOUT.flush()
        """.format(
            connect_vbms_path=self._connect_vbms_path,
            endpoint_url=self._endpoint_url,
            keyfile=self._path_to_ruby(self._keyfile),
            samlfile=self._path_to_ruby(self._samlfile),
            key=self._path_to_ruby(self._key),
            keypass=self._keypass,
            ca_cert=self._path_to_ruby(self._ca_cert),
            client_cert=self._path_to_ruby(self._client_cert),

            request=request,
            formatter=formatter,
        ).strip()
        with tempfile.NamedTemporaryFile(suffix=".rb", delete=False) as f:
            f.write(ruby_code)

        st = os.stat(f.name)
        os.chmod(f.name, st.st_mode | stat.S_IEXEC)

        @inlineCallbacks
        def run():
            timer = logger.time("process.spawn")
            try:
                stdout, stderr, exit_code = yield getProcessOutputAndValue(
                    '/bin/bash', [
                        '-lc',
                        '{} exec {} {}'.format(
                            self._bundle_path,
                            f.name,
                            " ".join(map(pipes.quote, args))
                        )
                    ],
                    env=os.environ,
                    path=self._connect_vbms_path,
                    reactor=self._reactor
                )
            finally:
                timer.stop()
            if exit_code != 0:
                raise VBMSError(stdout, stderr, exit_code)
            returnValue(stdout)

        return self._connect_vbms_semaphore.run(run)

    @inlineCallbacks
    def get_document_types(self, logger):
        response = yield self._execute_connect_vbms(
            logger.bind(process="GetDocumentTypes"),
            "VBMS::Requests::GetDocumentTypes.new()",
            "result.map(&:to_h).to_json",
            [],
        )
        returnValue(json.loads(response))

#.........这里部分代码省略.........