Python multiprocessing.Semaphore类代码示例

    def call_START(self, mock_pid_ptid_mapping=MagicMock(has_key=lambda *args, **kwargs: False)):
        self.Locks = [Lock() for _ in range(0, 3)]
        self.NL_L = Semaphore(2)
        self.free_threads = Semaphore(5)
        mock_proc_inter_instance = MagicMock(free_threads=self.free_threads,
                                             Locks=self.Locks,
                                             NL_L=self.NL_L,
                                             pid_ptid_mapping=MagicMock(
                                                 has_key=lambda *args, **kwargs: False)
                                             )

        with patch("processing_interface.ProcessingInterface.Instance", side_effect=lambda *args, **kwargs: mock_proc_inter_instance):
            LP = LongProcess.LongProcess(self.test_dicoms_dirs,
                                         [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs])

        self.__test_Locks()

        LP._LongProcess__START()

        self.__test_Locks()

        print LP._Process__success
        assert(LP._Process__success == "SUCCESS")
        assert(LP._Process__state == STATES.LongFS1)

        return LP

    def run(self, tasks, build_config, parallel_threads):
        semaphore = Semaphore(parallel_threads)
        process_finished_notify = Condition(Lock())
        while tasks.count_buildable_tasks() > 0:
            task = tasks.get_next()

            if task is None:
                self.wait_tasks_to_complete(parallel_threads, process_finished_notify, semaphore)
                continue

            semaphore.acquire()
            task.state = Task.State.RUNNING
            logging.debug("Starting task %s", task.name)
            self.start_new_process(process_finished_notify, semaphore, self.process_job, task, build_config)

        self.wait_tasks_to_complete(parallel_threads, process_finished_notify, semaphore)

        if tasks.count(Task.State.FAILED) > 0:
            logging.error('Some packages failed to build.')
            logging.error("  %s", tasks.print_name(Task.State.FAILED))
            return 1
        if tasks.count(Task.State.RUNNING) > 0:
            logging.error('Something went wrong, there are still some running tasks.')
            return 1
        if tasks.count(Task.State.NEW) > 0:
            logging.error('Something went wrong, there are still unprocessed tasks.')
            return 1

        logging.info("Build completed successfully.")
        return 0

def block_until_processed(cookie_jar: CookieJar, cookie_paths: Sequence[str],
                          expected_number_of_calls_to_mark_as_complete: int):
    """
    Puts the given cookies into the cookie jar and wait until they have been completed/marked for reprocessing.
    :param cookie_jar: the cookie jar to put cookies to process into
    :param cookie_paths: the cookie paths to process
    :param expected_number_of_calls_to_mark_as_complete: the number of calls expected to the Cookie jar's
    `mark_as_complete` method
    """
    if cookie_jar.queue_length() != 0:
        raise RuntimeError("Already cookies in the jar")

    mark_as_complete_semaphore = Semaphore(0)
    original_mark_as_complete = cookie_jar.mark_as_complete

    def mark_as_complete(path: str):
        mark_as_complete_semaphore.release()
        original_mark_as_complete(path)

    cookie_jar.mark_as_complete = MagicMock(side_effect=mark_as_complete)

    for cookie_path in cookie_paths:
        cookie_jar.mark_for_processing(cookie_path)

    calls_to_mark_as_complete = 0
    while calls_to_mark_as_complete != expected_number_of_calls_to_mark_as_complete:
        mark_as_complete_semaphore.acquire()
        assert cookie_jar.mark_as_complete.call_count <= expected_number_of_calls_to_mark_as_complete
        calls_to_mark_as_complete += 1

    assert calls_to_mark_as_complete == cookie_jar.mark_as_complete.call_count

 def __init__(self, n, timeout=None):
     self.n = n
     self.to = timeout
     self.count = Value('i', 0)
     self.mutex = Semaphore(1)
     self.turnstile1 = Semaphore(0)
     self.turnstile2 = Semaphore(1)

def launch_workers(outfile, start_index, end_index, score_flag, force, verbose):
	BASE_URL = "http://www.ign.com/games/all-ajax?startIndex="
	
	
	# Synchronization Tools
	num_workers = Semaphore(MAX_NUM_PROCESSES)
	outfile_lock = Lock()
	urlopen_lock = Lock()
	stderr_lock = Lock()
	print_lock = Lock()
	
	# Write the categories
	if (outfile != None):
		outfile.write("title,link,platform,publisher,score,date\n")

	# Launch the workers
	processes = []
	curr_index = start_index;
	while curr_index <= end_index:
		curr_url = BASE_URL + str(curr_index)
	 	worker = Process(target=open_url_and_parse,
	 		args=(outfile, curr_url, score_flag, force, verbose,
	 			outfile_lock, urlopen_lock, stderr_lock, print_lock,
	 			num_workers))
	 	processes.append(worker)
	 	if verbose:
			print_lock.acquire()
			print "Launching worker for url: %s" % curr_url
			print_lock.release()
	 	num_workers.acquire()
	 	worker.start()
	 	curr_index += INDEX_INCREMENT; 
	for p in processes:
	 	p.join()

 def setCurrentSimulationTime(self, currentSimulationTime):
    
     semaphore = Semaphore()
     semaphore.acquire()
     self.__currentSimulationTime = currentSimulationTime
     semaphore.release()
     return self.__currentSimulationTime

 def getPeerID(self, peerId):
     
     semaphore = Semaphore()
     semaphore.acquire()
     peer = self.__layout[peerId]
     semaphore.release()
     return peer

def run():
    algo = parameters["algo"]
    files = [open(x) for x in parameters["files"]]
    configs = []
    p = parameters["params"]
    max_processes = 3
    semaphore = Semaphore(max_processes)

    # generate configurations as compination of possible
    # keys and product of values
    for keys in it.combinations(p.keys(), len(p.keys())):
        v = [p[k] for k in keys]
        for values in it.product(*v):
            config = {}
            for i, k in enumerate(keys):
                config[k] = values[i]
            configs.append(config)
    for f in files:
        for conf in configs:
            config = {"FILENAME": f.name}
            config.update(conf)

            f.seek(0)
            num_vars, clauses = parser.parse(f)

            p = MyProcess(target=run_algorithm, args=(algo, num_vars, clauses, config, semaphore))

            semaphore.acquire()
            p.start()

 def getNeighbors(self, peer):
             
     semaphore = Semaphore()
     semaphore.acquire()
     neighbors = self.__layout[peer.getId()].getNeighbors()
     semaphore.release()
     return neighbors

 def countNeighbors(self, peer):
                    
     semaphore = Semaphore()
     semaphore.acquire()
     
     count = peer.countNeighbors()
     semaphore.release()
     return count

 def getNeighborIt(self, peer):
             
     semaphore = Semaphore()
     semaphore.acquire()
     neighbors = []
     for neighbor in self.__layout[peer.getId()].getNeighbors():
         neighbors.append(neighbor.getTargetPeer())
     neighborIt = neighbors.__iter__()
     semaphore.release()
     return neighborIt

 def addPeer(self, peer):
     
     if self.__layout.has_key(peer.getPID()):
         return False
     
     semaphore = Semaphore()
     semaphore.acquire()
     
     self.__layout[peer.getPID()] = peer
     semaphore.release()
     NetworkLogger().resgiterLoggingInfo("Add peer %s in Layout Network "%(peer.getPID()))
     return self.__layout.has_key(peer.getPID())

class Msg(object):
    """
    Data structure encapsulating a message.
    """

    def __init__(self, size):
        self.s_e = Semaphore(1)
        self.s_f = Semaphore(0)
        self.s_buf = Array(ct.c_ubyte, size)

    def send(self, func):
        self.s_e.acquire()
        self.s_buf.acquire()
        send_result = func(self.s_buf._obj)
        self.s_buf.release()
        self.s_f.release()
        return send_result

    def recv(self, func):
        self.s_f.acquire()
        self.s_buf.acquire()
        recv_result = func(self.s_buf._obj)
        self.s_buf.release()
        self.s_e.release()
        return recv_result

class ForkingWorker(BaseWorker):

    def __init__(self, num_processes=1):
        # Set up sync primitives, to communicate with the spawned children
        self._semaphore = Semaphore(num_processes)
        self._slots = Array('i', [0] * num_processes)

    def spawn_child(self):
        """Forks and executes the job."""
        self._semaphore.acquire()    # responsible for the blocking

        # Select an empty slot from self._slots (the first 0 value is picked)
        # The implementation guarantees there will always be at least one empty slot
        for slot, value in enumerate(self._slots):
            if value == 0:
                break

        # The usual hardcore forking action
        child_pid = os.fork()
        if child_pid == 0:
            # Within child

            # Disable signal handlers
            signal.signal(signal.SIGINT, signal.SIG_IGN)
            signal.signal(signal.SIGTERM, signal.SIG_IGN)

            random.seed()
            try:
                self.fake_work()
            finally:
                # This is the new stuff.  Remember, we're in the child process
                # currently. When all work is done here, free up the current
                # slot (by writing a 0 in the slot position).  This
                # communicates to the parent that the current child has died
                # (so can safely be forgotten about).
                self._slots[slot] = 0
                self._semaphore.release()
                os._exit(0)
        else:
            # Within parent, keep track of the new child by writing its PID
            # into the first free slot index.
            self._slots[slot] = child_pid

    def wait_for_children(self):
        for child_pid in self._slots:
            if child_pid != 0:
                os.waitpid(child_pid, 0)

    def get_id(self):
        return os.getpid()

 def disconnect(self, priority):
     sem = Semaphore()
     sem.acquire()
     if not self.getPeer().isConnected():
         return
     
     network = self.getPeer().getNetwork()
     neighbors = network.getNeighbors(self.getPeer())
     if len(neighbors) > 0:
         for n in neighbors:
             network.removeConnection(self.getPeer(), n)
             self.getPeer().disconnected()
     else:
         self.getPeer().disconnected()