Python multiprocessing.active_children函数代码示例

    def stop_processes(self):
        """Iterate through all of the consumer processes shutting them down."""
        self._set_state(self.STATE_SHUTTING_DOWN)
        LOGGER.debug('Stopping consumer processes')
        self._stop_timers()

        active_processes = multiprocessing.active_children()

        # Stop if we have no running consumers
        if not active_processes:
            LOGGER.info('All consumer processes have stopped')
            return self._set_state(self.STATE_STOPPED)

        # Iterate through all of the bindings and try and shutdown processes
        for process in active_processes:
            self._stop_process(process)

        # Wait for them to shutdown cleanly
        time.sleep(2)

        iterations = 0
        while multiprocessing.active_children():
            LOGGER.debug('Waiting on %i active processes to shut down',
                         self.total_process_count)
            time.sleep(1)
            iterations += 1

            # If the shutdown process waited long enough, kill the consumers
            if iterations == self._MAX_SHUTDOWN_WAIT:
                self._kill_processes()
                break

        LOGGER.debug('All consumer processes stopped')
        self._set_state(self.STATE_STOPPED)

def stop(processes, stop_event):
  """Stop queuing processes."""
  stop_event.set()
  for p in processes:
    if isinstance(p, mp.Process):
      p.terminate()
  mp.active_children()  # Use to join the killed processes above.

def brutePasses(userlist,passlist,hosti,pathi,porti,securei,psize,loginsi):
	global host
	host = hosti
	global port
	port = porti
	global secure
	secure = securei
	global logins
	logins = loginsi
	global path
	path = pathi
	global usernames
	usernames = userlist
	usersize = len(userlist)
	passsize = len(passlist)
	
	# manage pool
	if (psize == 0):	psize = 5
	if ((usersize*passsize) <= psize):	chunksize = 1
	else:	chunksize = (((usersize*passsize) / psize) + ((usersize*passsize) % psize))
	#print("%s" % ((ceil(float((usersize*passsize)) / psize)) + ((usersize*passsize) % psize)))
	print("Userlist size: %d\tPassword size: %d\tChunk size: %d\tPool size: %d" % (usersize,passsize,chunksize,psize))
	pool = Pool(processes=psize)
        for chunk in itertools.izip(grouper(passlist,chunksize)):  pool.map_async(worker,chunk)
        pool.close()
        try:
                while(len(active_children()) > 0): # how many active children do we have
                        sleep(2)
                        ignore = active_children()
        except KeyboardInterrupt:       exit('CTRL^C caught, exiting...\n\n')
	print("Password bruteforce attempts completed")

    def execute(self, *args, **options):
        if options['start-over']:
            init_db_for_crawl(DB_CONN_URL)

        num2crawl = options['count']
        interval = options['interval']
        if options['url']:
            # run once, test mode (no writeback)
            p = StoreDetailCrawler(interval=interval, count=num2crawl, writeback=False, url=options['url'])
            p.start()
            p.join()
        else:
            # run pool (could have pool size of 1)
            pool_size = options['pool-size']
            while (len(multiprocessing.active_children())<pool_size and not completed):
                num = pool_size - len(multiprocessing.active_children())
                for i in range(num):
                    p = StoreDetailCrawler(count=num2crawl, interval=interval)
                    p.start()
                    time.sleep(2)

                while(len(multiprocessing.active_children())>=pool_size):
                    time.sleep(0.5)
                
                print 'COMPLETED: ',completed

def fork_it(args):
    threads = int(args.t)
    childs = int(args.f)
    len_hosts = len(HOSTLIST)

    print "[*] attacking %d target(s)\n" "[*] cracking up to %d hosts parallel\n" "[*] threads per host: %d" % (
        len_hosts,
        childs,
        threads,
    )

    i = 1
    for host in HOSTLIST:
        host = host.replace("\n", "")
        print "[*] performing attacks against %s [%d/%d]" % (host, i, len_hosts)
        hostfork = multiprocessing.Process(target=thread_it, args=(host, args))
        hostfork.start()
        # checks that we have a max number of childs
        while len(multiprocessing.active_children()) >= childs:
            time.sleep(0.001)
        time.sleep(0.001)
        i += 1

    # waiting for child processes
    while multiprocessing.active_children():
        time.sleep(1)

    def GET(self, *args):
        """
        Inspect the chained requests of all chained campaigns, requires /all
        """
        if not args:
            return dumps({"results" : 'Error: No arguments were given'})
        if args[0] != 'all':
            return dumps({"results" : 'Error: Incorrect argument provided'})

        if len(multiprocessing.active_children()) < 1:  ##see if already running
            ccid_list = self.listAll()                  ##we run only 1 inspection
                              ## in background
            shuffle(ccid_list)
            try:
                p  = multiprocessing.Process(target = self.multiple_inspect,
                        args = (','.join( ccid_list ),))

                p.start()
                return dumps({"results" : True, "message":
                        "Successfully forked inspection to background. PID: %s"
                        % p.pid})

            except Exception as e:
                self.logger.error('Error while forking an inspection')
                self.logger.exception(e)
                return dumps({"results" : False,
                        message : "Failed in forking the process"})
        else:
            return dumps({"results" : True,
                    "message" : "Already running inspection in background. PID: %s"
                            %multiprocessing.active_children()[0].pid})

def bruteUser(userlist, psize, hosti, pathi, porti, securei, userfound):
    global host
    host = hosti
    global port
    port = porti
    global secure
    secure = securei
    global userout
    userout = userfound
    global path
    path = pathi
    f = open(userout, "w").close()
    usersize = len(userlist)
    # manage pool
    if usersize <= psize:
        chunksize = 1
    else:
        chunksize = (usersize / psize) + (usersize % psize)
    print("Userlist size: %d\tChunk size: %d\tPool size: %d" % (usersize, chunksize, psize))
    print("Bruteforcing usernames")
    pool = Pool(processes=psize)
    for chunk in itertools.izip(grouper(userlist, chunksize)):
        pool.map_async(worker, chunk)
    pool.close()
    try:
        while len(active_children()) > 0:  # how many active children do we have
            sleep(2)
            ignore = active_children()
    except KeyboardInterrupt:
        exit("CTRL^C caught, exiting...\n\n")
    print("Username bruteforce complete")

def find(mappe, cpu=6, minLengde=5):
    import os, time
    from multiprocessing import Process, Queue, active_children
    arbejdsliste = os.listdir(mappe)
    while arbejdsliste and len(active_children()):
        while len(active_children()) < cpu:
            #spawn child
        time.sleep(1)
    print 'Færdig med arbejdet!'
    return

def child(fil, mappe):
    #indlæs filen
    udfil = open(mappe +'/homopolymer-'+fil, 'wb')
    for sekvens in filen:
        for base in ('A', 'T', 'C', 'G'):
            ind = 0
            while 1:
                try:
                    homopolymer = sekvens[ind:].index(base)
                    udfil.write('>|' + str(homopolymer) + '|\n' + sekvens[homopolymer-100:homopolymer+100])
                    ind = homopolymer + 100
                except ValueError:
                    break
    udfil.close()
    return

if __name__ == "__main__":
    import sys
    find(sys.argv[1])
    exit()

def start_data_process(config, start_dt_arg, end_dt_arg):
    """Loop through the entity types and perform the main function """
    g_logger.info("Start processing data from %s to %s" %
                  (str(start_dt_arg), str(end_dt_arg)))

    processes = []
    for kind, fetch_intervals in config['kinds'].iteritems():
        # fetch_intervals is an array with format [int, int, bool, bool, str]
        # [Save interval, fetch interval, isMutable, is_ndb, json key]
        interval = dt.timedelta(seconds=int(fetch_intervals[0]))
        fetch_interval = fetch_intervals[1]
        start_dt = start_dt_arg
        end_dt = end_dt_arg
        while start_dt < end_dt:
            if len(active_children()) < config['max_threads']:
                next_dt = min(start_dt + interval, end_dt)
                p = Process(target=fetch_and_process_data,
                    args=(kind, start_dt, next_dt, fetch_interval, config))
                p.start()
                download_params = {"kind": kind, "start_dt": start_dt,
                                   "end_dt": next_dt, "start": time.time()}
                processes.append((p, download_params))
                start_dt = next_dt
            else:
                monitor(config, processes)
            # wait for 2 secs to space out the queries
            time.sleep(2)
    while len(active_children()) > 0:
        monitor(config, processes)
        time.sleep(10)

    def manager_thread_main(self):
        """ Checks for workers that died unexpectedly and listens to their
            status update messages.
        """
        while True:
            # wait up to 60 seconds
            try:
                command, argument = self.manager_thread_queue.get(True, 60)

                if command == 'quit':
                    break
                elif command == 'start':
                    with self._lock:
                        self.now_building[argument] = True
                elif command == 'end':
                    with self._lock:
                        self.now_building[argument] = False
                else:
                    logger.warn("Unknown command to manager thread: %s" % command)

            except Queue.Empty:
                pass

            # this one is to remove zombie processes
            multiprocessing.active_children()

            with self._lock:
                ids_to_restart = []
                for id, worker in self.workers.items():
                    if not worker.is_alive():
                        ids_to_restart.append(id)

                for id in ids_to_restart:
                    self.restart(id)

    def run_tasks(self, tasks):
        # Join whatever children are still sitting around
        multiprocessing.active_children()

        queue = multiprocessing.Queue()
        execute_ps = multiprocessing.Process(
            target=self._execute, args=[tasks, queue])
        execute_ps.start()

        def generate_response(execute_ps, queue):
            while execute_ps.is_alive():
                try:
                    data = queue.get_nowait()
                    yield data
                except Queue.Empty:
                    time.sleep(1)

            execute_ps.join()

            # suck the last goodness out of the queue before moving on
            while True:
                try:
                    data = queue.get_nowait()
                    yield data
                except Queue.Empty:
                    break

        return execute_ps, threadsafe_iter(
            generate_response(execute_ps, queue))

    def CheckIfStillUsed(self):
        import time
        if self.LoadItemFromSessionStore('status', 'processID') == None:
            return

        # if a new process ID is in the session data, another process was started and this process was abandoned
        if self.LoadItemFromSessionStore('status', 'processID') != os.getpid() and self.LoadItemFromSessionStore('status', 'processID') != 0:
            print "**** SMLRPP Exiting on process ID, session_status['processID'] = ", self.LoadItemFromSessionStore('status', 'processID'), " os.getpid() = ", os.getpid()
            sys.stdout.flush()

            time.sleep(1.0)
            if self.pool:
                self.pool.close()
                self.pool.join()
                self.pool = None
            for p in multiprocessing.active_children():
                p.terminate()
            os._exit(0) # kills pool processes

        # if the status has not been checked in the past 30 seconds, this process was abandoned
        if (time.time() - self.LoadItemFromSessionStore('status', 'time_of_last_status_check')) > 300:
            print "**** SMLRPP Exiting on time of last status check"
            sys.stdout.flush()

            time.sleep(1.0)
            if self.pool:
                self.pool.close()
                self.pool.join()
                self.pool = None
            for p in multiprocessing.active_children():
                p.terminate()
            os._exit(0) # kills pool processes

    def shutdown(self, c):
        '''
        Shutdown this process
        '''
        try:
            try:
                util.debug('manager received shutdown message')
                c.send(('#RETURN', None))

                if sys.stdout != sys.__stdout__:
                    util.debug('resetting stdout, stderr')
                    sys.stdout = sys.__stdout__
                    sys.stderr = sys.__stderr__

                util._run_finalizers(0)

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.terminate()

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.join()

                util._run_finalizers()
                util.info('manager exiting with exitcode 0')
            except:
                import traceback
                traceback.print_exc()
        finally:
            exit(0)

    def shutdown(self):
        BuiltinCore.shutdown(self)
        self.logger.info("Closing RPC command queues")
        self.rpc_q.close()

        def term_children():
            """ Terminate all remaining multiprocessing children. """
            for child in multiprocessing.active_children():
                self.logger.error("Waited %s seconds to shut down %s, "
                                  "terminating" % (self.shutdown_timeout,
                                                   child.name))
                child.terminate()

        timer = threading.Timer(self.shutdown_timeout, term_children)
        timer.start()
        while len(multiprocessing.active_children()):
            self.logger.info("Waiting for %s child(ren): %s" %
                             (len(multiprocessing.active_children()),
                              [c.name
                               for c in multiprocessing.active_children()]))
            time.sleep(1)
        timer.cancel()
        self.logger.info("All children shut down")

        while len(threading.enumerate()) > 1:
            threads = [t for t in threading.enumerate()
                       if t != threading.current_thread()]
            self.logger.info("Waiting for %s thread(s): %s" %
                             (len(threads), [t.name for t in threads]))
            time.sleep(1)
        self.logger.info("Shutdown complete")

    def ValidateProxies(self, proxyList):
            
        maxProc = 50
        
        tests = ["http://www.baidu.com"]
    
        result = Queue()
       
        start = time.clock()
        
        for i in proxyList:
            p = Process(target=self.CheckProxy, args=(i, tests, result))
            p.start()  
            
            if len(multiprocessing.active_children()) > maxProc:
                #print('active_children: ', multiprocessing.active_children())
                p.join()
            
        while len(multiprocessing.active_children()) > 0:
            time.sleep(3)
        end = time.clock()
        #print("total time for validation:", end - start, "s")
        
        self.pool = []
        
        for i in range(result.qsize()):
            a = result.get()
            self.pool += [Proxy(a[0], a[1])]

        
        print("{0} validated".format(len(self.pool)))