Python GreenPool.imap方法代码示例

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
    def imap(self, func, *args):
        reqid = request_id()

        def impl(*args):
            set_request_id(reqid)
            return func(*args)

        return GreenPool.imap(self, impl, *args)

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
def update_macmap(configmanager):
    """Interrogate switches to build/update mac table

    Begin a rebuild process.  This process is a generator that will yield
    as each switch interrogation completes, allowing a caller to
    recheck the cache as results become possible, rather
    than having to wait for the process to complete to interrogate.
    """
    global _macmap
    global _nodesbymac
    global _switchportmap
    # Clear all existing entries
    _macmap = {}
    _nodesbymac = {}
    _switchportmap = {}
    if configmanager.tenant is not None:
        raise exc.ForbiddenRequest('Network topology not available to tenants')
    nodelocations = configmanager.get_node_attributes(
        configmanager.list_nodes(), ('hardwaremanagement.switch',
                                     'hardwaremanagement.switchport'))
    switches = set([])
    for node in nodelocations:
        cfg = nodelocations[node]
        if 'hardwaremanagement.switch' in cfg:
            curswitch = cfg['hardwaremanagement.switch']['value']
            switches.add(curswitch)
            if 'hardwaremanagement.switchport' in cfg:
                portname = cfg['hardwaremanagement.switchport']['value']
                if curswitch not in _switchportmap:
                    _switchportmap[curswitch] = {}
                if portname in _switchportmap[curswitch]:
                    log.log({'warning': 'Duplicate switch topology config for '
                                        '{0} and {1}'.format(node,
                                                             _switchportmap[
                                                                 curswitch][
                                                                 portname])})
                _switchportmap[curswitch][portname] = node
    switchcfg = configmanager.get_node_attributes(
        switches, ('secret.hardwaremanagementuser',
                   'secret.hardwaremanagementpassword'), decrypt=True)
    switchauth = []
    for switch in switches:
        password = 'public'
        user = None
        if (switch in switchcfg and
                'secret.hardwaremanagementpassword' in switchcfg[switch]):
            password = switchcfg[switch]['secret.hardwaremanagementpassword'][
                'value']
            if 'secret.hardwaremanagementuser' in switchcfg[switch]:
                user = switchcfg[switch]['secret.hardwaremanagementuser'][
                    'value']
        switchauth.append((switch, password, user))
    pool = GreenPool()
    for res in pool.imap(_map_switch, switchauth):
        yield res
        print(repr(_macmap))

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
def imap(requests, prefetch=True, size=2):
    """Concurrently converts a generator object of Requests to
    a generator of Responses.

    :param requests: a generator of Request objects.
    :param prefetch: If False, the content will not be downloaded immediately.
    :param size: Specifies the number of requests to make at a time. default is 2
    """

    def send(r):
        r.send(prefetch)
        return r.response

    pool = GreenPool(size)
    for r in pool.imap(send, requests):
        yield r
    pool.waitall()

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
def build_main(commits, config):
    logger = logging.getLogger(__name__ + '.build_main')
    logger.info('triggered with %d commits', len(commits))
    logger.debug('commits = %r', commits)
    token = get_token(config)
    pool = GreenPool()
    commit_map = dict(commits)
    commit_ids = [commit_id for commit_id, _ in commits]
    results = pool.imap(
        functools.partial(download_archive, token=token, config=config),
        commit_ids
    )
    env = make_virtualenv(config)
    save_dir = config['SAVE_DIRECTORY']
    complete_hook = config.get('COMPLETE_HOOK')
    for commit, filename in results:
        working_dir = extract(filename, save_dir)
        try:
            build = build_sphinx(working_dir, env)
        except Exception:
            permalink = commit_map[commit]
            if not config.get('RECREATE_VIRTUALENV'):
                env = make_virtualenv(config, recreate=True)
                try:
                    build = build_sphinx(working_dir, env)
                except Exception:
                    if callable(complete_hook):
                        complete_hook(commit, permalink, sys.exc_info())
                    continue
            if callable(complete_hook):
                complete_hook(commit, permalink, sys.exc_info())
            continue
        result_dir = os.path.join(save_dir, commit)
        shutil.move(build, result_dir)
        logger.info('build complete: %s' % result_dir)
        if callable(complete_hook):
            complete_hook(commit, commit_map[commit], None)
        shutil.rmtree(working_dir)
        logger.info('working directory %s has removed' % working_dir)
        with open_head_file('w', config=config) as f:
            f.write(commit)
    logger.info('new head: %s', commits[0])

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
def run_stock_parser():
    stock_symbols = []
    with open('symbols.txt', 'r') as symfile:
        for n, line in enumerate(symfile):
            sym = line.strip()
            if sym:
                stock_symbols.append(sym)

    ncpu = cpu_count()

    pool = GreenPool(ncpu * 4)

    stock_prices = []
    for symbol, price in pool.imap(read_stock_url, stock_symbols):
        stock_prices.append((symbol, price))

    with open('stock_prices.csv', 'w') as outfile:
        outfile.write('Stock,Price\n')
        for symbol, price in stock_prices:
            outfile.write('%s,%s\n' % (symbol, price))

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
def download(files, settings):
    temp_dir = settings['temp_dir']
    download_path = settings['download_path']

    download = DownloadPool(settings)
    decode = Decode()
    pool = GreenPool(settings['connections'])
    progress_tracker = greenthread.spawn(show_progress)

    for file_ in files:
        # Check if file from subject exists.
        subject_filename = helper.get_filename_from(file_['file_subject'])

        if helper.file_exists(download_path, subject_filename):
            Tracker.total_size -= sum([i['segment_bytes'] for i in file_['segments']])
            print subject_filename, 'already exists'
            continue

        # Download.
        for segment_path in pool.imap(download.download, file_['segments']):
            # Decode.
            if segment_path:
                tpool.execute(decode.decode, segment_path, temp_dir, download_path)

    if decode.tracker:
        print 'have broken files...'
        #return False
        print decode.tracker
        broken_files = decode.tracker.keys()
        for fname in broken_files:
            #print 'decoding', fname
            decode.join_files(fname, temp_dir, download_path)

    #progress_tracker.kill()
    progress_tracker.wait()

    # All OK.
    return 0

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
class WorkerManager(threading.Thread):
    """
    Methods of this class must be thread safe unless explicitly stated.
    """
    daemon = True

    def __init__(self, game_state, users_url):
        """

        :param thread_pool:
        """
        self._data = _WorkerManagerData(game_state, {})
        self.users_url = users_url
        self._pool = GreenPool(size=3)
        super(WorkerManager, self).__init__()

    def get_code(self, player_id):
        return self._data.get_code(player_id)

    def get_persistent_state(self, player_id):
        """Get the persistent state for a worker."""

        return None

    def create_worker(self, player_id):
        """Create a worker."""

        raise NotImplemented

    def remove_worker(self, player_id):
        """Remove a worker for the given player."""

        raise NotImplemented

    # TODO handle failure
    def spawn(self, user):
        # Get persistent state from worker
        persistent_state = self.get_persistent_state(user['id'])

        # Kill worker
        LOGGER.info("Removing worker for user %s" % user['id'])
        self.remove_worker(user['id'])

        self._data.set_code(user)

        # Spawn worker
        LOGGER.info("Spawning worker for user %s" % user['id'])
        worker_url = self.create_worker(user['id'])

        # Add avatar back into game
        self._data.add_avatar(user, worker_url)
        LOGGER.info('Added user %s', user['id'])

    def _parallel_map(self, func, iterable_args):
        list(self._pool.imap(func, iterable_args))

    def update(self):
        try:
            LOGGER.info("Waking up")
            game_data = requests.get(self.users_url).json()
        except (requests.RequestException, ValueError) as err:
            LOGGER.error("Failed to obtain game data : %s", err)
        else:
            game = game_data['main']

            # Remove users with different code
            users_to_add = []
            for user in game['users']:
                if self._data.remove_user_if_code_is_different(user):
                    users_to_add.append(user)
            LOGGER.debug("Need to add users: %s" % [x['id'] for x in users_to_add])

            # Add missing users
            self._parallel_map(self.spawn, users_to_add)

            # Delete extra users
            known_avatars = set(user['id'] for user in game['users'])
            removed_user_ids = self._data.remove_unknown_avatars(known_avatars)
            LOGGER.debug("Removing users: %s" % removed_user_ids)
            self._parallel_map(self.remove_worker, removed_user_ids)

    def run(self):
        while True:
            self.update()
            LOGGER.info("Sleeping")
            time.sleep(10)

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
def parallel_select(collection, func):
    gp = GreenPool()
    return list(gp.imap(func, collection))

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
import httplib2 directly, httplib2 doesn't natively support cooperative 
yielding. 

monkey_patch httplib2 will make httplib2 green.
'''

from eventlet.greenpool import GreenPool
import eventlet
import random

import httplib2

# uncomment this line to green httplib2
# httplib2 = eventlet.import_patched('httplib2')

def worker(url):
    print "worker "+str(random.random())
    h = httplib2.Http()
    resp, content = h.request(url, "GET")
    return resp


pool = GreenPool(size=10)
results = pool.imap(worker, open("urls.txt", 'r'))

for result in results:
    print result
print "done...."
pool.waitall()

# 需要导入模块: from eventlet.greenpool import GreenPool [as 别名]
# 或者: from eventlet.greenpool.GreenPool import imap [as 别名]
class WorkerManager(object):
    """
    Methods of this class must be thread safe unless explicitly stated.
    """
    __metaclass__ = ABCMeta
    daemon = True

    def __init__(self, games_url):
        """

        :param thread_pool:
        """
        self._data = _WorkerManagerData()
        self.games_url = games_url
        self._pool = GreenPool(size=3)
        super(WorkerManager, self).__init__()

    def get_persistent_state(self, player_id):
        """Get the persistent state for a worker."""

        return None

    @abstractmethod
    def create_worker(self, player_id):
        """Create a worker."""

        raise NotImplemented

    @abstractmethod
    def remove_worker(self, player_id):
        """Remove a worker for the given player."""

        raise NotImplemented

    # TODO handle failure
    def spawn(self, game_id, game_data):
        # Kill worker
        LOGGER.info("Removing worker for game %s" % game_data['name'])
        self.remove_worker(game_id)

        # Spawn worker
        LOGGER.info("Spawning worker for game %s" % game_data['name'])
        game_data['GAME_API_URL'] = '{}{}/'.format(self.games_url, game_id)
        self.create_worker(game_id, game_data)

    def _parallel_map(self, func, *iterable_args):
        list(self._pool.imap(func, *iterable_args))

    def run(self):
        while True:
            self.update()
            LOGGER.info("Sleeping")
            time.sleep(10)

    def update(self):
        try:
            LOGGER.info("Waking up")
            games = requests.get(self.games_url).json()
        except (requests.RequestException, ValueError) as err:
            LOGGER.error("Failed to obtain game data : %s", err)
        else:
            games_to_add = {id: games[id]
                            for id in self._data.add_new_games(games.keys())}
            LOGGER.debug("Need to add games: %s" % games_to_add)

            # Add missing games
            self._parallel_map(self.spawn, games_to_add.keys(), games_to_add.values())

            # Delete extra games
            known_games = set(games.keys())
            removed_user_ids = self._data.remove_unknown_games(known_games)
            LOGGER.debug("Removing users: %s" % removed_user_ids)
            self._parallel_map(self.remove_worker, removed_user_ids)