Python BoundedSemaphore.acquire方法代码示例

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class XCATMessager(utils.Messager):
    def __init__(self, sock):
        self.sock = sock
        self.sem = BoundedSemaphore(1)

    def _send(self, d):
        buf = json.dumps(d)
        self.sem.acquire()
        self.sock.sendall(utils.int2bytes(len(buf)) + buf.encode('utf-8'))
        self.sem.release()

    def info(self, msg):
        d = {'type': MSG_TYPE, 'msg': {'type': 'info', 'data': msg}}
        self._send(d)

    def warn(self, msg):
        d = {'type': MSG_TYPE, 'msg': {'type': 'warning', 'data': msg}}
        self._send(d)

    def error(self,  msg, node=''):
        d = {'type': MSG_TYPE, 'msg': {'type': 'error', 'node': node, 'data': msg}}
        self._send(d)

    def syslog(self, msg):
        d = {'type': MSG_TYPE, 'msg': {'type': 'syslog', 'data': msg}}
        self._send(d)

    def info_with_host(self, msg):
        d = {'type': MSG_TYPE, 'msg': {'type': 'info_with_host', 'data': msg}}
        self._send(d)

    def update_node_attributes(self, attribute, node, data):
        d = {'type': DB_TYPE, 'attribute': {'name': attribute, 'method': 'set', 'type': 'node', 'node': node, 'value': data}}
        self._send(d)

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class _LeakySemaphore(object):
    def __init__(self, value = 1, maxSeconds = 10):
        self._semaphore = BoundedSemaphore(value)
        self._maxSeconds = maxSeconds
        self._timer = None
        self._leaked = 0
        self._stopped = False

    def _leak(self):
        sleep(self._maxSeconds)
        self._leaked += 1
        self._semaphore.release()

    @property
    def inUse(self):
        return self._semaphore._initial_value - self.semaphore.counter

    @property
    def waiting(self):
        return len(self._semaphore._links)

    def release(self):
        if self._stopped:
            return
        if self._leaked > 0:
            self._leaked -= 1
        else:
            self._semaphore.release()

    def stop(self):
        self._stopped = True

        if self._timer is not None:
            self._timer.kill(block = False)
            self._timer = None

        while self.waiting > 0:
            self._semaphore.release()
            sleep(0.1)

    def acquire(self):
        if self._stopped:
            return
        if self._semaphore.locked() and not self._timer:
            self._timer = spawn(self._leak)
        self._semaphore.acquire(blocking = True, timeout = None)
        if self._timer is not None:
            self._timer.kill(block = False)
            self._timer = None
            if self.waiting > 0:
                self._timer = spawn(self._leak)

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class RWLock( object ):
    def __init__( self ):
        self._canRead = Event()
        self._canWrite = Event()
        self._mutex = BoundedSemaphore( value = 1 )
        self._readers = 0
        self._isWriting = False
        self._canRead.set()
        self._canWrite.set()

    def rLock( self ):
        isReady = False
        while not isReady:
            self._canRead.wait()
            self._mutex.acquire( blocking = True, timeout = None )
            if not self._isWriting:
                self._canWrite.clear()
                self._readers += 1
                isReady = True
            self._mutex.release()

    def rUnlock( self ):
        self._mutex.acquire( blocking = True, timeout = None )
        self._readers -= 1
        if 0 == self._readers:
            self._canWrite.set()
        self._mutex.release()

    def wLock( self ):
        isReady = False
        while not isReady:
            self._canRead.clear()
            self._canWrite.wait()
            self._mutex.acquire( blocking = True, timeout = None )
            if not self._isWriting and 0 == self._readers:
                isReady = True
                self._isWriting = True
                self._canWrite.clear()
            self._mutex.release()

    def wUnlock( self ):
        self._mutex.acquire( blocking = True, timeout = None )
        self._isWriting = False
        self._canWrite.set()
        self._canRead.set()
        self._mutex.release()

    def writer( self ):
        return _rwlock_w( self )

    def reader( self ):
        return _rwlock_r( self )

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
    def test_concurrent_batching(self):
        lock = BoundedSemaphore(1)
        lock.acquire()  # now 0

        N_CALLS = [0]

        @batched()
        def fn(arg_list):
            N_CALLS[0] += 1
            lock.acquire()

        @batched()
        def fn2(arg_list):
            N_CALLS[0] += 1
            lock.release()

        @batch_context
        def test():
            a, b = spawn(fn), spawn(fn2)
            self.assertEquals(0, N_CALLS[0])
            a.get(), b.get()

        test()  # shouldn't hang.

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
        def acquire(self, blocking=True, timeout=-1):
            # Transform the default -1 argument into the None that our
            # semaphore implementation expects, and raise the same error
            # the stdlib implementation does.
            if timeout == -1:
                timeout = None
            if not blocking and timeout is not None:
                raise ValueError("can't specify a timeout for a non-blocking call")
            if timeout is not None:
                if timeout < 0:
                    # in C: if(timeout < 0 && timeout != -1)
                    raise ValueError("timeout value must be strictly positive")
                if timeout > self._TIMEOUT_MAX:
                    raise OverflowError('timeout value is too large')

            return BoundedSemaphore.acquire(self, blocking, timeout)

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class CrispinConnectionPool(object):
    """
    Connection pool for Crispin clients.

    Connections in a pool are specific to an IMAPAccount.

    Parameters
    ----------
    account_id : int
        Which IMAPAccount to open up a connection to.
    num_connections : int
        How many connections in the pool.
    readonly : bool
        Is the connection to the IMAP server read-only?
    """

    def __init__(self, account_id, num_connections, readonly):
        log.info('Creating Crispin connection pool for account {} with {} '
                 'connections'.format(account_id, num_connections))
        self.account_id = account_id
        self.readonly = readonly
        self._queue = Queue(num_connections, items=num_connections * [None])
        self._sem = BoundedSemaphore(num_connections)
        self._set_account_info()

    @contextlib.contextmanager
    def get(self):
        """ Get a connection from the pool, or instantiate a new one if needed.
        If `num_connections` connections are already in use, block until one is
        available.
        """
        # A gevent semaphore is granted in the order that greenlets tried to
        # acquire it, so we use a semaphore here to prevent potential
        # starvation of greenlets if there is high contention for the pool.
        # The queue implementation does not have that property; having
        # greenlets simply block on self._queue.get(block=True) could cause
        # individual greenlets to block for arbitrarily long.
        self._sem.acquire()
        client = self._queue.get()
        try:
            if client is None:
                client = self._new_connection()
            yield client
        except CONN_DISCARD_EXC_CLASSES as exc:
            # Discard the connection on socket or IMAP errors. Technically this
            # isn't always necessary, since if you got e.g. a FETCH failure you
            # could reuse the same connection. But for now it's the simplest
            # thing to do.
            log.info('IMAP connection error; discarding connection',
                     exc_info=True)
            if client is not None and \
               not isinstance(exc, CONN_UNUSABLE_EXC_CLASSES):
                try:
                    client.logout()
                except Exception:
                    log.info('Error on IMAP logout', exc_info=True)
            client = None
            raise exc
        except:
            raise
        finally:
            self._queue.put(client)
            self._sem.release()

    def _set_account_info(self):
        with session_scope(self.account_id) as db_session:
            account = db_session.query(ImapAccount).get(self.account_id)
            self.sync_state = account.sync_state
            self.provider = account.provider
            self.provider_info = account.provider_info
            self.email_address = account.email_address
            self.auth_handler = account.auth_handler
            if account.provider == 'gmail':
                self.client_cls = GmailCrispinClient
            else:
                self.client_cls = CrispinClient

    def _new_raw_connection(self):
        """Returns a new, authenticated IMAPClient instance for the account."""
        with session_scope(self.account_id) as db_session:
            if self.provider == 'gmail':
                account = db_session.query(GmailAccount).options(
                    joinedload(GmailAccount.auth_credentials)).get(
                    self.account_id)
            else:
                account = db_session.query(GenericAccount).options(
                    joinedload(GenericAccount.imap_secret)).get(self.account_id)
            db_session.expunge(account)

        return self.auth_handler.connect_account(account)

    def _new_connection(self):
        conn = self._new_raw_connection()
        return self.client_cls(self.account_id, self.provider_info,
                               self.email_address, conn,
                               readonly=self.readonly)

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class EchoNode:

    def __init__(self, api, token_address):
        assert isinstance(api, RaidenAPI)
        self.ready = Event()

        self.api = api
        self.token_address = token_address

        existing_channels = self.api.get_channel_list(
            api.raiden.default_registry.address,
            self.token_address,
        )

        open_channels = [
            channel_state
            for channel_state in existing_channels
            if channel.get_status(channel_state) == CHANNEL_STATE_OPENED
        ]

        if len(open_channels) == 0:
            token = self.api.raiden.chain.token(self.token_address)
            if not token.balance_of(self.api.raiden.address) > 0:
                raise ValueError('not enough funds for echo node %s for token %s' % (
                    pex(self.api.raiden.address),
                    pex(self.token_address),
                ))
            self.api.token_network_connect(
                self.api.raiden.default_registry.address,
                self.token_address,
                token.balance_of(self.api.raiden.address),
                initial_channel_target=10,
                joinable_funds_target=.5,
            )

        self.last_poll_offset = 0
        self.received_transfers = Queue()
        self.stop_signal = None  # used to signal REMOVE_CALLBACK and stop echo_workers
        self.greenlets = list()
        self.lock = BoundedSemaphore()
        self.seen_transfers = deque(list(), TRANSFER_MEMORY)
        self.num_handled_transfers = 0
        self.lottery_pool = Queue()
        # register ourselves with the raiden alarm task
        self.api.raiden.alarm.register_callback(self.echo_node_alarm_callback)
        self.echo_worker_greenlet = gevent.spawn(self.echo_worker)
        log.info('Echo node started')

    def echo_node_alarm_callback(self, block_number):
        """ This can be registered with the raiden AlarmTask.
        If `EchoNode.stop()` is called, it will give the return signal to be removed from
        the AlarmTask callbacks.
        """
        if not self.ready.is_set():
            self.ready.set()
        log.debug('echo_node callback', block_number=block_number)
        if self.stop_signal is not None:
            return REMOVE_CALLBACK
        else:
            self.greenlets.append(gevent.spawn(self.poll_all_received_events))
            return True

    def poll_all_received_events(self):
        """ This will be triggered once for each `echo_node_alarm_callback`.
        It polls all channels for `EventPaymentReceivedSuccess` events,
        adds all new events to the `self.received_transfers` queue and
        respawns `self.echo_node_worker`, if it died. """

        locked = False
        try:
            with Timeout(10):
                locked = self.lock.acquire(blocking=False)
                if not locked:
                    return
                else:
                    received_transfers = self.api.get_raiden_events_payment_history(
                        token_address=self.token_address,
                        offset=self.last_poll_offset,
                    )

                    # received transfer is a tuple of (block_number, event)
                    received_transfers = [
                        event
                        for event in received_transfers
                        if type(event) == EventPaymentReceivedSuccess
                    ]

                    for event in received_transfers:
                        transfer = copy.deepcopy(event)
                        self.received_transfers.put(transfer)

                    # set last_poll_block after events are enqueued (timeout safe)
                    if received_transfers:
                        self.last_poll_offset += len(received_transfers)

                    if not self.echo_worker_greenlet.started:
                        log.debug(
                            'restarting echo_worker_greenlet',
                            dead=self.echo_worker_greenlet.dead,
                            successful=self.echo_worker_greenlet.successful(),
#.........这里部分代码省略.........

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class CrispinConnectionPool(object):
    """
    Connection pool for Crispin clients.

    Connections in a pool are specific to an IMAPAccount.

    Parameters
    ----------
    account_id : int
        Which IMAPAccount to open up a connection to.
    num_connections : int
        How many connections in the pool.
    readonly : bool
        Is the connection to the IMAP server read-only?
    """
    def __init__(self, account_id, num_connections, readonly):
        log.info('Creating Crispin connection pool for account {} with {} '
                 'connections'.format(account_id, num_connections))
        self.account_id = account_id
        self.readonly = readonly
        self._queue = Queue(num_connections, items=num_connections * [None])
        self._sem = BoundedSemaphore(num_connections)
        self._set_account_info()

    @contextlib.contextmanager
    def get(self):
        """ Get a connection from the pool, or instantiate a new one if needed.
        If `num_connections` connections are already in use, block until one is
        available.
        """
        # A gevent semaphore is granted in the order that greenlets tried to
        # acquire it, so we use a semaphore here to prevent potential
        # starvation of greenlets if there is high contention for the pool.
        # The queue implementation does not have that property; having
        # greenlets simply block on self._queue.get(block=True) could cause
        # individual greenlets to block for arbitrarily long.
        self._sem.acquire()
        client = self._queue.get()
        try:
            if client is None:
                client = self._new_connection()
            yield client
        except CONN_DISCARD_EXC_CLASSES as exc:
            # Discard the connection on socket or IMAP errors. Technically this
            # isn't always necessary, since if you got e.g. a FETCH failure you
            # could reuse the same connection. But for now it's the simplest
            # thing to do.
            log.info('IMAP connection error; discarding connection',
                     exc_info=True)
            if client is not None:
                try:
                    client.logout()
                except:
                    log.error('Error on IMAP logout', exc_info=True)
                client = None
            raise exc
        except:
            raise
        finally:
            self._queue.put(client)
            self._sem.release()

    def _set_account_info(self):
        with session_scope() as db_session:
            account = db_session.query(Account).get(self.account_id)
            self.sync_state = account.sync_state
            self.provider_info = account.provider_info
            self.email_address = account.email_address
            self.auth_handler = account.auth_handler
            if account.provider == 'gmail':
                self.client_cls = GmailCrispinClient
            elif (getattr(account, 'supports_condstore', None) or
                  account.provider_info.get('condstore')):
                self.client_cls = CondStoreCrispinClient
            else:
                self.client_cls = CrispinClient

    def _new_connection(self):
        try:
            with session_scope() as db_session:
                account = db_session.query(Account).get(self.account_id)
                conn = self.auth_handler.connect_account(account)
                # If we can connect the account, then we can set the state
                # to 'running' if it wasn't already
                if self.sync_state != 'running':
                    self.sync_state = account.sync_state = 'running'
            return self.client_cls(self.account_id, self.provider_info,
                                   self.email_address, conn,
                                   readonly=self.readonly)
        except ValidationError, e:
            log.error('Error validating',
                      account_id=self.account_id,
                      logstash_tag='mark_invalid')
            with session_scope() as db_session:
                account = db_session.query(Account).get(self.account_id)
                account.mark_invalid()
                account.update_sync_error(str(e))
            raise

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
                    decoded['p']['v'] = Computation(decoded['p']['v'])
                except Exception as e:
                    decoded['p']['v'] = Nil
                    decoded['p']['h'] = True
                    decoded['p']['e'] = str(e) #''.join(traceback.format_exception( *sys.exc_info())[-2:]).strip().replace('\n',': ')
            encoded = json.dumps(decoded['p'])
            #print(resultKey)
            r.set(resultKey + decoded['i'], encoded)
            r.hdel(pipeline, pipelineId)
            #print(encoded)
            release()
            gevent.sleep(0)
            break #finish loop
        else:
            #print("No messages " + str(getcurrent()))
            notification_semaphore.acquire(timeout = 0.1)
            gevent.sleep(0)
    
while True:
    if(not concurrency_semaphore.acquire(timeout = 0.1)):
         #print("cannot acquire semaphore")
        pass
    else:
        #print("acquired concurrency semaphore")
        gevent.Greenlet.spawn(process)
        

        
        

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class ConnectionPool(object):
    """dynamic service connection pool"""

    def __init__(self, server_node, iface_cls, config):

        self._section_name = utils.get_module(__name__)
        self._logger = logging.getLogger(__name__)
        self._host = server_node.split(":")[0]
        self._port = int(server_node.split(":")[1])
        self._iface_cls = iface_cls

        self._get_conn_timeout = config.getint(self._section_name, "pool_timeout",
                                               default=settings.DEFAULT_POOL_TIMEOUT)
        self._socket_timeout = config.getint(self._section_name, "request_timeout",
                                             default=settings.DEFAULT_REQUEST_TIMEOUT) * 1000
        self._size = config.getint(self._section_name, "pool_size", default=settings.DEFAULT_POOL_SIZE)

        self._c_module_serialize = config.getboolean(self._section_name, "c_module_serialize",
                                                     default=settings.USE_C_MODULE_SERIALIZE)

        self._closed = False
        if ASYNC_TAG:
            from gevent.lock import BoundedSemaphore
            from gevent import queue as Queue
            self._semaphore = BoundedSemaphore(self._size)
            self._connection_queue = Queue.LifoQueue(self._size)
            self._QueueEmpty = Queue.Empty
        else:
            from threading import BoundedSemaphore
            import Queue
            self._semaphore = BoundedSemaphore(self._size)
            self._connection_queue = Queue.LifoQueue(self._size)
            self._QueueEmpty = Queue.Empty

    def close(self):
        self._closed = True
        while not self._connection_queue.empty():
            try:
                conn = self._connection_queue.get(block=False)
                try:
                    self._close_connection(conn)
                except:
                    pass
            except self._QueueEmpty:
                pass

    def _create_connection(self):
        self._logger.debug("create a new connection ip:%s port:%s" %(self._host, self._port))
        socket_ = TSocket.TSocket(self._host, self._port)
        if self._socket_timeout > 0:
            socket_.setTimeout(self._socket_timeout)
        transport = TTransport.TBufferedTransport(socket_)
        if self._c_module_serialize:
            protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
        else:
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
        connection = self._iface_cls(protocol)
        transport.open()
        return connection

    def _close_connection(self, conn):
        try:
            conn._iprot.trans.close()
        except:
            pass
        try:
            conn._oprot.trans.close()
        except:
            pass

    def get_connection(self):
        """ get a connection from the pool. This blocks until one is available."""
        self._semaphore.acquire()
        if self._closed:
            raise RuntimeError('connection pool closed')
        try:
            return self._connection_queue.get(block=False)
        except self._QueueEmpty:

            try:
                return self._create_connection()
            except Exception as e:
                self._semaphore.release()
                raise e

    def return_connection(self, conn):
        """ return a connection to the pool."""
        if self._closed:
            self._close_connection(conn)
            return
        self._connection_queue.put(conn)
        self._semaphore.release()

    def release_connection(self, conn):
        """ call when the connect is no usable anymore"""
        try:

            self._close_connection(conn)
        except:
            pass
#.........这里部分代码省略.........

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class ConnectionPool(object):
    """
    Generic TCP connection pool, with the following features:
        * Configurable pool size
        * Auto-reconnection when a broken socket is detected
        * Optional periodic keepalive
    """

    # Frequency at which the pool is populated at startup
    SPAWN_FREQUENCY = 0.1

    def __init__(self, size, exc_classes=DEFAULT_EXC_CLASSES, keepalive=None):
        self.size = size
        self.conn = deque()
        self.lock = BoundedSemaphore(size)
        self.keepalive = keepalive
        # Exceptions list must be in tuple form to be caught properly
        self.exc_classes = tuple(exc_classes)
        for i in iter(range(size)):
            self.lock.acquire()
        for i in iter(range(size)):
            gevent.spawn_later(self.SPAWN_FREQUENCY*i, self._addOne)
        if self.keepalive:
            gevent.spawn(self._keepalive_periodic)

    def _new_connection(self):
        """
        Estabilish a new connection (to be implemented in subclasses).
        """
        raise NotImplementedError

    def _keepalive(self, c):
        """
        Implement actual application-level keepalive (to be
        reimplemented in subclasses).

        :raise: socket.error if the connection has been closed or is broken.
        """
        raise NotImplementedError()

    def _keepalive_periodic(self):
        delay = float(self.keepalive) / self.size
        while 1:
            try:
                with self.get() as c:
                    self._keepalive(c)
            except self.exc_classes:
                # Nothing to do, the pool will generate a new connection later
                pass
            gevent.sleep(delay)

    def _addOne(self):
        stime = 0.1
        while 1:
            c = self._new_connection()
            if c:
                break
            gevent.sleep(stime)
            if stime < 400:
                stime *= 2

        self.conn.append(c)
        self.lock.release()

    @contextmanager
    def get(self):
        """
        Get a connection from the pool, to make and receive traffic.

        If the connection fails for any reason (socket.error), it is dropped
        and a new one is scheduled. Please use @retry as a way to automatically
        retry whatever operation you were performing.
        """
        self.lock.acquire()
        try:
            c = self.conn.popleft()
            yield c
        except self.exc_classes:
            # The current connection has failed, drop it and create a new one
            gevent.spawn_later(1, self._addOne)
            raise
        except:
            self.conn.append(c)
            self.lock.release()
            raise
        else:
            # NOTE: cannot use finally because MUST NOT reuse the connection
            # if it failed (socket.error)
            self.conn.append(c)
            self.lock.release()

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class LibMLK(object):
    def __init__(self, serverIP, Plist):
        self.serverIP = serverIP
        self.baseUrl = "http://%s/" % serverIP
        self.crypt = Account(Plist.userID, Plist.devideToken)
        self.mlkLock = BoundedSemaphore(200)
        self.IID = Plist.iid
        self.VID = Plist.vid


    def __init__(self):
        pass

    @property
    def headers(self):
        return {
            "VID": self.VID,
            "PID": "-",
            "IID": self.IID,
            "DEVICE_INFO": "iPad2,1:::iPhone OS 8.1.2",
            "Device": "ios",
            "AppVersion": 28,
            "APP_ID_3": self.crypt.deviceToken,
            "APP_ID_2": self.crypt.hashedUserID,
            "APP_ID_1": self.crypt.cryptedUserID,
            "Encrypted": True,
            "User-Agent": "toto/1.1.25.2 CFNetwork/711.1.16 Darwin/14.0.0",
            "Accept-Language": "zh-cn",
            "Accept": "application/json"
        }

    def _post(self, url, params={}, data={}):
        data["_method"] = "GET"
        data = urllib.urlencode(data)
        data = self.crypt.encrypt(data)
        url = urlparse.urljoin(self.baseUrl, url)
        if len(params) > 0:
            e = self.crypt.encrypt(urllib.urlencode(params)).encode("base64").replace("\n", "")
            url = "%s?e=%s" % (url, e)
        ret = None
        try:
            self.mlkLock.acquire()
            ret = requests.post(url, data=data, headers=self.headers, proxies=proxies)
        except:
            traceback.print_exc()
        finally:
            self.mlkLock.release()
        if ret is None:
            raise BaseException()
        if "encrypted" in ret.headers and ret.headers["encrypted"] == "true":
            rtn = self.crypt.decrypt(ret.content)
        else:
            rtn = ret.content
        return rtn

    def get(self, url, params={}, data={}):
        url = urlparse.urlparse(url)
        path = url.path
        query = dict(urlparse.parse_qsl(url.query))
        query.update(params)
        return self._post(path, params=query, data=data)

    def setUsername(self, name):
        ret = self._post("users/update", data={"user_name": name})
        self.user_name = name
        return json.loads(ret)

    def finishTutorial(self):
        ret = self._post("users/update",
                         data={"user_name": self.user_name, "tutorial_finish": True})
        return json.loads(ret)

    def getMessages(self, page_type="Home"):
        params = {
            "last_read_at": int(time.time()),
            "page_type": page_type
        }
        ret = self._post("users/messages", params=params)
        return json.loads(ret)

    def getStages(self):
        ret = self._post("stages")
        return json.loads(ret)

    def getAreas(self, stage_id):
        ret = self._post("areas", params={"stage_id": stage_id})
        return json.loads(ret)

    def getMonsters(self):
        ret = self._post("user_monsters")
        return json.loads(ret)

    def getDecks(self):
        ret = self._post("user_decks")
        return json.loads(ret)

    def getUnits(self):
        ret = self._post("user_units")
        return json.loads(ret)

#.........这里部分代码省略.........

# 需要导入模块: from gevent.lock import BoundedSemaphore [as 别名]
# 或者: from gevent.lock.BoundedSemaphore import acquire [as 别名]
class Master(object):
    """ workers: a dictionary to store the information about workers
        format: {ip_port: ('Status', Remote_call)}

        jobs_tracker: a dictionary to store the information about jobs
        format:
        { task_name:
              { "mappers":
                      { mapper_id: [remote_call, mapper_ip_port, split_info, finished]},
                "reducers":
                      { reducer_id: [remote_call, reducer_ip_port, finished]},
                "num_mapper":
                      num_mapper,
                "num_reducer":
                      num_reducer,
                "task_file":
                      [filename, codes],
                "split_infos":
                      split_infos,
                "file_info":
                      file_info,
                "output_file:
                      output_file
                "done":
                      True/False
                 }
        }

        mapper_queue: free mapper queue
        format: [(ip_port, remote_call)]

        reducer_queue: free reducer queue
        format: [(ip_port, remote_call)]
    """

    def __init__(self, port, data_dir):
        gevent.spawn(self.controller)
        self.state = STATE_READY
        self.workers = {}
        self.jobs_tracker = {}
        self.port = port
        self.data_dir = data_dir
        self.mapper_queue = Queue()
        self.reducer_queue = Queue()
        self.jobs_tracker_lock = BoundedSemaphore(1)
        self.workers_lock = BoundedSemaphore(1)

    def controller(self):
        while True:
            print '[Master:%s] ' % self.state
            # down_workers = []
            # self.workers_lock.acquire()
            local_workers = dict(self.workers)
            for w in local_workers:
                print '(%s, %s)' % (w, local_workers[w][0])
                # not spawn a coroutine to ping this worker
                if local_workers[w][2] == False:
                    local_workers[w][2] = True
                    gevent.spawn(self.each_ping, w, local_workers[w][1])
            gevent.sleep(1)

    def each_ping(self, ip_port, c):
        alive = True
        while alive:
            try:
                c.ping()
            except:
                print "**** Error: Worker %s is down" % ip_port
                self.workers.pop(ip_port)
                print "********** Reassign jobs in %s" % ip_port
                gevent.spawn(self.reassign_job, [ip_port])
                alive = False
            gevent.sleep(1)

    # Failure tolerance
    # reassign the failure worker's job to another worker
    # remote = (ip_port, c)
    # def assign_mapper(self, split_info, task_file,
    #                  mapper_id, num_mapper, num_reducer, task_name, file_info):
    # def assign_reducer(self, task_file, num_mapper, reducer_id, output_file, task_name):
    def reassign_job(self, down_workers):
        self.jobs_tracker_lock.acquire()
        reassign_list = []
        for down_worker in down_workers:
            for task_name in self.jobs_tracker:
                # whether deal with failure after the job is done
                # if self.jobs_tracker[task_name]["done"] == False:
                job_dict = self.jobs_tracker[task_name]
                for mapper_id in job_dict["mappers"]:
                    if job_dict["mappers"][mapper_id][1] == down_worker:
                        print "********** down %s did %s mapper %d" % (down_worker, task_name, mapper_id)
                        job_dict["mappers"][mapper_id][3] = False
                        reassign_list.append([task_name, mapper_id, 0])
                for reducer_id in job_dict["reducers"]:
                    if job_dict["reducers"][reducer_id][1] == down_worker:
                        print "********** down %s did %s reducer %d" % (down_worker, task_name, reducer_id)
                        job_dict["reducers"][reducer_id][2] = False
                        reassign_list.append([task_name, reducer_id, 1])
        self.jobs_tracker_lock.release()
        for reassign in reassign_list:
#.........这里部分代码省略.........