Python gevent.Timeout类代码示例

 def initial_test(self, address):
     try:
         timeout = Timeout(self.TIMEOUT, TestTimeout('The server timed out on the first command.'))
         timeout.start()
         TestClient(address).put('key', 'value')
     finally:
         timeout.cancel()

def make_conn(url,timelimit):
    global SUCCESS_RECORD,FAIL_RECORD,TOTAL_SIZE,REQTIME_ARR
    time_start = time.time()
    
    if timelimit:
        timeout = Timeout(timelimit)
        timeout.start()
    
    try:
        f = urlopen(url)
        if f.getcode() == 200:
            time_end = time.time()
            server_info = f.info()
            content_type = server_info['content-type'].split(";")[0]
            if content_type == "text/html":
                data = f.read()
		size = int(server_info['content-length'])
            else:
                size = int(server_info['content-length'])
            REQTIME_ARR.append((time_end - time_start) * 1000)
            TOTAL_SIZE = TOTAL_SIZE + size
            SUCCESS_RECORD += 1
        else:
            FAIL_RECORD += 1
    except Timeout:
        FAIL_RECORD += 1
        return
    except Exception,e:
        FAIL_RECORD += 1
        return

 def _recv_command(self):
     timeout = Timeout(self.command_timeout)
     timeout.start()
     try:
         return self.io.recv_command()
     finally:
         timeout.cancel()

    def __init__(self, url):
        self.url = url
        self.protocol, self.domain = self.url.split("://")  #e.g. news.bbc.co.uk
        self.domain = self.domain.split('/')[0]
        self.site_data = sites[self.domain]

        self.total_words = {}

        timeout = Timeout(30, TimeoutError)
        timeout.start()
        try:
            self.html = self.read_url()
        except TimeoutError:
            print url + " timed out"
            return 
        finally:
            timeout.cancel()

        self.text = self.boiler_extract()
        self.soup = BeautifulSoup(self.html, 'lxml')
        self.article = self.is_article()

        if self.article:
            self.calc_total_words()
            articles.put(self)

        self.find_links()

    def _get_message_data(self):
        max_size = self.extensions.getparam('SIZE', filter=int)
        reader = DataReader(self.io, max_size)

        err = None
        timeout = Timeout(self.data_timeout)
        timeout.start()
        try:
            data = reader.recv()
        except ConnectionLost:
            raise
        except SmtpError as e:
            data = None
            err = e
        finally:
            timeout.cancel()

        reply = Reply('250', '2.6.0 Message Accepted for Delivery')
        self._call_custom_handler('HAVE_DATA', reply, data, err)

        self.io.send_reply(reply)
        self.io.flush_send()

        self.have_mailfrom = None
        self.have_rcptto = None

def sample():
    timeout = Timeout(5)
    timeout.start()
    try:
        gevent.spawn(wait).join()
    except Timeout:
        print 'Could not complete'

    def execute(self, name, args, func_type='PY'):        

        module_path = settings['MODULE_PATH'] #"c:/mtp/mabotech/maboss1.1"
        
        info = "[%s]%s:%s" % (module_path, func_type, name)
        log.debug( info)
        
        t = time.time()
        
        if name == "time":
            #for reconnection testing
            return t
        
        #Sync Code Here !!!
        
        timeout = Timeout(5, Exception)
        timeout.start()
        try:
            #...  # exception will be raised here, after *seconds* passed since start() call
            rtn = "OK"
            rtn = py_executor.execute(name, args, module_path) 
            #gevent.sleep(0.02)
            pf_log.debug("%10.5f,%s,%s" %(time.time()-t, func_type, name ) )
            
            return rtn                    
        except Exception, e:
            log.error(e.message)

def func1():
    
        
    utc = arrow.utcnow()
    local = utc.to('Asia/Shanghai')
    ts = local.timestamp
    print arrow.get(ts)
    #print local.format('YYYY-MM-DD HH:mm:ss ZZ')
    
    """function and heartbeat"""
    
    ex = TimeoutException("timeout ex")
    
    #gevent timeout
    timeout = Timeout(6, ex)
    #start
    timeout.start()
    try:
        
        # exception will be raised here, after *seconds* 
        # passed since start() call
        
        gevent.sleep(3 * random.randint(1,4))
        #print "f1 heart beat"
        heartbeat("f1")

    except TimeoutException as ex:
        print ex
    finally:
        #cancel timeout
        timeout.cancel()

def read_url(url):
    timeout = Timeout(10)
    timeout.start()
    try:
        response = urllib2.urlopen(url)
        reason, other = response.getcode(), response.msg
    except Timeout, t:
        reason, other = 'gevent timeout', 0

def run_cmd(args, timeout=None):
    _init()
    args = list(args)
    for i, x in enumerate(args):
        if isinstance(x, unicode):
            args[i] = x.encode("utf-8")

    sp = socket.socketpair()
    pid = os.fork()
    if pid == 0:
        # client
        try:
            os.dup2(sp[1].fileno(), 1)
            os.dup2(sp[1].fileno(), 2)
            sp[0].close()
            sp[1].close()
            os.execvp(args[0], args)
        except:
            stderr = os.fdopen(2, "w", 0)
            os.write(2, "failed to exec child process: %r\nPATH=%r" % (args, os.environ.get('PATH')))
            traceback.print_exc(file=stderr)
        finally:
            os._exit(97)

    pid2status[pid] = event.AsyncResult()
    if not _nochild:

        def cb():
            pid2status[pid].set(child_watcher.rstatus)
            child_watcher.stop()

        child_watcher = get_hub().loop.child(pid)
        child_watcher.start(cb)

    sp[1].close()

    chunks = []

    # prevent loopexit. see test_run_cmd_trigger_loopexit in test_proc.py
    if timeout is None:
        timeout = 2 ** 30

    timeout = Timeout(timeout)
    timeout.start()
    try:
        while 1:
            chunk = sp[0].recv(4096)
            if not chunk:
                break
            chunks.append(chunk)

        st = pid2status[pid].get()
        del pid2status[pid]

        return st, "".join(chunks)
    except Timeout, t:
        if t is not timeout:
            raise

def reposts_crawler():
    '''
    greenlet reposts crawler
    '''
    while not reposts_fetch_queue.empty():
        IS_NEED_REFETCH = False #when timeout or errors occur,put the url back into the task queue and the make sure the task is not set to done!
        try:
            wait_time = Timeout(MAX_WAIT_TIME)
            wait_time.start()
            url = reposts_fetch_queue.get()
            gevent.sleep(0.0)
            reposts_time = _http_call(url)
            for status in reposts_time['reposts']:
                if not status.get('deleted'):
                    weibo_created_at = datetime.strptime(status.get('created_at'), '%a %b %d %H:%M:%S +0800 %Y')
                    user_created_at = datetime.strptime(status.get('user').get('created_at'),
                                                        '%a %b %d %H:%M:%S +0800 %Y')
                    reposts_status_id = -1

                    if status.get('retweeted_status') is not None:
                        reposts_status = status['retweeted_status']
                        reposts_status_id = reposts_status['id']

                    weibo_params = (
                        status['id'], status['user']['id'], status['text'], status['source'], weibo_created_at,
                        reposts_status_id)
                    user_params = (
                        status['user']['id'], status['user']['screen_name'], status['user']['name'],
                        status['user']['province'],
                        status['user']['city'], status['user']['location'], status['user']['description'],
                        status['user']['profile_image_url'], status['user']['domain'], status['user']['gender'],
                        status['user']['followers_count'], status['user']['friends_count'],
                        status['user']['statuses_count']
                        ,
                        status['user']['favourites_count'], user_created_at, status['user']['verified'],
                        status['user']['verified_type'], status['user']['verified_reason'],
                        status['user']['bi_followers_count'] )
                    cursor.execute(REPOSTS_WEIBO_INSERT_SQL, weibo_params)
                    cursor.execute(REPOSTS_USER_INSERT_SQL, user_params)
        except Timeout as t:
            if t is wait_time:
            #                print '处理超时，等待重新抓取!'
                #put timeout url back into the task queue
                IS_NEED_REFETCH = True
        except Exception as e:
            IS_NEED_REFETCH = True
            logger.error(traceback.format_exc())
        finally:
            wait_time.cancel()
            if IS_NEED_REFETCH is not True:
                reposts_fetch_queue.task_done()
            #                print url + ' 抓取完成 --- 转发'
            else:
                reposts_fetch_queue.put(url)
                print status
                print url + ' 抓取失败 --- 转发'

 def timeout_wrapper(*args, **kwargs):
     t = Timeout(seconds,
         TestTimeout('Timed out after %d seconds' % seconds)
     )
     t.start()
     try:
         ret = func(*args, **kwargs)
     finally:
         t.cancel()
     return ret

 def serve_for_test(self):
     timeout = Timeout(10)
     timeout.start()
     try:
         while self.is_connected():
             if len(self.re_schedule_events) == 10 and len(self.heartbeat_events) == 10:
                 break
             gevent.sleep(0.01)
     finally:
         timeout.cancel()

 def requestGet(self, url):
     wait = random.random() * (wait_time[1] - wait_time[0])
     sleep(wait)
     timeout = Timeout(request_timeout)
     timeout.start()
     try:
         req = requests.get(url=url, verify=True, headers=headers, proxies=proxies)
     except IncompleteRead:
         pass
         # todo:未知错误，暂还未查清
     timeout.cancel()
     return req

def foo1():
    timeout = Timeout(seconds)
    timeout.start()
    
    def wait():
        gevent.sleep(10)
        
    try:
        gevent.spawn(wait).join()
    except Timeout:
        print('Could not complete')
    else:
        print('Complete!')