本文整理汇总了Python中eventlet.Queue.put方法的典型用法代码示例。如果您正苦于以下问题:Python Queue.put方法的具体用法?Python Queue.put怎么用?Python Queue.put使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类eventlet.Queue
的用法示例。
在下文中一共展示了Queue.put方法的10个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Dispatcher
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
class Dispatcher(EventThread):
def __init__(self):
self.proposer_queue = Queue()
self.acceptor_queue = Queue()
def _run(self):
while True:
#get recved string
u = do_recv()
m = Message()
if not m.depack(u):
log.debug('depack %s error', u)
continue
#check timestamp
if m.timestamp - get_utc_time() > conf.proposer_timeout:
log.debug('receive expired package, drop it')
continue
#check version
if m.version != VERSION:
log.debug('version not match')
continue
#check signature
if m.signature != m.build_hmac_signature():
log.info('message signature failed, may changed in network')
continue
if m.method in (PREPARE, ACCEPT, LEARN, RENEW):
self.acceptor_queue.put(m)
elif m.method in (NACK, ACK, OUTDATE):
self.proposer_queue.put(m)
示例2: WSTestGenerator
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
class WSTestGenerator(WebSocketView):
def handle_websocket(self, ws):
self._ws = ws
return super(RangeWebsocket, self)
def handler(self, ws):
self.queue = Queue()
while True:
m = ws.wait()
# import ipdb; ipdb.set_trace()
if m is None:
break
self.queue.put(m)
示例3: test_connection_pooling
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
def test_connection_pooling(self):
with patch('swift.common.memcached.socket') as mock_module:
# patch socket, stub socket.socket, mock sock
mock_sock = mock_module.socket.return_value
# track clients waiting for connections
connected = []
connections = Queue()
def wait_connect(addr):
connected.append(addr)
connections.get()
mock_sock.connect = wait_connect
memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'],
connect_timeout=10)
# sanity
self.assertEquals(1, len(memcache_client._client_cache))
for server, pool in memcache_client._client_cache.items():
self.assertEquals(2, pool.max_size)
# make 10 requests "at the same time"
p = GreenPool()
for i in range(10):
p.spawn(memcache_client.set, 'key', 'value')
for i in range(3):
sleep(0.1)
self.assertEquals(2, len(connected))
# give out a connection
connections.put(None)
for i in range(3):
sleep(0.1)
self.assertEquals(2, len(connected))
# finish up
for i in range(8):
connections.put(None)
self.assertEquals(2, len(connected))
p.waitall()
self.assertEquals(2, len(connected))
示例4: test_connection_pooling
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
def test_connection_pooling(self):
with patch('swift.common.memcached.socket') as mock_module:
# patch socket, stub socket.socket, mock sock
mock_sock = mock_module.socket.return_value
# track clients waiting for connections
connected = []
connections = Queue()
errors = []
def wait_connect(addr):
connected.append(addr)
sleep(0.1) # yield
val = connections.get()
if val is not None:
errors.append(val)
mock_sock.connect = wait_connect
memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'],
connect_timeout=10)
# sanity
self.assertEquals(1, len(memcache_client._client_cache))
for server, pool in memcache_client._client_cache.items():
self.assertEqual(2, pool.max_size)
# make 10 requests "at the same time"
p = GreenPool()
for i in range(10):
p.spawn(memcache_client.set, 'key', 'value')
for i in range(3):
sleep(0.1)
self.assertEqual(2, len(connected))
# give out a connection
connections.put(None)
# at this point, only one connection should have actually been
# created, the other is in the creation step, and the rest of the
# clients are not attempting to connect. we let this play out a
# bit to verify.
for i in range(3):
sleep(0.1)
self.assertEqual(2, len(connected))
# finish up, this allows the final connection to be created, so
# that all the other clients can use the two existing connections
# and no others will be created.
connections.put(None)
connections.put('nono')
self.assertEqual(2, len(connected))
p.waitall()
self.assertEqual(2, len(connected))
self.assertEqual(0, len(errors),
"A client was allowed a third connection")
connections.get_nowait()
self.assertTrue(connections.empty())
示例5: GreenletsThread
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
class GreenletsThread(Thread):
"""
Main thread for the program. If running stand alone this will be running
as a greenlet instead.
"""
def __init__ (self, server_url, login_params):
self.running = True
self.agent = True
self.cmd_out_queue = []
self.cmd_in_queue = []
self.out_queue = Queue()
self.in_queue = Queue()
self.server_url = server_url
self.login_params = login_params
Thread.__init__(self)
def apply_position(self, obj_uuid, pos, rot=None):
cmd = ['pos', obj_uuid, pos, rot]
self.addCmd(cmd)
def __getattr__(self, name):
return ProxyFunction(name, self)
def apply_scale(self, obj_uuid, scale):
cmd = ['scale', obj_uuid, scale]
self.addCmd(cmd)
def run(self):
agent = AgentManager(self.in_queue,
self.out_queue)
error = agent.login(self.server_url, self.login_params)
if error:
self.out_queue.put(["error", str(error)])
self.out_queue.put(["agentquit", str(error)])
while self.out_queue.qsize():
api.sleep(0.1)
agent.logger.debug("Quitting")
self.agent = agent
self.running = False
def addCmd(self, cmd):
self.in_queue.put(cmd)
def getQueue(self):
out_queue = []
while self.out_queue.qsize():
out_queue.append(self.out_queue.get())
return out_queue
示例6: run_stock_parser
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
def run_stock_parser():
symbol_q = Queue()
price_q = Queue()
stock_symbols = []
with open('symbols.txt', 'r') as symfile:
for n, line in enumerate(symfile):
sym = line.strip()
if sym:
stock_symbols.append(sym)
ncpu = cpu_count()
pool = [spawn(read_stock_worker, symbol_q, price_q) for _ in range(ncpu * 2)]
output = spawn(write_output_file, price_q)
for symbol in stock_symbols:
symbol_q.put(symbol)
symbol_q.put(_sentinel)
for p in pool:
p.wait()
price_q.put(_sentinel)
output.wait()
示例7: ECWriter
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
class ECWriter(object):
"""
Writes an EC chunk
"""
def __init__(self, chunk, conn):
self._chunk = chunk
self._conn = conn
self.failed = False
self.bytes_transferred = 0
self.checksum = hashlib.md5()
@property
def chunk(self):
return self._chunk
@property
def conn(self):
return self._conn
@classmethod
def connect(cls, chunk, sysmeta, reqid=None):
raw_url = chunk["url"]
parsed = urlparse(raw_url)
chunk_path = parsed.path.split('/')[-1]
h = {}
h["transfer-encoding"] = "chunked"
h[chunk_headers["content_id"]] = sysmeta['id']
h[chunk_headers["content_path"]] = sysmeta['content_path']
h[chunk_headers["content_chunkmethod"]] = sysmeta['chunk_method']
h[chunk_headers["container_id"]] = sysmeta['container_id']
h[chunk_headers["chunk_pos"]] = chunk["pos"]
h[chunk_headers["chunk_id"]] = chunk_path
h[chunk_headers["content_policy"]] = sysmeta['policy']
h[chunk_headers["content_version"]] = sysmeta['version']
if reqid:
h['X-oio-req-id'] = reqid
# in the trailer
# metachunk_size & metachunk_hash
h["Trailer"] = (chunk_headers["metachunk_size"],
chunk_headers["metachunk_hash"])
with ConnectionTimeout(io.CONNECTION_TIMEOUT):
conn = io.http_connect(
parsed.netloc, 'PUT', parsed.path, h)
conn.chunk = chunk
return cls(chunk, conn)
def start(self, pool):
# we use eventlet Queue to pass data to the send coroutine
self.queue = Queue(io.PUT_QUEUE_DEPTH)
# spawn the send coroutine
pool.spawn(self._send)
def _send(self):
# this is the send coroutine loop
while True:
# fetch input data from the queue
d = self.queue.get()
# use HTTP transfer encoding chunked
# to write data to RAWX
if not self.failed:
# format the chunk
to_send = "%x\r\n%s\r\n" % (len(d), d)
try:
with ChunkWriteTimeout(io.CHUNK_TIMEOUT):
self.conn.send(to_send)
self.bytes_transferred += len(d)
except (Exception, ChunkWriteTimeout) as e:
self.failed = True
msg = str(e)
logger.warn("Failed to write to %s (%s)", self.chunk, msg)
self.chunk['error'] = msg
self.queue.task_done()
def wait(self):
# wait until all data in the queue
# has been processed by the send coroutine
if self.queue.unfinished_tasks:
self.queue.join()
def send(self, data):
# do not send empty data because
# this will end the chunked body
if not data:
return
# put the data to send into the queue
# it will be processed by the send coroutine
self.queue.put(data)
def finish(self, metachunk_size, metachunk_hash):
parts = [
'0\r\n',
'%s: %s\r\n' % (chunk_headers['metachunk_size'],
metachunk_size),
'%s: %s\r\n' % (chunk_headers['metachunk_hash'],
metachunk_hash),
'\r\n'
]
to_send = "".join(parts)
#.........这里部分代码省略.........
示例8: Manager
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
class Manager(object):
"""Class encapsulating Heroshi URL server state."""
def __init__(self):
self.active = False
self.prefetch_queue = Queue(settings.prefetch['queue_size'])
self.prefetch_thread = spawn(self.prefetch_worker)
self.prefetch_thread.link(reraise_errors, greenthread.getcurrent())
self.given_items = Cache()
self.postreport_queue = Queue(settings.postreport['queue_size'])
self.postreport_thread = spawn(self.postreport_worker)
self.postreport_thread.link(reraise_errors, greenthread.getcurrent())
self.storage_connections = eventlet.pools.Pool(max_size=settings.storage['max_connections'])
self.storage_connections.create = StorageConnection
def close(self):
self.active = False
self.prefetch_thread.kill()
self.postreport_thread.kill()
def ping_storage(self):
with self.storage_connections.item() as storage:
pass
def get_from_prefetch_queue(self, size):
result = []
while len(result) < size:
sleep()
try:
pack = self.prefetch_queue.get(timeout=settings.prefetch['get_timeout'])
except eventlet.queue.Empty:
break
result.extend(pack)
return result
def prefetch_worker(self):
if not self.active:
sleep(0.01)
while self.active:
with self.storage_connections.item() as storage:
docs = storage.query_new_random(settings.prefetch['single_limit'])
if len(docs) == 0:
sleep(10.)
continue
else:
# Note: putting a *list* as a single item into queue
self.prefetch_queue.put(docs)
# and respawn again
self.prefetch_thread = spawn(self.prefetch_worker)
@log_exceptions
def _postreport_worker(self):
docs = []
while len(docs) < settings.postreport['flush_size']: # inner accumulator loop
try:
item = self.postreport_queue.get(timeout=settings.postreport['flush_delay'])
except eventlet.queue.Empty:
break
# Quick dirty duplicate filtering.
# Note that this code only finds dups in current "flush pack". `report_result` uses
# `is_duplicate_report` which finds dups in whole `postreport_queue` but it can't find dups here.
# Thus two dups searchers.
# It is still possible that at most 2 duplicate reports exist: one in `postreport_queue`
# and one in current "flush pack". This is acceptable, because most of the dups are filtered out.
for doc in docs:
if item['url'] == doc['url']:
item = None
break
if item is None:
continue
if 'result' not in item:
# It's a link, found on some reported page.
# Just add it to bulk insert, don't try to update any document here.
docs.append(item)
continue
docs.append(item)
if not docs:
return
with self.storage_connections.item() as storage:
for doc in docs:
content = doc.pop('content', None)
storage.save(doc)
if content is None:
continue
headers = doc.get('headers') or {}
content_type = headers.get('content-type', "application/octet-stream")
storage.save_content(doc, content, content_type)
#.........这里部分代码省略.........
示例9: Crawler
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
class Crawler(object):
"""
A crawler will traverse all the pages of a site and process the content
in a defined way.
:param init_urls: the very first urls to start with.
:param q: the queue that stores all urls to be crawled
:param urls: a set stores all urls already crawled
"""
def __init__(self, init_urls, max_workers=200):
self.init_urls = init_urls
self.max_workers = max_workers
self.q = Queue()
self.urls = set()
self.s = requests.Session()
self.root_hosts = set()
for url in init_urls:
self.q.put(url)
self.urls.add(url)
self.root_hosts.add(get_netloc(url))
def url_allowed(self, url):
"""Check if given url will be crawled.
Current, only if the url belongs to the same host as init_urls.
"""
return get_netloc(url) in self.root_hosts
def save(self, response):
"""Save data at the given url."""
raise NotImplementedError(
"Please implement your own save logic in subclass.")
def parse(self, response):
self.save(response)
new_links = set()
for url in self.find_links(response):
if url not in self.urls and self.url_allowed(url):
new_links.add(url)
self.urls.add(url)
self.q.put(url)
if len(new_links) != 0:
print("Find %d new urls to crawl" % len(new_links))
def fetch(self, url):
"""Fetch content of the url from network."""
response = self.s.get(url)
print("Getting content from %s, length: %d" % (url,
len(response.content)))
return response
def work(self, i):
"""Define the work process.
Retrieve a url from queue, fetch the content from it,
process it and get new urls to crawl.
Continue the process until all pages are crawled.
:param i: indicate the worker number
"""
while True:
url = self.q.get()
print("Worker %d: Getting url %s from queue." % (i, url))
response = self.fetch(url)
self.parse(response)
self.q.task_done()
def run(self):
"""Start the crawling process.
This is the main entrance for our crawler. It will start several
workers, crawling in parallel.
"""
pool = eventlet.GreenPool()
start = time.time()
for i in range(self.max_workers):
pool.spawn(self.work, i)
self.q.join()
end = time.time()
print("Finished crawling, takes %s seconds." % str(end - start))
print("Have fun hacking!")
示例10: Interpreter
# 需要导入模块: from eventlet import Queue [as 别名]
# 或者: from eventlet.Queue import put [as 别名]
#.........这里部分代码省略.........
for h in s.history:
if h.type == "deep":
f = lambda s0: isAtomicState(s0) and isDescendant(s0,s)
else:
f = lambda s0: s0.parent == s
self.historyValue[h.id] = filter(f,self.configuration) #+ s.parent
for s in statesToExit:
for content in s.onexit:
self.executeContent(content)
for inv in s.invoke:
self.cancelInvoke(inv)
self.configuration.delete(s)
def cancelInvoke(self, inv):
inv.cancel()
def executeTransitionContent(self, enabledTransitions):
for t in enabledTransitions:
self.executeContent(t)
def enterStates(self, enabledTransitions):
statesToEnter = OrderedSet()
statesForDefaultEntry = OrderedSet()
for t in enabledTransitions:
if t.target:
tstates = self.getTargetStates(t.target)
if t.type == "internal" and isCompoundState(t.source) and all(map(lambda s: isDescendant(s,t.source), tstates)):
ancestor = t.source
else:
ancestor = self.findLCA([t.source] + tstates)
for s in tstates:
self.addStatesToEnter(s,statesToEnter,statesForDefaultEntry)
for s in tstates:
for anc in getProperAncestors(s,ancestor):
statesToEnter.add(anc)
if isParallelState(anc):
for child in getChildStates(anc):
if not any(map(lambda s: isDescendant(s,child), statesToEnter)):
self.addStatesToEnter(child, statesToEnter,statesForDefaultEntry)
statesToEnter.sort(key=enterOrder)
for s in statesToEnter:
self.statesToInvoke.add(s)
self.configuration.add(s)
if self.doc.binding == "late" and s.isFirstEntry:
s.initDatamodel()
s.isFirstEntry = False
for content in s.onentry:
self.executeContent(content)
if s in statesForDefaultEntry:
self.executeContent(s.initial)
if isFinalState(s):
parent = s.parent
grandparent = parent.parent
self.internalQueue.put(Event(["done", "state", parent.id], s.donedata()))
if isParallelState(grandparent):
if all(map(self.isInFinalState, getChildStates(grandparent))):
self.internalQueue.put(Event(["done", "state", grandparent.id]))
for s in self.configuration:
if isFinalState(s) and isScxmlState(s.parent):
self.running = False;
def addStatesToEnter(self, state,statesToEnter,statesForDefaultEntry):
if isHistoryState(state):
if state.id in self.historyValue:
for s in self.historyValue[state.id]:
self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry)
for anc in getProperAncestors(s,state):
statesToEnter.add(anc)
else:
for t in state.transition:
for s in self.getTargetStates(t.target):
self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry)
else:
statesToEnter.add(state)
if isCompoundState(state):
statesForDefaultEntry.add(state)
for s in self.getTargetStates(state.initial):
self.addStatesToEnter(s, statesToEnter, statesForDefaultEntry)
elif isParallelState(state):
for s in getChildStates(state):
self.addStatesToEnter(s,statesToEnter,statesForDefaultEntry)
def isInFinalState(self, s):
if isCompoundState(s):
return any(map(lambda s: isFinalState(s) and s in self.configuration, getChildStates(s)))
elif isParallelState(s):
return all(map(self.isInFinalState, getChildStates(s)))
else:
return False
def findLCA(self, stateList):
for anc in filter(isCompoundState, getProperAncestors(stateList[0], None)):
# for anc in getProperAncestors(stateList[0], None):
if all(map(lambda(s): isDescendant(s,anc), stateList[1:])):
return anc