本文整理汇总了Python中queue.LifoQueue.join方法的典型用法代码示例。如果您正苦于以下问题：Python LifoQueue.join方法的具体用法？Python LifoQueue.join怎么用？Python LifoQueue.join使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类queue.LifoQueue的用法示例。
在下文中一共展示了LifoQueue.join方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: ThreadedNormalWorker
# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import join [as 别名]
class ThreadedNormalWorker(object):
def __init__(self, print_errors=False):
self.print_errors = print_errors
self.queue = LifoQueue()
def get_url_bulk(self):
normals = Normals.objects(access_success=False)
for i in normals:
self.queue.put(item=i)
def grab_from_queue(self):
while not self.queue.empty():
url = self.queue.get()
normals_finder = NormalsSpider(url=url.url,
print_errors=self.print_errors)
normals_finder.update_normals_data()
print(url.url)
self.queue.task_done()
def start(self, n_threads):
self.get_url_bulk()
for i in range(n_threads):
thread = Thread(target=self.grab_from_queue())
thread.start()
self.queue.join()示例2: get_friends
# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import join [as 别名]
def get_friends(inlist, output_path, subpath):
##########
# task assignment
##########
global lock
lock = Lock()
global q
q = LifoQueue() # screen_name queue
global friend_list
friend_list = dict()
with open("./twitter_api_keys.pydict", "r") as f:
keys = eval(f.read())
# initiate task for crawler()
for input in inlist:
friend_list[input] = set()
q.put({input_type: input, "count": 5000, "cursor": -1})
for key in keys:
t = Thread(target=uid_crawler, kwargs={"key": key, "name": keys.index(key), "rest_url": "friends/ids"})
# t.daemon = True
# time.sleep(2)
t.start()
q.join()
# # RUN THIS AFTER FINISHED.
try:
mkdir("{}/{}".format(output_path, subpath))
except OSError:
pass
print("writing to disk.", end=".", flush=True)
if subpath == 0:
for key, vals in list(friend_list.items()):
print(".", end="", flush=True)
with BZ2File("{}/{}/{}.bz2".format(output_path, subpath, key), "w") as f:
f.writelines(map(lambda item: (str(item) + "\n").encode("utf-8"), vals))
else:
for key, vals in list(friend_list.items()):
print(".", end="", flush=True)
with ZipFile("{}/{}/{}.zip".format(output_path, subpath, str(key)[:3]), "a") as zf:
zf.writestr(str(key), "\n".join([str(item) for item in vals]).encode("utf-8"), ZIP_LZMA)
print("Done. Waiting remaining threads to quit", end=".", flush=True)
while activeCount() > 1:
print(activeCount(), end=".", flush=True)
time.sleep(2)
return friend_list示例3: print
# 需要导入模块: from queue import LifoQueue [as 别名]
# 或者: from queue.LifoQueue import join [as 别名]
print("{}/{} users will be updated.".format(n_existed, i - n_existed))
####################
# dispatch threads
####################
dq_thread = Thread(target = dq_helper, kwargs = {'root': root})
dq_thread.daemon = True
dq_thread.start()
uq_thread = Thread(target = uq_helper, kwargs = {'root': root})
uq_thread.daemon = True
uq_thread.start()
for key in keys:
if tq.qsize() == 0: break
t = Thread(target = crawler,
kwargs ={'key': key,
'name': '{min}:{max}-{no:03}'.format(min = min(RG), max = max(RG), no = keys.index(key)),
'rest_url':'users/lookup'})
t.daemon = True
time.sleep(2)
t.start()
tq.join()
dq.put(None)
uq.put(None)
print("Complete!")