本文整理汇总了Python中multiprocessing.pool.MaybeEncodingError方法的典型用法代码示例。如果您正苦于以下问题：Python pool.MaybeEncodingError方法的具体用法？Python pool.MaybeEncodingError怎么用？Python pool.MaybeEncodingError使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类multiprocessing.pool的用法示例。
在下文中一共展示了pool.MaybeEncodingError方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_unpickleable_result
# 需要导入模块: from multiprocessing import pool [as 别名]
# 或者: from multiprocessing.pool import MaybeEncodingError [as 别名]
def test_unpickleable_result(self):
from multiprocessing.pool import MaybeEncodingError
p = multiprocessing.Pool(2)
# Make sure we don't lose pool processes because of encoding errors.
for iteration in range(20):
scratchpad = [None]
def errback(exc):
scratchpad[0] = exc
res = p.apply_async(unpickleable_result, error_callback=errback)
self.assertRaises(MaybeEncodingError, res.get)
wrapped = scratchpad[0]
self.assertTrue(wrapped)
self.assertIsInstance(scratchpad[0], MaybeEncodingError)
self.assertIsNotNone(wrapped.exc)
self.assertIsNotNone(wrapped.value)
p.close()
p.join() 示例2: main
# 需要导入模块: from multiprocessing import pool [as 别名]
# 或者: from multiprocessing.pool import MaybeEncodingError [as 别名]
def main():
client = Elasticsearch()
create_db(cursor)
clear_db(conn, cursor)
try:
logging.info(f'Getting photo URL...')
pool = Pool(NUM_WORKERS, init_worker)
data = pool.imap_unordered(_http_get, get_pictures(client))
except KeyboardInterrupt:
pool.terminate()
pool.join()
raise SystemExit(1)
except MaybeEncodingError:
pass
insert_sqlite_data(data)
update_es_data(cursor, client)
conn.close() 示例3: test_unpickleable_result
# 需要导入模块: from multiprocessing import pool [as 别名]
# 或者: from multiprocessing.pool import MaybeEncodingError [as 别名]
def test_unpickleable_result(self):
from multiprocessing.pool import MaybeEncodingError
p = multiprocessing.Pool(2)
# Make sure we don't lose pool processes because of encoding errors.
for iteration in range(20):
res = p.apply_async(unpickleable_result)
self.assertRaises(MaybeEncodingError, res.get)
p.close()
p.join() 示例4: __init__
# 需要导入模块: from multiprocessing import pool [as 别名]
# 或者: from multiprocessing.pool import MaybeEncodingError [as 别名]
def __init__(self, exc, value):
self.exc = repr(exc)
self.value = repr(value)
super(MaybeEncodingError, self).__init__(self.exc, self.value) 示例5: __repr__
# 需要导入模块: from multiprocessing import pool [as 别名]
# 或者: from multiprocessing.pool import MaybeEncodingError [as 别名]
def __repr__(self):
return "<MaybeEncodingError: %s>" % str(self)
# Python 2 and 3 raise a different error when they exit 示例6: worker
# 需要导入模块: from multiprocessing import pool [as 别名]
# 或者: from multiprocessing.pool import MaybeEncodingError [as 别名]
def worker(
inqueue,
outqueue,
initializer=None,
initargs=(),
maxtasks=None,
wrap_exception=False,
finalizer=None,
finalargs=(),
): # pragma: no cover
assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0)
put = outqueue.put
get = inqueue.get
if hasattr(inqueue, "_writer"):
inqueue._writer.close()
outqueue._reader.close()
if initializer is not None:
try:
initializer(*initargs)
except InitializerOrFinalizerError as e:
print(str(e))
completed = 0
while maxtasks is None or (maxtasks and completed < maxtasks):
try:
task = get()
except (EOFError, PortableOSError):
util.debug("worker got EOFError or OSError -- exiting")
break
if task is None:
util.debug("worker got sentinel -- exiting")
break
job, i, func, args, kwds = task
try:
result = (True, func(*args, **kwds))
except Exception as e:
if wrap_exception:
e = ExceptionWithTraceback(e, e.__traceback__)
result = (False, e)
try:
put((job, i, result))
except Exception as e:
wrapped = MaybeEncodingError(e, result[1])
util.debug("Possible encoding error while sending result: %s" % (wrapped))
put((job, i, (False, wrapped)))
completed += 1
if finalizer:
try:
finalizer(*finalargs)
except InitializerOrFinalizerError as e:
print(str(e))
util.debug("worker exiting after %d tasks" % completed)
# Unmodified (see above)