如何解决Python：从multiprocessing.Process中获取回溯
| 我试图从multiprocessing.Process获取一个追溯对象。 不幸的是，通过管道传递异常信息不起作用，因为不能对回溯对象进行腌制：def foo(pipe_to_parent):
try:
raise Exception(\'xxx\')
except:
pipe_to_parent.send(sys.exc_info())
to_child,to_self = multiprocessing.Pipe()
process = multiprocessing.Process(target = foo,args = (to_self,))
process.start()
exc_info = to_child.recv()
process.join()
print traceback.format_exception(*exc_info)
to_child.close()
to_self.close()
Traceback (most recent call last):
File \"/usr/lib/python2.6/multiprocessing/process.py\",line 231,in _bootstrap
self.run()
File \"/usr/lib/python2.6/multiprocessing/process.py\",line 88,in run
self._target(*self._args,**self._kwargs)
File \"foo\",line 7,in foo
to_parent.send(sys.exc_info())
PicklingError: Can\'t pickle <type \'traceback\'>: attribute lookup __builtin__.traceback Failed
解决方法
使用
tblib
import tblib.pickling_support
tblib.pickling_support.install()
from multiprocessing import Pool
import sys
class ExceptionWrapper(object):
def __init__(self,ee):
self.ee = ee
__,__,self.tb = sys.exc_info()
def re_raise(self):
raise self.ee.with_traceback(self.tb)
# for Python 2 replace the previous line by:
# raise self.ee,None,self.tb
# example how to use ExceptionWrapper
def inverse(i):
\"\"\"will fail for i == 0\"\"\"
try:
return 1.0 / i
except Exception as e:
return ExceptionWrapper(e)
def main():
p = Pool(1)
results = p.map(inverse,[0,1,2,3])
for result in results:
if isinstance(result,ExceptionWrapper):
result.re_raise()
if __name__ == \"__main__\":
main()
multiprocessing.map
def run_functor(functor):
\"\"\"
Given a no-argument functor,run it and return its result. We can
use this with multiprocessing.map and map it over a list of job
functors to do them.
Handles getting more than multiprocessing\'s pitiful exception output
\"\"\"
try:
# This is where you do your actual work
return functor()
except:
# Put all exception text into an exception and raise that
raise Exception(\"\".join(traceback.format_exception(*sys.exc_info())))
sys.exc_info()
import multiprocessing
import sys
import traceback
def foo(pipe_to_parent):
try:
raise Exception(\'xxx\')
except:
except_type,except_class,tb = sys.exc_info()
pipe_to_parent.send((except_type,traceback.extract_tb(tb)))
to_child,to_self = multiprocessing.Pipe()
process = multiprocessing.Process(target = foo,args = (to_self,))
process.start()
exc_info = to_child.recv()
process.join()
print exc_info
to_child.close()
to_self.close()
(<type \'exceptions.Exception\'>,Exception(\'xxx\',),[(\'test_tb.py\',7,\'foo\',\"raise Exception(\'xxx\')\")])
multiprocessing.pool.Async
get
def full_traceback(func):
import traceback,functools
@functools.wraps(func)
def wrapper(*args,**kwargs):
try:
return func(*args,**kwargs)
except Exception as e:
msg = \"{}\\n\\nOriginal {}\".format(e,traceback.format_exc())
raise type(e)(msg)
return wrapper
def func0():
raise NameError(\"func0 exception\")
def func1():
return func0()
# Key is here!
@full_traceback
def main(i):
return func1()
if __name__ == \'__main__\':
from multiprocessing import Pool
pool = Pool(4)
try:
results = pool.map_async(main,range(5)).get(1e5)
finally:
pool.close()
pool.join()
Traceback (most recent call last):
File \"bt.py\",line 34,in <module>
results = pool.map_async(main,range(5)).get(1e5)
File \"/opt/anaconda/lib/python2.7/multiprocessing/pool.py\",line 567,in get
raise self._value
NameError: Exception in func0
Original Traceback (most recent call last):
File \"bt.py\",line 13,in wrapper
return func(*args,**kwargs)
File \"bt.py\",line 27,in main
return func1()
File \"bt.py\",line 23,in func1
return func0()
File \"bt.py\",line 20,in func0
raise NameError(\"Exception in func0\")
NameError: Exception in func0
Traceback (most recent call last):
File \"bt.py\",in get
raise self._value
NameError: Exception in func0
worker
try
except
import tblib.pickling_support
tblib.pickling_support.install()
import sys
class DelayedException(Exception):
def __init__(self,self.tb = sys.exc_info()
super(DelayedException,self).__init__(str(ee))
def re_raise(self):
raise self.ee,self.tb
def worker():
try:
raise ValueError(\'Something went wrong.\')
except Exception as e:
raise DelayedException(e)
if __name__ == \'__main__\':
import multiprocessing
pool = multiprocessing.Pool()
try:
pool.imap(worker,[1,3])
except DelayedException as e:
e.re_raise()
import sys
import traceback
import functools
def catch_remote_exceptions(wrapped_function):
\"\"\" https://stackoverflow.com/questions/6126007/python-getting-a-traceback \"\"\"
@functools.wraps(wrapped_function)
def new_function(*args,**kwargs):
try:
return wrapped_function(*args,**kwargs)
except:
raise Exception( \"\".join(traceback.format_exception(*sys.exc_info())) )
return new_function
class ProcessLocker(object):
@catch_remote_exceptions
def __init__(self):
super().__init__()
@catch_remote_exceptions
def create_process_locks(self,total_processes):
self.process_locks = []
# ...
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。