I'm trying to pickle quite an involved object hierarchy and getting the exception:
pickle.PicklingError: Can't pickle <class 'function'>: attribute lookup builtins.function failed
Are there any reasonable methods one can use to test the pickleablility of an object hierarchy? My aim would be to find the location of the offending function
To do this, I'd use dill, which can serialize almost anything in python. Dill also has some good tools for helping you understand what is causing your pickling to fail when your code fails.
>>> import dill
>>> dill.loads(dill.dumps(your_bad_object))
>>> ...
>>> # if you get a pickling error, use dill's tools to figure out a workaround
>>> dill.detect.badobjects(your_bad_object, depth=0)
>>> dill.detect.badobjects(your_bad_object, depth=1)
>>> ...
If you absolutely wanted to, you could use dill's badobjects (or one of the other detection functions) to dive recursively into your object's reference chain, and pop out the unpickleable objects, instead of calling it at at every depth, as above.
Also, objgraph is a pretty handy compliment to the test suite too.
>>> # visualize the references in your bad objects
>>> objgraph.show_refs(your_bad_object, filename='your_bad_object.png')
I did this, it does the trick for me a lot of the time... I'll update this once I have found something totally foolproof
It makes a bunch of prints then raises an exception if there is one to be raised so you can see what part of the object hierarchy is causing the problem.
def test_pickle(xThing,lTested = []):
import pickle
if id(xThing) in lTested:
return lTested
sType = type(xThing).__name__
print('Testing {0}...'.format(sType))
if sType in ['type','int','str']:
print('...too easy')
return lTested
if sType == 'dict':
print('...testing members')
for k in xThing:
lTested = Pickalable.test_pickle(xThing[k],lTested)
print('...tested members')
return lTested
if sType == 'list':
print('...testing members')
for x in xThing:
lTested = Pickalable.test_pickle(x)
print('...tested members')
return lTested
lTested.append(id(xThing))
oClass = type(xThing)
for s in dir(xThing):
if s.startswith('_'):
print('...skipping *private* thingy')
continue
#if it is an attribute: Skip it
try:
xClassAttribute = oClass.__getattribute__(oClass,s)
except AttributeError:
pass
else:
if type(xClassAttribute).__name__ == 'property':
print('...skipping property')
continue
xAttribute = xThing.__getattribute__(s)
print('Testing {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
#if it is a function make sure it is stuck to the class...
if type(xAttribute).__name__ == 'function':
raise Exception('ERROR: found a function')
if type(xAttribute).__name__ == 'method':
print('...skipping method')
continue
if type(xAttribute).__name__ == 'HtmlElement':
continue
if type(xAttribute) == dict:
print('...testing dict values for {0}.{1}'.format(sType,s))
for k in xAttribute:
lTested = Pickalable.test_pickle(xAttribute[k])
continue
print('...finished testing dict values for {0}.{1}'.format(sType,s))
try:
oIter = xAttribute.__iter__()
except AttributeError:
pass
except AssertionError:
pass #lxml elements do this
else:
print('...testing iter values for {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
for x in xAttribute:
lTested = Pickalable.test_pickle(x,lTested)
print('...finished testing iter values for {0}.{1}'.format(sType,s))
try:
xAttribute.__dict__
except AttributeError:
pass
else:
#this attribute should be explored seperately...
lTested = Pickalable.test_pickle(xAttribute,lTested)
continue
pickle.dumps(xAttribute)
print('Testing {0} as complete object'.format(sType))
pickle.dumps(xThing)
return lTested
Here is a slightly tweaked version of @Sheena's code that also works with python 2 and some additional types:
def test_pickle(xThing, lTested = []):
import pickle
if id(xThing) in lTested:
return lTested
sType = type(xThing).__name__
print('Testing {0}...'.format(sType))
if sType in ['type','int','str', 'bool', 'NoneType', 'unicode']:
print('...too easy')
return lTested
if sType == 'dict':
print('...testing members')
for k in xThing:
lTested = test_pickle(xThing[k],lTested)
print('...tested members')
return lTested
if sType == 'list':
print('...testing members')
for x in xThing:
lTested = test_pickle(x)
print('...tested members')
return lTested
lTested.append(id(xThing))
oClass = type(xThing)
for s in dir(xThing):
if s.startswith('_'):
print('...skipping *private* thingy')
continue
#if it is an attribute: Skip it
try:
xClassAttribute = oClass.__getattribute__(oClass,s)
except (AttributeError, TypeError):
pass
else:
if type(xClassAttribute).__name__ == 'property':
print('...skipping property')
continue
xAttribute = xThing.__getattribute__(s)
print('Testing {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
if type(xAttribute).__name__ == 'function':
print("...skipping function")
continue
if type(xAttribute).__name__ in ['method', 'instancemethod']:
print('...skipping method')
continue
if type(xAttribute).__name__ == 'HtmlElement':
continue
if type(xAttribute) == dict:
print('...testing dict values for {0}.{1}'.format(sType,s))
for k in xAttribute:
lTested = test_pickle(xAttribute[k])
continue
print('...finished testing dict values for {0}.{1}'.format(sType,s))
try:
oIter = xAttribute.__iter__()
except (AttributeError, TypeError):
pass
except AssertionError:
pass #lxml elements do this
else:
print('...testing iter values for {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
for x in xAttribute:
lTested = test_pickle(x,lTested)
print('...finished testing iter values for {0}.{1}'.format(sType,s))
try:
xAttribute.__dict__
except AttributeError:
pass
else:
#this attribute should be explored seperately...
lTested = test_pickle(xAttribute,lTested)
continue
print(0, xThing, xAttribute)
pickle.dumps(xAttribute)
print('Testing {0} as complete object'.format(sType))
pickle.dumps(xThing)
return lTested
I found this the most useful option (also from the far more forgiving dill at places where pickle wasn't). You can simply run it with
test_pickle(my_complex_object)
print("Done!")
