I am using pickle to save an object graph by dumping the root. When I load the root it has all the instance variables and connected object nodes. However I am saving all the nodes in a class variable of type dictionary. The class variable is full before being saved but after I unpickle the data it is empty.
Here is the class I am using:
class Page():
__crawled = {}
def __init__(self, title = '', link = '', relatedURLs = []):
self.__title = title
self.__link = link
self.__relatedURLs = relatedURLs
self.__related = []
#property
def relatedURLs(self):
return self.__relatedURLs
#property
def title(self):
return self.__title
#property
def related(self):
return self.__related
#property
def crawled(self):
return self.__crawled
def crawl(self,url):
if url not in self.__crawled:
webpage = urlopen(url).read()
patFinderTitle = re.compile('<title>(.*)</title>')
patFinderLink = re.compile('<link rel="canonical" href="([^"]*)" />')
patFinderRelated = re.compile('<li><a href="([^"]*)"')
findPatTitle = re.findall(patFinderTitle, webpage)
findPatLink = re.findall(patFinderLink, webpage)
findPatRelated = re.findall(patFinderRelated, webpage)
newPage = Page(findPatTitle,findPatLink,findPatRelated)
self.__related.append(newPage)
self.__crawled[url] = newPage
else:
self.__related.append(self.__crawled[url])
def crawlRelated(self):
for link in self.__relatedURLs:
self.crawl(link)
I save it like such:
with open('medTwiceGraph.dat','w') as outf:
pickle.dump(root,outf)
and I load it like such:
def loadGraph(filename): #returns root
with open(filename,'r') as inf:
return pickle.load(inf)
root = loadGraph('medTwiceGraph.dat')
All the data loads except for the class variable __crawled.
What am I doing wrong?
Python doesn't really pickle class objects. It simply saves their names and where to find them. From the documentation of pickle:
Similarly, classes are pickled by named reference, so the same
restrictions in the unpickling environment apply. Note that none of
the class’s code or data is pickled, so in the following example the
class attribute attr is not restored in the unpickling environment:
class Foo:
attr = 'a class attr'
picklestring = pickle.dumps(Foo)
These restrictions are why picklable functions and classes must be
defined in the top level of a module.
Similarly, when class instances are pickled, their class’s code and
data are not pickled along with them. Only the instance data are
pickled. This is done on purpose, so you can fix bugs in a class or
add methods to the class and still load objects that were created with
an earlier version of the class. If you plan to have long-lived
objects that will see many versions of a class, it may be worthwhile
to put a version number in the objects so that suitable conversions
can be made by the class’s __setstate__() method.
In your example you could fix your problems changing __crawled to be an instance attribute or a global variable.
By default pickle will only use the contents of self.__dict__ and not use self.__class__.__dict__ which is what you think you want.
I say, "what you think you want" because unpickling an instance should not mutate class level sate.
If you want to change this behavior then look at __getstate__ and __setstate__ in the docs
For anyone interested, what I did was make a superclass Graph which contained an instance variable __crawled and moved my crawling functions into Graph. Page now only contains attributes describing the page and its related pages. I pickle my instance of Graph which contains all my instances of Page. Here is my code.
from urllib import urlopen
#from bs4 import BeautifulSoup
import re
import pickle
###################CLASS GRAPH####################
class Graph(object):
def __init__(self,roots = [],crawled = {}):
self.__roots = roots
self.__crawled = crawled
#property
def roots(self):
return self.__roots
#property
def crawled(self):
return self.__crawled
def crawl(self,page,url):
if url not in self.__crawled:
webpage = urlopen(url).read()
patFinderTitle = re.compile('<title>(.*)</title>')
patFinderLink = re.compile('<link rel="canonical" href="([^"]*)" />')
patFinderRelated = re.compile('<li><a href="([^"]*)"')
findPatTitle = re.findall(patFinderTitle, webpage)
findPatLink = re.findall(patFinderLink, webpage)
findPatRelated = re.findall(patFinderRelated, webpage)
newPage = Page(findPatTitle,findPatLink,findPatRelated)
page.related.append(newPage)
self.__crawled[url] = newPage
else:
page.related.append(self.__crawled[url])
def crawlRelated(self,page):
for link in page.relatedURLs:
self.crawl(page,link)
def crawlAll(self,obj,limit = 2,i = 0):
print 'number of crawled pages:', len(self.crawled)
i += 1
if i > limit:
return
else:
for rel in obj.related:
print 'crawling', rel.title
self.crawlRelated(rel)
for rel2 in obj.related:
self.crawlAll(rel2,limit,i)
def loadGraph(self,filename):
with open(filename,'r') as inf:
return pickle.load(inf)
def saveGraph(self,obj,filename):
with open(filename,'w') as outf:
pickle.dump(obj,outf)
###################CLASS PAGE#####################
class Page(Graph):
def __init__(self, title = '', link = '', relatedURLs = []):
self.__title = title
self.__link = link
self.__relatedURLs = relatedURLs
self.__related = []
#property
def relatedURLs(self):
return self.__relatedURLs
#property
def title(self):
return self.__title
#property
def related(self):
return self.__related
####################### MAIN ######################
def main(seed):
print 'doing some work...'
webpage = urlopen(seed).read()
patFinderTitle = re.compile('<title>(.*)</title>')
patFinderLink = re.compile('<link rel="canonical" href="([^"]*)" />')
patFinderRelated = re.compile('<li><a href="([^"]*)"')
findPatTitle = re.findall(patFinderTitle, webpage)
findPatLink = re.findall(patFinderLink, webpage)
findPatRelated = re.findall(patFinderRelated, webpage)
print 'found the webpage', findPatTitle
#root = Page(findPatTitle,findPatLink,findPatRelated)
G = Graph([Page(findPatTitle,findPatLink,findPatRelated)])
print 'crawling related...'
G.crawlRelated(G.roots[0])
G.crawlAll(G.roots[0])
print 'now saving...'
G.saveGraph(G, 'medTwiceGraph.dat')
print 'done'
return G
#####################END MAIN######################
#'http://medtwice.com/am-i-pregnant/'
#'medTwiceGraph.dat'
#G = main('http://medtwice.com/menopause-overview/')
#print G.crawled
def loadGraph(filename):
with open(filename,'r') as inf:
return pickle.load(inf)
G = loadGraph('MedTwiceGraph.dat')
print G.roots[0].title
print G.roots[0].related
print G.crawled
for key in G.crawled:
print G.crawled[key].title
Using dill can solve this problem.
dill package: https://pypi.python.org/pypi/dill
reference: https://stackoverflow.com/a/28543378/6301132
According Asker's code, into this:
#notice：open the file in binary require
#save
with open('medTwiceGraph.dat','wb') as outf:
dill.dump(root,outf)
#load
def loadGraph(filename): #returns root
with open(filename,'rb') as inf:
return dill.load(inf)
root = loadGraph('medTwiceGraph.dat')
I wrote another example:
#Another example (with Python 3.x)
import dill
import os
class Employee:
def __init__ (self ,name='',contact={}) :
self.name = name
self.contact = contact
def print_self(self):
print(self.name, self.contact)
#save
def save_employees():
global emp
with open('employees.dat','wb') as fh:
dill.dump(emp,fh)
#load
def load_employees():
global emp
if os.path.exists('employees.dat'):
with open('employees.dat','rb') as fh:
emp=dill.load(fh)
#---
emp=[]
load_employees()
print('loaded:')
for tmpe in emp:
tmpe.print_self()
e=Employee() #new employee
if len(emp)==0:
e.name='Jack'
e.contact={'phone':'+086-12345678'}
elif len(emp)==1:
e.name='Jane'
e.contact={'phone':'+01-15555555','email':'a#b.com'}
else:
e.name='sb.'
e.contact={'telegram':'x'}
emp.append(e)
save_employees()
Related
I have been working on a small project which is a web-crawler template. Im having an issue in pycharm where I am getting a warning Unresolved attribute reference 'domain' for class 'Scraper'
from abc import abstractmethod
import requests
import tldextract
class Scraper:
scrapers = {}
def __init_subclass__(scraper_class):
Scraper.scrapers[scraper_class.domain] = scraper_class # Unresolved attribute reference 'domain' for class 'Scraper'
#classmethod
def for_url(cls, url):
k = tldextract.extract(url)
# Returns -> <scraper.SydsvenskanScraper object at 0x000001E94F135850> & Scraped BBC News<!DOCTYPE html><html Which type annotiation?
return cls.scrapers[k.registered_domain](url)
#abstractmethod
def scrape(self):
pass
class BBCScraper(Scraper):
domain = 'bbc.co.uk'
def __init__(self, url):
self.url = url
def scrape(self):
rep = requests.Response = requests.get(self.url)
return "Scraped BBC News" + rep.text[:20] # ALL HTML CONTENT
class SydsvenskanScraper(Scraper):
domain = 'sydsvenskan.se'
def __init__(self, url):
self.url = url
def scrape(self):
rep = requests.Response = requests.get(self.url)
return "Scraped Sydsvenskan News" + rep.text[:20] # ALL HTML CONTENT
if __name__ == "__main__":
URLS = ['https://www.sydsvenskan.se/', 'https://www.bbc.co.uk/']
for urls in URLS:
get_product = Scraper.for_url(urls)
r = get_product.scrape()
print(r)
Of course I could ignore it as it is working but I do not like to ignore a warning as I believe pycharm is smart and should solve the warning rather than ignoring it and I wonder what is the reason of it warns me regarding that?
There are a few different levels on how you can remove this warning:
Assign a default value:
class Scraper:
scrapers = {}
domain = None # Or a sensible value of one exists
You can in additon or alternatly annotate the type.
from typing import ClassVar
class Scraper:
scrapers: ClassVar[dict[str, 'Scraper']] = {}
domain: ClassVar[str]
Note that ClassVar is required because otherwise it is assume that they are instance attributes.
To ignore it, put
# noinspection PyUnresolvedReferences
on the line above the line causing the warning.
Just tell yrou Scraper class that this attribut exists
class Scraper:
scrapers = {}
domain: str
def __init_subclass__(scraper_class):
Scraper.scrapers[scraper_class.domain] = scraper_class
Hello Stackoverflow folks,...
I hope this questions is not already answered.
After half a day of googeling I did resign myself to asking a question here.
My problem is the following:
I want to create a class which takes some information and processes this information:
#Klassendefinition für eine Instanz von Rohdaten
class raw_data():
def __init__(self, filename_rawdata, filename_metadata,
file_format, path, category, df_raw, df_meta):
self.filename_rawdata = filename_rawdata
self.filename_metadata = filename_metadata
self.file_format = file_format
self.path = path
self.category = category
self.df_raw = getDF(self.filename_rawdata)
self.df_meta = getDF(self.filename_metadata)
# generator
def parse(self, path):
g = gzip.open(path, 'rb')
for l in g:
yield eval(l)
# function that returns a pandas dataframe with the data
def getDF(self, filename):
i = 0
df = {}
for d in self.parse(filename):
df[i] = d
i += 1
return pd.DataFrame.from_dict(df, orient='index')
Now I have a problem with the init method, I would like to run the class method below on default when the class in instantiated, but I somehow cannot manage to get this working. I have seen several other posts here like [Calling a class function inside of __init__ [1]: Python 3: Calling a class function inside of __init__
but I am still not able to do it. The first question did work for me, but I would like to call the instance variable after the constructor ran.
I tried this:
class raw_data():
def __init__(self, filename_rawdata, filename_metadata,
file_format, path, category):
self.filename_rawdata = filename_rawdata
self.filename_metadata = filename_metadata
self.file_format = file_format
self.path = path
self.category = category
getDF(self.filename_rawdata)
getDF(self.filename_metadata)
# generator
def parse(self, path):
g = gzip.open(path, 'rb')
for l in g:
yield eval(l)
# function that returns a pandas dataframe with the data
def getDF(self, filename):
i = 0
df = {}
for d in self.parse(filename):
df[i] = d
i += 1
return pd.DataFrame.from_dict(df, orient='index')
But I get an error because getDF is not defined (obviously)..
I hope this questions is not silly by any means. I need to do it that way, because afterwards I want to run like 50-60 instance calls and I do not want to repeat like Instance.getDF() ... for every instance, but rather would like to have it called directly.
All you need to so is call getDF like any other method, using self as the object on which it should be invoked.
self.df_raw = self.getDF(self.filename_rawdata)
That said, this class could be greatly simplified by making it a dataclass.
from dataclasses import dataclass
#dataclass
class RawData:
filename_rawdata: str
filename_metadata: str
path: str
category: str
def __post_init__(self):
self.df_raw = self.getDF(self.filename_rawdata)
self.df_meta = self.getDF(self.filename_metadata)
#staticmethod
def parse(path):
with gzip.open(path, 'rb') as g:
yield from map(eval, g)
#staticmethod
def getDF(filename):
return pd.DataFrame.from_records(enumerate(RawData.parse(filename)))
The auto-generated __init__ method will set the four defined attributes for you. __post_init__ will be called after __init__, giving you the opportunity to call getDF on the two given file names.
I'm trying to serialize a large number of custom classes to disk using python's pickle. However, the classes aren't serializing properly.
Having looked at the docs it's my understanding what I'm trying to do ought to work, but perhaps it's not simply because I'm not understanding something.
My classes are defined at the top-level of the module. And no PicklingError exception is being fired when trying to pickle.
Here is my sample code. Uncomment Save() to serialize; uncomment Load() to load. When loading, the Synonyms array of Term isn't being populated, but the Main object of Term is being deserialized. You can see this by inspecting the "loadedTerms" object being returned from the Load() function.
What am I doing wrong? Thanks.
import pickle
class Entry:
Text = ""
def __init__(self, text):
self.Text = text
class Term:
Main = None
Synonyms = []
def Save():
term = Term()
term.Main = Entry("Dog")
term.Synonyms.append(Entry("Canine"))
term.Synonyms.append(Entry("Pursue"))
term.Synonyms.append(Entry("Follow"))
term.Synonyms.append(Entry("Plague"))
terms = []
terms.append(term)
with open('output.pickle', 'wb') as p:
pickle.dump(terms, p)
def Load():
loadedTerms = []
with open('output.pickle', 'rb') as p:
loadedTerms = pickle.load(p)
return loadedTerms
#Save()
#terms = Load()
Pickle only saves the instance attributes of a class, but Synonyms is a list, defined at class level. You should create the list in a __init__-method:
import pickle
class Entry:
def __init__(self, text):
self.Text = text
class Term:
def __init__(self):
self.Main = None
self.Synonyms = []
def Save():
term = Term()
term.Main = Entry("Dog")
term.Synonyms.append(Entry("Canine"))
term.Synonyms.append(Entry("Pursue"))
term.Synonyms.append(Entry("Follow"))
term.Synonyms.append(Entry("Plague"))
terms = []
terms.append(term)
with open('output.pickle', 'wb') as p:
pickle.dump(terms, p)
def Load():
with open('output.pickle', 'rb') as p:
loadedTerms = pickle.load(p)
return loadedTerms
I have a question regarding a Python class I use in Blender. Basically, I wonder how the class works because some attributes are recorded without me specifically writing self.value = something. Here's the code:
class DialogOperator(bpy.types.Operator):
bl_idname = "object.dialog_operator"
bl_label = "Save/Load animation"
saving = bpy.props.BoolProperty(name="Save ? Else load.")
path_to_anim = bpy.props.StringProperty(name="Path to folder")
anim_name = bpy.props.StringProperty(name="Animation name:")
# path_to_anim += "/home/mehdi/Blender/Scripts/"
def execute(self, context):
# print('This is execute with: Saving: {} Name:{}'.format(self.saving, self.path_to_anim))
if self.saving:
self.launch_save()
message = 'Animation {} saved at {}'.format(self.anim_name, self.path_to_anim)
else:
self.launch_load()
message = 'Animation {} loaded'.format(self.anim_name)
self.report({'INFO'}, message)
return {'FINISHED'}
def invoke(self, context, event):
wm = context.window_manager
return wm.invoke_props_dialog(self)
def launch_load(self):
full_path = self.path_to_anim + self.anim_name
target_armature = Humanoid(bpy.data.objects['Armature'])
load_all(full_path, target_armature, 'LastLoaded')
def launch_save(self):
full_path = self.path_to_anim + self.anim_name
source_armature = Humanoid(bpy.data.objects['Armature'])
curves = source_armature.get_curves()
save_all(curves, source_armature,full_path)
Now, how come saving, path_to_anim and anim_name are considered as attributes (I'm able to call them in execute() and launch()) even though I did not write self.saving = saving
Thanks !
This is because saving,path_to_anim and anim_name are class attributes. They are defined for the class and not for a particular instance. They are shared among the instances. Here is a link for further explanation class-instance-attributes-python
I'm trying to serialize a large number of custom classes to disk using python's pickle. However, the classes aren't serializing properly.
Having looked at the docs it's my understanding what I'm trying to do ought to work, but perhaps it's not simply because I'm not understanding something.
My classes are defined at the top-level of the module. And no PicklingError exception is being fired when trying to pickle.
Here is my sample code. Uncomment Save() to serialize; uncomment Load() to load. When loading, the Synonyms array of Term isn't being populated, but the Main object of Term is being deserialized. You can see this by inspecting the "loadedTerms" object being returned from the Load() function.
What am I doing wrong? Thanks.
import pickle
class Entry:
Text = ""
def __init__(self, text):
self.Text = text
class Term:
Main = None
Synonyms = []
def Save():
term = Term()
term.Main = Entry("Dog")
term.Synonyms.append(Entry("Canine"))
term.Synonyms.append(Entry("Pursue"))
term.Synonyms.append(Entry("Follow"))
term.Synonyms.append(Entry("Plague"))
terms = []
terms.append(term)
with open('output.pickle', 'wb') as p:
pickle.dump(terms, p)
def Load():
loadedTerms = []
with open('output.pickle', 'rb') as p:
loadedTerms = pickle.load(p)
return loadedTerms
#Save()
#terms = Load()
Pickle only saves the instance attributes of a class, but Synonyms is a list, defined at class level. You should create the list in a __init__-method:
import pickle
class Entry:
def __init__(self, text):
self.Text = text
class Term:
def __init__(self):
self.Main = None
self.Synonyms = []
def Save():
term = Term()
term.Main = Entry("Dog")
term.Synonyms.append(Entry("Canine"))
term.Synonyms.append(Entry("Pursue"))
term.Synonyms.append(Entry("Follow"))
term.Synonyms.append(Entry("Plague"))
terms = []
terms.append(term)
with open('output.pickle', 'wb') as p:
pickle.dump(terms, p)
def Load():
with open('output.pickle', 'rb') as p:
loadedTerms = pickle.load(p)
return loadedTerms