We Keep Coding

Save data into two separate Parse apps

I wrote a small Python Django program that parses data from a JSON API call and saves it into Parse, using ParsePy.
I have a python file that collects the data and saves it into a Parse app DB. The Python file also passes some data into a different file that should save the passed data into a different Parse app.
In pseudocode:
File1.py
register('key1', 'restKey1')
file2.class1(passedData)
file1.saveData
File2.py
register('key2','restKey2')
file2.saveData
When I run the files individually, the code works perfectly. However, when I execute the program through the first file, the data is all getting saved into the first Parse app database instead of the second one.

I think you can use pattern like this:
#!/usr/bin/python
class SourceInterface(object):
def get_data(self):
raise NotImplementedError("Subclasses should implement this!")
class DestinationInterface(object):
def put_data(self, data):
raise NotImplementedError("Subclasses should implement this!")
class FileSource(SourceInterface):
def __init__(self, filename):
self.filename = filename
def get_data(self):
lines = None
with open(self.filename, 'r') as f:
lines = f.readlines()
if lines:
with open(self.filename, 'w') as f:
if lines[1:]:
f.writelines(lines[1:])
return lines[0]
class FileDestination(DestinationInterface):
def __init__(self, filename):
self.filename = filename
def put_data(self, data):
print 'put data', data
with open(self.filename, 'a+') as f:
f.write(data)
class DataProcessor(object):
sources_list = []
destinitions_list = []
def register_source(self, source):
self.sources_list.append(source)
def register_destinition(self, destinition):
self.destinitions_list.append(destinition)
def process(self):
for source in self.sources_list:
data = source.get_data()
if data:
for destinition in self.destinitions_list:
destinition.put_data(data)
if __name__ == '__main__':
processor = DataProcessor()
processor.register_source(FileSource('/tmp/source1.txt'))
processor.register_source(FileSource('/tmp/source2.txt'))
processor.register_destinition(FileDestination('/tmp/destinition1.txt'))
processor.register_destinition(FileDestination('/tmp/destinition2.txt'))
processor.process()
Just define you own Source and Destination classes

python class file context manager

I am trying to open a file in a class and close it on exit in this manner.
class PlanetaryImage(object):
#classmethod
def open(cls, filename):
with open(filename, 'rb') as fp:
return cls(fp, filename)
def __init__(self, stream, filename=None, memory_layout='DISK'):
self.filename = filename
self._parse_data(stream)
def _parse_data(self, stream):
data_stream = stream
try:
if self.data_filename is not None:
dirpath = os.path.dirname(self.filename)
data_file = os.path.abspath(
os.path.join(dirpath, self.data_filename))
data_stream = open(data_file, 'rb')
data_stream.seek(self.start_byte)
if self.format in self.BAND_STORAGE_TYPE:
return getattr(self, self.BAND_STORAGE_TYPE[self.format])(data_stream)
raise Exception('Unkown format (%s)' % self.format)
finally:
data_stream.close()
There are certain cases where I am having to use open one more file in _parse_data function. I wanted to use with but the if statements make it difficult. Any suggestions on how to make the try section more pythonic.

There's no reason for _parse_data to try to open a file. It should be the caller's responsibility to either use PlanetaryImage.open with a file name or to provide an open file handle to __init__. _parse_data should do just one thing: parse the data from its stream argument.
class PlanetaryImage(object):
#classmethod
def open(cls, filename):
with open(filename, 'rb') as fp:
return cls(fp, filename)
def __init__(self, stream, memory_layout='DISK'):
self._parse_data(stream)
def _parse_data(self, data_stream):
try:
data_stream.seek(self.start_byte)
if self.format in self.BAND_STORAGE_TYPE:
return getattr(self, self.BAND_STORAGE_TYPE[self.format])(data_stream)
raise Exception('Unkown format (%s)' % self.format)
finally:
data_stream.close()
Now, there are simply two options for using the class:
with open(filename, 'rb') as fp:
x = PlanetaryImage(fp)
...
or
x = PlanetaryImage(filename)
....

python: prevent creating empty files

For a bulk task I create a couple of instances of the ProgressLog object which will each create an empty log-file no matter if there actually will be any errors. what is the best way to prevent this?
class ProgressLog(object):
"""Write a message to log + progress indicator.
"""
total = 0
def __init__(self, name):
source_path, file_name = os.path.split(name)
self.name = file_name
self.source_path = source_path
self.log_dir_name = r'log'
self.make_log_dir(self.source_path, self.log_dir_name)
self.reset()
log_file = self._logfilename()
try:
self.f = open(log_file, 'w')
print('\n***logging errors to {0}***\n'.format(log_file))
except IOError, err:
msg = 'Cannot open logfile {0}. Traceback is: {1}'.format(
log_file, err)
raise msg
def _logfilename(self):
## hms_ddmmyyyy format
log_name = r'{1}_{0}{2}_errors.csv'.format(
time.strftime("%I%M%S"),
time.strftime("%d%m%Y"),
self.name)
return os.path.join(self.source_path, self.log_dir_name, log_name)

There is no "magical" way to do it, you simply need to refactor the code to open the log file only on first actual call to log.
To achieve this, extract the part of __init__ that opens the log file into a separate _open_log method. In __init__ initialize self.f to None. Then, your actual logging method can begin with:
if self.f is None:
self._open_log()

Python use lxml with fileinput

Having a simple xml
<?xml version="1.0" encoding="UTF-8" ?>
<root>
<child>abc</child>
</root>
I wanted to parse it from file and this works well:
with open('tst.xml') as test_xml:
for _, element in lxml.etree.iterparse(test_xml, tag='child'):
print element.text # prints abc as expected
However, I tried to modify script then to allow it parse xml either from file or from stdin and did not succeed:
fi = fileinput.input('tst.xml')
for _, element in lxml.etree.iterparse(fi, tag='child'):
print element.text
# File "iterparse.pxi", line 371, in lxml.etree.iterparse.__init__ (src/lxml/lxml.etree.c:97283)
# File "apihelpers.pxi", line 1411, in lxml.etree._encodeFilename (src/lxml/lxml.etree.c:22515)
# TypeError: Argument must be string or unicode.
I'm not sure in what I'm doing wrong. Is the FileInput object not a file-like object in python?

Without deep investigation, it seems that the reason of an exception is that FileInput class does not provide read method.
To achieve my goal, I ended up writing my own wrapper for now:
class FileInput(object):
def __init__(self, filename=None, *args, **kwargs):
self.file = open(filename, *args, **kwargs) if filename and filename != "-" else sys.stdin
def __enter__(self):
return self.file
def __exit__(self, type, value, traceback):
if self.file is not sys.stdin:
self.file.close()
def __getattr__(self, name):
return getattr(self.file, name)
I'll be waiting for a better answer though.

You should not try to use the fileinput module, but directly do it like that:
if filename == '-': # or, if we don't have a filename argument
f = sys.stdin
else:
f = open(filename, 'r')

how to open a csv in universal new line mode through django upload?

I am trying to upload a csv file in a django form:
class CSVUploadForm(forms.Form):
csv_file = forms.FileField(label='Select a CSV file to import:',)
def clean(self):
file_csv = self.cleaned_data['csv_file']
records = csv.reader(open('/mypath/'+file_csv.name, 'rU'), dialect=csv.excel_tab)
I need to open the file in universal new line mode. I can do that with "open" method above, but that will not work for this form because the file I am dealing with is an in memory uploaded version of the csv.
How do I pass the universal new line mode flag rU to something like this:
records = csv.reader(file_csv, dialect=csv.excel_tab)
?

You can use str.splitlines() -- which automatically splits on universale line-breaks -- in the following manner:
def clean(self):
file_csv = self.cleaned_data['csv_file']
lines = file_csv.read().splitlines()
records = csv.reader(lines, dialect=csv.excel_tab)
If you are worried about the memory cost of creating the lines variable, you can force Django to save the file to a local file on disk changing the FILE_UPLOAD_MAX_MEMORY_SIZE variable in settings.py (more on this variable here):
# add to your settings.py
FILE_UPLOAD_MAX_MEMORY_SIZE = 0
FILE_UPLOAD_TEMP_DIR = '/tmp'
Then to process the file from it's tmp folder using universal mode:
def clean(self):
file_csv = open(self.cleaned_data['csv_file'].temporary_file_path, 'rU')
records = csv.reader(file_csv, dialect=csv.excel_tab)

Problem with the solution above is that it reads the whole file all at once, make it not acceptable when processing large csv file. For small CSV files files will be saved to disk instead of being kept in memory which is also not so great.
I've created a class to handle new lines
class FileWithUniversalNewLine(object):
def __init__(self, file_obj):
self.file = file_obj
def lines(self):
buff = "" # In case of reading incomplete line, buff will temporarly keep the incomplete line
while True:
line = self.file.read(2048)
if not line:
if buff:
yield buff
raise StopIteration
# Convert all new lines into linux new line
line = buff + line.replace("\r\n", "\n").replace("\r", "\n")
lines = line.split("\n")
buff = lines.pop()
for sline in lines:
yield sline
def close(self):
self.file.close()
def __exit__(self, *args, **kwargs):
return self.file.__exit__(*args, **kwargs)
def __enter__(self, *args, **kwargs):
return self
def __iter__(self):
return self.lines()
Usage:
csvfile = FileWithUniversalNewLine(file_csv)
records = csv.reader(csvfile, dialect=csv.excel_tab)