I have a form with a column of checkboxes corresponding to the columns in my database. I'm setting the value of each checkbox (in javascript) to the name of the column, but when I try to read the checkbox value in Flask/Python all I can get is True or False. How do I read the text value of the value attribute of the checkboxes?
Just to complicate things, I'm generating the form as a FieldList of FormFields, so I can't simply hardcode the field names. (Well, I could, but that would make it fragile to schema changes.)
My form code is
class ImportFilterSubForm(Form):
use = BooleanField(
'Use',
render_kw={'class': 'use'}
)
sel = SelectField(
'Maps to:',
choices=[],
render_kw={'class': 'sel'},
validators=[Optional()]
)
class ImportFilterForm(FlaskForm):
rows = FieldList(FormField(ImportFilterSubForm))
My view code, with error handling removed, is
def prefilter_import():
db_columns = Contact.__table__.columns.keys()
filename = request.cookies.get('workfile')
with open(filename) as fh:
reader = csv.DictReader(fh)
file_columns = reader.fieldnames
form = ImportFilterForm()
for col in db_columns:
new_row = form.rows.append_entry()
new_row.use.label = col
new_row.sel.choices = file_columns
input_file = request.cookies.get('input_file')
return render_template('filter_import.html', form=form, filename=input_file)
def postfilter_import():
form = ImportFilterForm()
db_columns = Contact.__table__.columns.keys()
filename = request.cookies.get('workfile')
with open(filename) as fh:
reader = csv.DictReader(fh)
file_columns = reader.fieldnames
missing_columns = db_columns
extra_columns = file_columns
mappings = dict()
for i, row in enumerate(form.rows):
if row.use.data: # Problem arises here
mappings[row.use.data] = row.sel.data
for key, value in mappings.items():
missing_columns.remove(key) # Problem manifests here
extra_columns.remove(value)
I'm trying to create a dict mapping the values of the checkboxes to the values of the selects, but I'm only getting True and False for the checkboxes, even though I've verified that the checkboxes' value attributes are correctly returned as the names of the corresponding columns.
How can I get Flask/WTForms to return the text of the value attributes?
After some investigation, I discovered the raw_data attribute of the field object, which contains a list of the values of the value attributes of the HTML control. Thus, the code
mappings = dict()
for row in form.rows:
if row.use.data:
mappings[row.sel.data] = row.use.raw_data[0]
for key, value in mappings.items():
missing_columns.remove(value)
extra_columns.remove(key)
does what I need it to do.
Related
I currently override the xlsxwriter.Workbook, called rpt.Workbook. Would like to add method to xlsxwriter.Worksheet(), but since xlsxwriter.Workbook() imports Worksheet,not sure how this can be done without major convulsions. Currently, I have to pass the worksheet object as an argument.
Seems like I need to write override methods for xlsxwriter.Workbook() to point to a custom class of xlsxwriter.Worksheet() , but can't figure out how.
Here is the current override rpt.Workbook() being used:
####################################################################
class Workbook(xlsxwriter.Workbook):
####################################################################
"""\nSpreadsheet class provides methods to build a spreadsheet.
"""
####################################################################
def __init__(self,filename=None, options={}):
####################################################################
try:
filename = rpt.fname(filename)
except FileNotFoundError as err:
log.error(err)
return False
log.info("Initializing excel file " + filename)
super().__init__(filename,options)
####################################################################
def add_tab(self,name=None,data=None,header=None,
srow=0,scol=0,autocol=True):
####################################################################
"""\nProvides method to add_worksheet and add_table in 1 call.
Required Attribute args:
name = TAB name
header = list of header names
data = list of lists for spreadsheet contents
Optional Attribute args:
srow = starting row for table, default 0
scol = starting col for table, default 0
autocol = True/False, auto set the column sizes
add_tab also adds the worksheet.header attribute to
allow the set_col_by_name function to work
"""
if not data:
log.warning("data=[][] required")
return None
if not header:
log.warning("header=[] required")
return False
columns = []
for field in header:
columns.append({ 'header' : field })
worksheet = self.add_worksheet(name)
worksheet.header = header
tableinfo= {
'data' : data,
'columns' : columns
}
lastcol = scol + (len(header) - 1)
lastrow = srow + (len(data) + 1)
worksheet.add_table(srow,scol,lastrow,lastcol,tableinfo)
#if autocol:
#self.auto_set_columns(worksheet=worksheet,data=data,scol=scol)
worksheet.freeze_panes(0,1)
return worksheet
####################################################################
def auto_set_columns(self,worksheet=None,data=None,header=None,scol=0):
####################################################################
"""\nDetermines the max length of each column and then set
that column width.
Required Attribute args:
worksheet = worksheet object
data = list of lists data
Optional Attribute args:
scol = Column start
header = row of headers for data list of lists.
If header not specified, worksheet
must have been created with self.add_tab()
"""
if not header and worksheet.header:
header = worksheet.header
## table = [] list of lists, combine header and data
table = []
table.append(header) for row in data:
table.append(row)
ziptable = list(zip (*table))
colsizes = []
for idx,val in enumerate(table[0]):
size = max(len(s) for s in ziptable[idx])
colnum = idx + scol
log.debug("Setting column => {} col size => {} => {}".format(colnum,val,size))
worksheet.set_column(colnum,colnum,size)
I want to add a method to xlsxwriter.Worksheet() called auto_set_columns(). Currently I have to pass the worksheet object (worksheet=worksheet) as an object to get this to work.I would like to utilize worksheet.auto_set_columns() instead. Currently auto_set_columns() is a method of rpt.Workbook.
Would like auto_set_columns() to be an extended method of xlsxwriter.Worksheet.
The script side utilization currently looks like this and works:
excelfile = nashomes + '/nas_homes.xlsx'
spreadsheet = rpt.Workbook(excelfile)
worksheet = spreadsheet.add_tab(name='Nas Homes',data=hrpt.data,header=hrpt.header)
spreadsheet.auto_set_columns(worksheet=worksheet,data=hrpt.data,scol=0)
What I desire, notice the last line changes:
excelfile = nashomes + '/nas_homes.xlsx'
spreadsheet = rpt.Workbook(excelfile)
worksheet = spreadsheet.add_tab(name='Nas Homes',data=hrpt.data,header=hrpt.header)
worksheet.auto_set_columns(data=hrpt.data,scol=0)
Goal desired here is that worksheet object (which is xlsxwriter.Worksheet() ) can have an extended "auto_set_columns" method. However, since the worksheet object is created from a add_worksheet() method in the xlsxwriter.Workbook() class, I can't figure out how to extend xlsxwriter.Worksheet() without major override methods to xlsxwriter.Workbook() also. How can I get xlsxwriter.Workbook() to reference my extended xlsxwriter.Worksheet() when Workbook.add_worksheet() creates the Worksheet object?
How about monkey patching the worksheet before returning it from add_tab?
First create the standalone function outside of any class definitions:
import types
def auto_set_columns(self,data=None,header=None,scol=0):
if not header and self.header:
header = self.header
## table = [] list of lists, combine header and data
table = []
table.append(header)
for row in data:
table.append(row)
ziptable = list(zip (*table))
colsizes = []
for idx,val in enumerate(table[0]):
size = max(len(s) for s in ziptable[idx])
colnum = idx + scol
print "Setting column => {} col size => {} => {}".format(colnum,val,size)
self.set_column(colnum,colnum,size)
And then inside your Worksheet.add_tab function, patch in the method before returning:
....
worksheet.freeze_panes(0,1)
worksheet.auto_set_columns = types.MethodType( auto_set_columns, worksheet )
return worksheet
Now you should be able to run:
worksheet = spreadsheet.add_tab(name='Nas Homes',data=hrpt.data,header=hrpt.header)
worksheet.auto_set_columns(data=hrpt.data,scol=0)
I'm writing a python scraper code for OpenData and I have one question about : how to check if all values aren't filled in site and if it is null change value to null.
My scraper is here.
Currently I'm working on it to optimalize.
My variables now look like:
evcisloval = soup.find_all('td')[3].text.strip()
prinalezival = soup.find_all('td')[5].text.strip()
popisfaplnenia = soup.find_all('td')[7].text.replace('\"', '')
hodnotafaplnenia = soup.find_all('td')[9].text[:-1].replace(",", ".").replace(" ", "")
datumdfa = soup.find_all('td')[11].text
datumzfa = soup.find_all('td')[13].text
formazaplatenia = soup.find_all('td')[15].text
obchmenonazov = soup.find_all('td')[17].text
sidlofirmy = soup.find_all('td')[19].text
pravnaforma = soup.find_all('td')[21].text
sudregistracie = soup.find_all('td')[23].text
ico = soup.find_all('td')[25].text
dic = soup.find_all('td')[27].text
cislouctu = soup.find_all('td')[29].text
And Output :
scraperwiki.sqlite.save(unique_keys=["invoice_id"],
data={ "invoice_id":number,
"invoice_price":hodnotafaplnenia,
"evidence_no":evcisloval,
"paired_with":prinalezival,
"invoice_desc":popisfaplnenia,
"date_received":datumdfa,
"date_payment":datumzfa,
"pay_form":formazaplatenia,
"trade_name":obchmenonazov,
"trade_form":pravnaforma,
"company_location":sidlofirmy,
"court":sudregistracie,
"ico":ico,
"dic":dic,
"accout_no":cislouctu,
"invoice_attachment":urlfa,
"invoice_url":url})
I googled it but without success.
First, write a configuration dict of your variables in the form:
conf = {'evidence_no': (3, str.strip),
'trade_form': (21, None),
...}
i.e. key is the output key, value is a tuple of id from soup.find_all('td') and of an optional function that has to be applied to the result, None otherwise. You don't need those Slavic variable names that may confuse other SO members.
Then iterate over conf and fill the data dict.
Also, run soup.find_all('td') before the loop.
tds = soup.find_all('td')
data = {}
for name, (num, func) in conf.iteritems():
text = tds[num].text
# replace text with None or "NULL" or whatever if needed
...
if func is None:
data[name] = text
else:
data[name] = func(text)
This will remove a lot of duplicated code. Easier to maintain.
Also, I am not sure the strings "NULL" are the best way to write missing data. Doesn't sqlite support Python's real None objects?
Just read your attached link, and it seems what you want is
evcisloval = soup.find_all('td')[3].text.strip() or "NULL"
But be careful. You should only do this with strings. If the part before or is either empty or False or None, or 0, they will all be replaced with "NULL"
Want to prompt browser to save csv
^^working off above question, file is exporting correctly but the data is not displaying correctly.
#view_config(route_name='csvfile', renderer='csv')
def csv(self):
name = DBSession.query(table).join(othertable).filter(othertable.id == 9701).all()
header = ['name']
rows = []
for item in name:
rows = [item.id]
return {
'header': header,
'rows': rows
}
Getting _csv.Error
Error: sequence expected but if I change in my renderer writer.writerows(value['rows']) to writer.writerow(value['rows']) the file will download via the browser just fine. Problem is, it's not displaying data in each row. The entire result/dataset is in one row, so each entry is in it's own column rather than it's own row.
First, I wonder if having a return statement inside your for loop isn't also causing problems; from the linked example it looks like their loop was in the prior statement.
I think what it looks like it's doing is it's building a collection of rows based on "table" having columns with the same name as the headers. What are the fields in your table table?
name = DBSession.query(table).join(othertable).filter(othertable.id == 9701).all()
This is going to give you back essentially a collection of rows from table, as if you did a SELECT query on it.
Something like
name = DBSession.query(table).join(othertable).filter(othertable.id == 9701).all()
header = ['name']
rows = []
for item in name:
rows.append(item.name)
return {
'header': header,
'rows': r
}
Figured it out. kept getting Error: sequence expected so I was looking at the output. Decided to try putting the result inside another list.
#view_config(route_name='csv', renderer='csv')
def csv(self):
d = datetime.now()
query = DBSession.query(table, othertable).join(othertable).join(thirdtable).filter(
thirdtable.sid == 9701)
header = ['First Name', 'Last Name']
rows = []
filename = "csvreport" + d.strftime(" %m/%d").replace(' 0', '')
for i in query:
items = [i.table.first_name, i.table.last_name, i.othertable.login_time.strftime("%m/%d/%Y"),
]
rows.append(items)
return {
'header': header,
'rows': rows,
'filename': filename
}
This accomplishes 3 things. Fills out the header, fills the rows, and passes through a filename.
Renderer should look like this:
class CSVRenderer(object):
def __init__(self, info):
pass
def __call__(self, value, system):
fout = StringIO.StringIO()
writer = csv.writer(fout, delimiter=',',quotechar =',',quoting=csv.QUOTE_MINIMAL)
writer.writerow(value['header'])
writer.writerows(value['rows'])
resp = system['request'].response
resp.content_type = 'text/csv'
resp.content_disposition = 'attachment;filename='+value['filename']+'.csv'
return fout.getvalue()
This way, you can use the same csv renderer anywhere else and be able to pass through your own filename. It's also the only way I could figure out how to get the data from one column in the database to iterate through one column in the renderer. It feels a bit hacky but it works and works well.
Suppose in my python below function, i am getting the json feeds like below
def mapper_1(self, key, line):
j_feed = json.loads(line)
unicoded = j_feed[u'category_description'].encode("utf-8")
cn = j_feed[u'categoryname']
location = j_feed[u'location']
How to check if there is any blank fields for data in categoryname/categorydescription/location from the input.json.
Say you are unsure of your fields, you can use .get and provide it a default sentinel value
fields = ['categoryname', 'categorydescription', 'location']
for field in fields:
print j_feed.get(field, "not set!")
I am facing a peculiar problem. I will describe in brief bellow
Suppose i have this piece of code -
class MyClass:
__postBodies = []
.
.
.
for the_file in os.listdir("/dir/path/to/file"):
file_path = os.path.join(folder, the_file)
params = self.__parseFileAsText(str(file_path)) #reads the file and gets some parsed data back
dictData = {'file':str(file_path), 'body':params}
self.__postBodies.append(dictData)
print self.__postBodies
dictData = None
params = None
Problem is, when i print the params and the dictData everytime for different files it has different values (the right thing), but as soon as the append occurs, and I print __postBodies a strange thing happens. If there are thee files, suppose A,B,C, then
first time __postBodies has the content = [{'body':{A dict with some
data related to file A}, 'file':'path/of/A'}]
second time it becomes = [{'body':{A dict with some data relaed to
file B}, 'file':'path/of/A'}, {'body':{A dict with some data relaed to
file B}, 'file':'path/of/B'}]
AND third time = [{'body':{A dict with some data relaed to file C},
'file':'path/of/A'}, {'body':{A dict with some data relaed to file C},
'file':'path/of/B'}, {'body':{A dict with some data relaed to file C},
'file':'path/of/C'}]
So, you see the 'file' key is working very fine. Just strangely the 'body' key is getting overwritten for all the entries with the one last appended.
Am i making any mistake? is there something i have to? Please point me to a direction.
Sorry if I am not very clear.
EDIT ------------------------
The return from self.__parseFileAsText(str(file_path)) call is a dict that I am inserting as 'body' in the dictData.
EDIT2 ----------------------------
as you asked, this is the code, but i have checked that params = self.__parseFileAsText(str(file_path)) call is returning a diff dict everytime.
def __parseFileAsText(self, fileName):
i = 0
tempParam = StaticConfig.PASTE_PARAMS
tempParam[StaticConfig.KEY_PASTE_PARAM_NAME] = ""
tempParam[StaticConfig.KEY_PASTE_PARAM_PASTEFORMAT] = "text"
tempParam[StaticConfig.KEY_PASTE_PARAM_EXPIREDATE] = "N"
tempParam[StaticConfig.KEY_PASTE_PARAM_PRIVATE] = ""
tempParam[StaticConfig.KEY_PASTE_PARAM_USER] = ""
tempParam[StaticConfig.KEY_PASTE_PARAM_DEVKEY] = ""
tempParam[StaticConfig.KEY_PASTE_FORMAT_PASTECODE] = ""
for line in fileinput.input([fileName]):
temp = str(line)
temp2 = temp.strip()
if i == 0:
postValues = temp2.split("|||")
if int(postValues[(len(postValues) - 1)]) == 0 or int(postValues[(len(postValues) - 1)]) == 2:
tempParam[StaticConfig.KEY_PASTE_PARAM_NAME] = str(postValues[0])
if str(postValues[1]) == '':
tempParam[StaticConfig.KEY_PASTE_PARAM_PASTEFORMAT] = 'text'
else:
tempParam[StaticConfig.KEY_PASTE_PARAM_PASTEFORMAT] = postValues[1]
if str(postValues[2]) != "N":
tempParam[StaticConfig.KEY_PASTE_PARAM_EXPIREDATE] = str(postValues[2])
tempParam[StaticConfig.KEY_PASTE_PARAM_PRIVATE] = str(postValues[3])
tempParam[StaticConfig.KEY_PASTE_PARAM_USER] = StaticConfig.API_USER_KEY
tempParam[StaticConfig.KEY_PASTE_PARAM_DEVKEY] = StaticConfig.API_KEY
else:
tempParam[StaticConfig.KEY_PASTE_PARAM_USER] = StaticConfig.API_USER_KEY
tempParam[StaticConfig.KEY_PASTE_PARAM_DEVKEY] = StaticConfig.API_KEY
i = i+1
else:
if tempParam[StaticConfig.KEY_PASTE_FORMAT_PASTECODE] != "" :
tempParam[StaticConfig.KEY_PASTE_FORMAT_PASTECODE] = str(tempParam[StaticConfig.KEY_PASTE_FORMAT_PASTECODE])+"\n"+temp2
else:
tempParam[StaticConfig.KEY_PASTE_FORMAT_PASTECODE] = temp2
return tempParam
You are likely returning the same dictionary with every call to MyClass.__parseFileAsText(), a couple of common ways this might be happening:
__parseFileAsText() accepts a mutable default argument (the dict that you eventually return)
You modify an attribute of the class or instance and return that instead of creating a new one each time
Making sure that you are creating a new dictionary on each call to __parseFileAsText() should fix this problem.
Edit: Based on your updated question with the code for __parseFileAsText(), your issue is that you are reusing the same dictionary on each call:
tempParam = StaticConfig.PASTE_PARAMS
...
return tempParam
On each call you are modifying StaticConfig.PASTE_PARAMS, and the end result is that all of the body dictionaries in your list are actually references to StaticConfig.PASTE_PARAMS. Depending on what StaticConfig.PASTE_PARAMS is, you should change that top line to one of the following:
# StaticConfig.PASTE_PARAMS is an empty dict
tempParam = {}
# All values in StaticConfig.PASTE_PARAMS are immutable
tempParam = dict(StaticConfig.PASTE_PARAMS)
If any values in StaticConfig.PASTE_PARAMS are mutable, you could use copy.deepcopy but it would be better to populate tempParam with those default values on your own.
What if __postBodies wasn't a class attribute, as it is defined now, but just an instance attribute?