We Keep Coding

How to insert values to a Sqlite database from a Python Set or Dict?

I have the following function which produces results;
myNames = ['ULTA', 'CSCO', ...]
def get_from_min_match(var):
temp = []
count_elem = generate_elem_count()
for item in count_elem:
if var <= count_elem[item]:
temp.append(item)
return set(temp) if len(set(temp)) > 0 else "None"
def generate_elem_count():
result_data = []
for val in mapper.values():
if type(val) == list:
result_data += val
elif type(val) == dict:
for key in val:
result_data.append(key)
count_elem = {elem: result_data.count(elem) for elem in result_data}
return count_elem
I call this function like this;
myNames_dict_1 = ['AME', 'IEX', 'PAYC']
myNames_dict_1 = ['ULTA', 'CSCO', 'PAYC']
mapper = {1: myNames_dict_1, 2: myNames_dict_2}
print(" These meet three values ", get_from_min_match(3))
print(" These meet four values ", get_from_min_match(4))
The output I get from these functions are as follows;
These meet three values {'ULTA', 'CSCO', 'SHW', 'MANH', 'TTWO', 'SAM', 'RHI', 'PAYC', 'AME', 'CCOI', 'RMD', 'AMD', 'UNH', 'AZO', 'APH', 'EW', 'FFIV', 'IEX', 'IDXX', 'ANET', 'SWKS', 'HRL', 'ILMN', 'PGR', 'ATVI', 'CNS', 'EA', 'ORLY', 'TSCO'}
These meet four values {'EW', 'PAYC', 'TTWO', 'AME', 'IEX', 'IDXX', 'ANET', 'RMD', 'SWKS', 'HRL', 'UNH', 'CCOI', 'ORLY', 'APH', 'PGR', 'TSCO'}
Now, I want to insert the output, of the get_from_min_match function into a Sqlite database. Its structure looks like this;
dbase.execute("INSERT OR REPLACE INTO min_match (DATE, SYMBOL, NAME, NUMBEROFMETRICSMET) \
VALUES (?,?,?,?)", (datetime.today(), symbol, name, NUMBEROFMETRICSMET?))
dbase.commit()
So, it's basically a new function to calculate the "NUMBEROFMETRICSMET" parameter rather than calling each of these functions many times. And I want the output of the function inserted into the database. How to achieve this? Here 3, 4 would be the number of times the companies matched.
date ULTA name 3
date EW name 4
...
should be the result.
How can I achieve this? Thanks!

I fixed this by just using my already written function;
count_elem = generate_elem_count()
print("Count Elem: " + str(count_elem))
This prints {'AMPY': 1} and so on.

Is there any way to link a field in a Scapy Packet to its portion in the built str?

I'm using Scapy to manipulate CoAP packets. Now I found one thing I want to do which I actually can't figure out how.
I have the following CoAP Packet:
>>> pkt
<CoAP ver=1L type=CON tkl=8L code=PUT msg_id=<RandShort> token=<RandBin> options=[('If-Match', '\x04\xfc\xbbL'), ('Content-Format', ''), ('Uri-Path', '.well-known'), ('Uri-Path', 'core')] paymark='\xff' |<Raw load='CC22 at 12:39:31.495' |>>
>>> str(pkt)
'H\x03\xf4\x08\xbdR_\x85\xa3\x04\xe9\xe9\x14\x04\xfc\xbbL\xab.well-known\x04core\x10\xffCC22 at 12:39:31.495'
What I'd like to do now is to replace the whole part (not the Option Value '') representing the option at index 1 ('Content-Format', '') to null (\x00) chars. I know this part is represented by str(pkt)[-22:-21] near the end of the str(pkt), which has value \x10 to represent the Content-Format option with no actual value on it.
So, I wonder: is there a way to obtain \x10 (actually, to obtain the reference str(pkt)[-22:-21]) from the high-level field (pkt.options[1]), so I can replace this part of the string by another value?
Thanks in advance!

Ok, a few hours on the problem and I came up with a solution which, if I'm correct to assume there's no easy way to do this through Scapy itself (probably due to the way CoAP Options are implemented in the CoAP contrib layer), seems to be a fair enough one.
The following function can be used to "nullify" an entire CoAP Option field:
def coap_option_to_null(pkt, opt_idx):
''' Changes the whole CoAP Option field at #opt_idx to null '''
opt = pkt.options[opt_idx]
l = pkt.options
# Sort the list of options by option number
l.sort( lambda x, y: cmp(coap_options[1][x[0]], coap_options[1][y[0]]) )
# Replace options for the sorted one and grabs new opt_idx
pkt.options = l
opt_idx = pkt.options.index(opt)
cur_delta = 0
opt_total_len = []
for opt in pkt.options:
if (option_model[opt[0]][0] - cur_delta) < 13:
delta_extended = 0
elif (option_model[opt[0]][0] - cur_delta) < 269:
delta_extended = 1
else:
delta_extended = 2
cur_delta += (option_model[opt[0]][0] - cur_delta)
if len(opt[1]) < 13:
len_extended = 0
elif len(opt[1]) < 269:
len_extended = 1
else:
len_extended = 2
opt_total_len.append(1+delta_extended+len_extended+len(opt[1]))
opt_lidx = sum(opt_total_len[:opt_idx])
opt_hidx = sum(opt_total_len[opt_idx+1:])
new_pkt = str(pkt)[:4+pkt.tkl+opt_lidx] + '\x00'*opt_total_len[opt_idx] + str(pkt)[-(len(pkt.payload)+1+sum(opt_total_len[opt_idx+1:])):]
return CoAP(new_pkt)
Below is a demonstration:
>>> pkt
<CoAP ver=1L type=CON tkl=8L code=PUT msg_id=<RandShort> token=<RandBin> options=[('If-Match', '\x04\xfc\xbbL'), ('Content-Format', ''), ('Uri-Path', 'separate')] paymark='\xff' |<Raw load='CC22 at 12:39:31.495' |>>
>>> str(pkt)
'H\x03*\x88%T\xbe\xac\xd9\xee\xcd\xbd\x14\x04\xfc\xbbL\xa8separate\x10\xffCC22 at 12:39:31.495'
>>> npkt = coap_option_to_null(pkt, 1)
>>> npkt
<CoAP ver=1L type=CON tkl=8L code=PUT msg_id=47784 token='\x90L\x0f\xd4u\xbe&\xd7' options=[('If-Match', '\x04\xfc\xbbL'), ('Uri-Path', 'separate'), ('Uri-Path', '')] paymark='\xff' |<Raw load='CC22 at 12:39:31.495' |>>
>>> str(npkt)
'H\x03\xba\xa8\x90L\x0f\xd4u\xbe&\xd7\x14\x04\xfc\xbbL\xa8separate\x00\xffCC22 at 12:39:31.495'
>>> pkt
<CoAP ver=1L type=CON tkl=8L code=PUT msg_id=<RandShort> token=<RandBin> options=[('If-Match', '\x04\xfc\xbbL'), ('Content-Format', ''), ('Uri-Path', 'separate')] paymark='\xff' |<Raw load='CC22 at 12:39:31.495' |>>
>>> npkt = coap_option_to_null(pkt, 0)
>>> npkt
<CoAP ver=1L type=CON tkl=8L code=PUT msg_id=14606 token='s\xad\xd0\xf5\x05\xac\x87C' options=[(0L, ''), (0L, ''), (0L, ''), (0L, ''), (0L, ''), (10L, 'separate'), ('Uri-Path', '')] paymark='\xff' |<Raw load='CC22 at 12:39:31.495' |>>
>>> str(npkt)
'H\x039\x0es\xad\xd0\xf5\x05\xac\x87C\x00\x00\x00\x00\x00\xa8separate\x10\xffCC22 at 12:39:31.495'

Append items to dictionary Python

I am trying to write a function in python that opens a file and parses it into a dictionary. I am trying to make the first item in the list block the key for each item in the dictionary data. Then each item is supposed to be the rest of the list block less the first item. For some reason though, when I run the following function, it parses it incorrectly. I have provided the output below. How would I be able to parse it like I stated above? Any help would be greatly appreciated.
Function:
def parseData() :
filename="testdata.txt"
file=open(filename,"r+")
block=[]
for line in file:
block.append(line)
if line in ('\n', '\r\n'):
album=block.pop(1)
data[block[1]]=album
block=[]
print data
Input:
Bob Dylan
1966 Blonde on Blonde
-Rainy Day Women #12 & 35
-Pledging My Time
-Visions of Johanna
-One of Us Must Know (Sooner or Later)
-I Want You
-Stuck Inside of Mobile with the Memphis Blues Again
-Leopard-Skin Pill-Box Hat
-Just Like a Woman
-Most Likely You Go Your Way (And I'll Go Mine)
-Temporary Like Achilles
-Absolutely Sweet Marie
-4th Time Around
-Obviously 5 Believers
-Sad Eyed Lady of the Lowlands
Output:
{'-Rainy Day Women #12 & 35\n': '1966 Blonde on Blonde\n',
'-Whole Lotta Love\n': '1969 II\n', '-In the Evening\n': '1979 In Through the Outdoor\n'}

You can use groupby to group the data using the empty lines as delimiters, use a defaultdict for repeated keys extending the rest of the values from each val returned from groupby after extracting the key/first element.
from itertools import groupby
from collections import defaultdict
d = defaultdict(list)
with open("file.txt") as f:
for k, val in groupby(f, lambda x: x.strip() != ""):
# if k is True we have a section
if k:
# get key "k" which is the first line
# from each section, val will be the remaining lines
k,*v = val
# add or add to the existing key/value pairing
d[k].extend(map(str.rstrip,v))
from pprint import pprint as pp
pp(d)
Output:
{'Bob Dylan\n': ['1966 Blonde on Blonde',
'-Rainy Day Women #12 & 35',
'-Pledging My Time',
'-Visions of Johanna',
'-One of Us Must Know (Sooner or Later)',
'-I Want You',
'-Stuck Inside of Mobile with the Memphis Blues Again',
'-Leopard-Skin Pill-Box Hat',
'-Just Like a Woman',
"-Most Likely You Go Your Way (And I'll Go Mine)",
'-Temporary Like Achilles',
'-Absolutely Sweet Marie',
'-4th Time Around',
'-Obviously 5 Believers',
'-Sad Eyed Lady of the Lowlands'],
'Led Zeppelin\n': ['1979 In Through the Outdoor',
'-In the Evening',
'-South Bound Saurez',
'-Fool in the Rain',
'-Hot Dog',
'-Carouselambra',
'-All My Love',
"-I'm Gonna Crawl",
'1969 II',
'-Whole Lotta Love',
'-What Is and What Should Never Be',
'-The Lemon Song',
'-Thank You',
'-Heartbreaker',
"-Living Loving Maid (She's Just a Woman)",
'-Ramble On',
'-Moby Dick',
'-Bring It on Home']}
For python2 the unpack syntax is slightly different:
with open("file.txt") as f:
for k, val in groupby(f, lambda x: x.strip() != ""):
if k:
k, v = next(val), val
d[k].extend(map(str.rstrip, v))
If you want to keep the newlines remove the map(str.rstrip..
If you want the album and songs separately for each artist:
from itertools import groupby
from collections import defaultdict
d = defaultdict(lambda: defaultdict(list))
with open("file.txt") as f:
for k, val in groupby(f, lambda x: x.strip() != ""):
if k:
k, alb, songs = next(val),next(val), val
d[k.rstrip()][alb.rstrip()] = list(map(str.rstrip, songs))
from pprint import pprint as pp
pp(d)
{'Bob Dylan': {'1966 Blonde on Blonde': ['-Rainy Day Women #12 & 35',
'-Pledging My Time',
'-Visions of Johanna',
'-One of Us Must Know (Sooner or '
'Later)',
'-I Want You',
'-Stuck Inside of Mobile with the '
'Memphis Blues Again',
'-Leopard-Skin Pill-Box Hat',
'-Just Like a Woman',
'-Most Likely You Go Your Way '
"(And I'll Go Mine)",
'-Temporary Like Achilles',
'-Absolutely Sweet Marie',
'-4th Time Around',
'-Obviously 5 Believers',
'-Sad Eyed Lady of the Lowlands']},
'Led Zeppelin': {'1969 II': ['-Whole Lotta Love',
'-What Is and What Should Never Be',
'-The Lemon Song',
'-Thank You',
'-Heartbreaker',
"-Living Loving Maid (She's Just a Woman)",
'-Ramble On',
'-Moby Dick',
'-Bring It on Home'],
'1979 In Through the Outdoor': ['-In the Evening',
'-South Bound Saurez',
'-Fool in the Rain',
'-Hot Dog',
'-Carouselambra',
'-All My Love',
"-I'm Gonna Crawl"]}}

I guess this is what you want?
Even if this is not the format you wanted, there are a few things you might learn from the answer:
use with for file handling
nice to have:
PEP8 compilant code, see http://pep8online.com/
a shebang
numpydoc
if __name__ == '__main__'
And SE does not like a list being continued by code...
#!/usr/bin/env python
""""Parse text files with songs, grouped by album and artist."""
def add_to_data(data, block):
"""
Parameters
----------
data : dict
block : list
Returns
-------
dict
"""
artist = block[0]
album = block[1]
songs = block[2:]
if artist in data:
data[artist][album] = songs
else:
data[artist] = {album: songs}
return data
def parseData(filename='testdata.txt'):
"""
Parameters
----------
filename : string
Path to a text file.
Returns
-------
dict
"""
data = {}
with open(filename) as f:
block = []
for line in f:
line = line.strip()
if line == '':
data = add_to_data(data, block)
block = []
else:
block.append(line)
data = add_to_data(data, block)
return data
if __name__ == '__main__':
data = parseData()
import pprint
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(data)
which gives:
{ 'Bob Dylan': { '1966 Blonde on Blonde': [ '-Rainy Day Women #12 & 35',
'-Pledging My Time',
'-Visions of Johanna',
'-One of Us Must Know (Sooner or Later)',
'-I Want You',
'-Stuck Inside of Mobile with the Memphis Blues Again',
'-Leopard-Skin Pill-Box Hat',
'-Just Like a Woman',
"-Most Likely You Go Your Way (And I'll Go Mine)",
'-Temporary Like Achilles',
'-Absolutely Sweet Marie',
'-4th Time Around',
'-Obviously 5 Believers',
'-Sad Eyed Lady of the Lowlands']},
'Led Zeppelin': { '1969 II': [ '-Whole Lotta Love',
'-What Is and What Should Never Be',
'-The Lemon Song',
'-Thank You',
'-Heartbreaker',
"-Living Loving Maid (She's Just a Woman)",
'-Ramble On',
'-Moby Dick',
'-Bring It on Home'],
'1979 In Through the Outdoor': [ '-In the Evening',
'-South Bound Saurez',
'-Fool in the Rain',
'-Hot Dog',
'-Carouselambra',
'-All My Love',
"-I'm Gonna Crawl"]}}

Delete index in list if multiple strings are matched

I've scraped a website containing a table and I want to format the headers for my desired final out.
headers = []
for row in table.findAll('tr'):
for item in row.findAll('th'):
for link in item.findAll('a', text=True):
headers.append(link.contents[0])
print headers
Which returns:
[u'Rank ', u'University Name ', u'Entry Standards', u'Click here to read more', u'Student Satisfaction', u'Click here to read more', u'Research Quality', u'Click here to read more', u'Graduate Prospects', u'Click here to read more', u'Overall Score', u'Click here to read more', u'\r\n 2016\r\n ']
I don't want the "Click here to read more' or '2016' headers so I've done the following:
for idx, i in enumerate(headers):
if 'Click' in i:
del headers[idx]
for idx, i in enumerate(headers):
if '2016' in i:
del headers[idx]
Which returns:
[u'Rank ', u'University Name ', u'Entry Standards', u'Student Satisfaction', u'Research Quality', u'Graduate Prospects', u'Overall Score']
Perfect. But is there a better/neater way of removing the unwanted items? Thanks!

headers = filter(lambda h: not 'Click' in h and not '2016' in h, headers)
If you want to be more generic:
banned = ['Click', '2016']
headers = filter(lambda h: not any(b in h for b in banned), headers)

You can consider using list comprehension to get a new, filtered list, something like:
new_headers = [header for header in headers if '2016' not in header]

If you can be sure that '2016' will always be last:
>>> [x for x in headers[:-1] if 'Click here' not in x]
['Rank ', 'University Name ', 'Entry Standards', 'Student Satisfaction', 'Research Quality', 'Graduate Prospects', 'Overall Score']

pattern = '^Click|^2016'
new = [x for x in header if not re.match(pattern,str(x).strip())]

Failing to append to dictionary. Python

I am experiencing a strange faulty behaviour, where a dictionary is only appended once and I can not add more key value pairs to it.
My code reads in a multi-line string and extracts substrings via split(), to be added to a dictionary. I make use of conditional statements. Strangely only the key:value pairs under the first conditional statement are added.
Therefore I can not complete the dictionary.
How can I solve this issue?
Minimal code:
#I hope the '\n' is sufficient or use '\r\n'
example = "Name: Bugs Bunny\nDOB: 01/04/1900\nAddress: 111 Jokes Drive, Hollywood Hills, CA 11111, United States"
def format(data):
dic = {}
for line in data.splitlines():
#print('Line:', line)
if ':' in line:
info = line.split(': ', 1)[1].rstrip() #does not work with files
#print('Info: ', info)
if ' Name:' in info: #middle name problems! /maiden name
dic['F_NAME'] = info.split(' ', 1)[0].rstrip()
dic['L_NAME'] = info.split(' ', 1)[1].rstrip()
elif 'DOB' in info: #overhang
dic['DD'] = info.split('/', 2)[0].rstrip()
dic['MM'] = info.split('/', 2)[1].rstrip()
dic['YY'] = info.split('/', 2)[2].rstrip()
elif 'Address' in info:
dic['STREET'] = info.split(', ', 2)[0].rstrip()
dic['CITY'] = info.split(', ', 2)[1].rstrip()
dic['ZIP'] = info.split(', ', 2)[2].rstrip()
return dic
if __name__ == '__main__':
x = format(example)
for v, k in x.iteritems():
print v, k

Your code doesn't work, at all. You split off the name before the colon and discard it, looking only at the value after the colon, stored in info. That value never contains the names you are looking for; Name, DOB and Address all are part of the line before the :.
Python lets you assign to multiple names at once; make use of this when splitting:
def format(data):
dic = {}
for line in data.splitlines():
if ':' not in line:
continue
name, _, value = line.partition(':')
name = name.strip()
if name == 'Name':
dic['F_NAME'], dic['L_NAME'] = value.split(None, 1) # strips whitespace for us
elif name == 'DOB':
dic['DD'], dic['MM'], dic['YY'] = (v.strip() for v in value.split('/', 2))
elif name == 'Address':
dic['STREET'], dic['CITY'], dic['ZIP'] = (v.strip() for v in value.split(', ', 2))
return dic
I used str.partition() here rather than limit str.split() to just one split; it is slightly faster that way.
For your sample input this produces:
>>> format(example)
{'CITY': 'Hollywood Hills', 'ZIP': 'CA 11111, United States', 'L_NAME': 'Bunny', 'F_NAME': 'Bugs', 'YY': '1900', 'MM': '04', 'STREET': '111 Jokes Drive', 'DD': '01'}
>>> from pprint import pprint
>>> pprint(format(example))
{'CITY': 'Hollywood Hills',
'DD': '01',
'F_NAME': 'Bugs',
'L_NAME': 'Bunny',
'MM': '04',
'STREET': '111 Jokes Drive',
'YY': '1900',
'ZIP': 'CA 11111, United States'}