We Keep Coding

Parsing a dictionary in Python to my current table

I have a table that contains a few categories and two of them are: mac address and device name. I had a the list of my mac address written in my code (hardcoded) with their corresponding device names (ie deviceDict['00:00:00:00:00:00']= name)
Now, I passed those mac addresses and device names to a text file to be read from that same Python code and parse it onto my table. The code currently recognizes the text file but it is not parsing that information onto the table.
Here is the code:
# File: WapLogParser.py
# Desc: Parses a WAP log file and pulls out information relating to connected clients
# Usage: python WapLogParser.py [file glob]
import re
import sys
import glob
import os
deviceDict = dict()
# Base table for storing client info
# All names must match what is in the Wap Log file
# Exceptions: Date, Wap Name, Device Name - which are provided outside of the result parsing
table = [["Ssid", "Vlan", "Mac Address", "Connected Time", "Ip Address", "Rssi", "Date", "Wap Name", "Device Name"]]
def ParseResult(result, date, wapName):
lines = result.split('\n')
lines = list(filter(None, lines))
# Any useful info will be at least 2 lines long
if len(lines) == 1:
return
# create empty row
data = [""] * len(table[0])
# for each item in the result place it in the correct spot in the row
for line in lines:
if line != "":
# Parse the key/value pair
m = re.match(r"(.*):\s\.*\s?(.*)", line)
if m is not None:
for idx in range(len(table[0])):
if table[0][idx].lower() == m[1].lower():
data[idx] = m[2]
else:
break
# Remove the '(dBm)' from the RSSI value
data[5] = data[5].split()[0]
# Append WAP specific items to row
data[6] = date
data[7] = wapName
data[8] = GetDeviceName(data[2].upper())
# Add row to table
table.append(data)
def ParseFile(path):
with open(path) as f:
lines = f.readlines()
result = ""
command = ""
date = ""
# WAP name is always on the first line 16 characters in with 4
# unnecessary characters trailing
wapName = lines[0].strip()[16:-4]
for line in lines:
line = line.strip()
# Is an issued command?
if line.startswith("/#"):
if command != "":
ParseResult(result, date, wapName)
command = ""
# reset the result for the new command
result = ""
m = re.match(r"^/#.*show\sclient.*stats$", line)
if m is not None:
command = line
# Anything that is not a command add to the result
else:
result += line + "\n"
# Do we have the date?
if line.startswith("Current date:"):
date = line.replace("Current date: ", "")
# Print output to stderr
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
# Print a 2d array in a csv format
def PrintAsCsv(table):
for row in table:
print(",".join(row))
def Main():
InitDeviceDict()
numArgs = len(sys.argv)
for filename in glob.iglob(sys.argv[numArgs - 1], recursive=True):
# Globs get directories too
if os.path.isfile(filename):
eprint("Parsing " + filename)
try:
ParseFile(filename)
except Exception as e: # Mainly for if we see a binary file
eprint("Bad file: " + e)
# Print in a format we can use
PrintAsCsv(table)
def GetDeviceName(macAddress):
if macAddress in deviceDict:
return deviceDict[macAddress]
manufacturerPart = macAddress[:8]
if manufacturerPart in deviceDict:
return deviceDict[manufacturerPart]
return 'Unknown Device'
def InitDeviceDict():
with open('try.txt','r') as fo:
for line in fo:
deviceDict = {}
line = line.split(',')
macAddress = line[0].strip()
manufacturerPart = line[1].strip()
if macAddress in deviceDict:
deviceDict[macAddress].append(manufacturerPart)
else:
deviceDict[macAddress]=(manufacturerPart)
print(deviceDict)
# entry point
# script arguments:
# WapLogParser.py [file glob]
if __name__ == "__main__":
Main()
The issue is on the functions GetDeviceName and InitDeviceDict. When I run the code and then a batch file to display my info on excel, I keep getting "unknown device" (as if it is not recognizing the mac address I entered to produce the device name)
Any way I can correct this? Thank you

The deviceDict that is populated in InitDeviceDict is not the global deviceDict. You are only modifying a function-local dictionary (and resetting it every line as well). Remove deviceDict = {} from that function and, at the top of the function use global deviceDict to declare that you are modifying the global.
def InitDeviceDict():
global deviceDict
with open('try.txt','r') as fo:
for line in fo:
line = line.split(',')
macAddress = line[0].strip()
manufacturerPart = line[1].strip()
if macAddress in deviceDict:
deviceDict[macAddress].append(manufacturerPart)
else:
deviceDict[macAddress]=[manufacturerPart]

sys.argv list index out of range error [duplicate]

This question already has answers here:
What does "sys.argv[1]" mean? (What is sys.argv, and where does it come from?)
(9 answers)
Closed 7 months ago.
I am trying to run this script that grabs rss feeds on the environment "Thonny" but I just keep receiving this error of "IndexError: List index out of range"
Traceback (most recent call last):
File "C:\Users\uri\rssfeedfour.py", line 11, in <module>
url = sys.argv[1]
IndexError: list index out of range
How do I resolve this to keep from getting this error over and over again. Im not sure how to solve this as I am a beginner. Do I need to define it, if so how? or could I take it out and go a different direction? Here is the code.
import feedparser
import time
from subprocess import check_output
import sys
#feed_name = 'TRIBUNE'
#url = 'http://chicagotribune.feedsportal.com/c/34253/f/622872/index.rss'
feed_name = sys.argv[1]
url = sys.argv[2]
db = 'http://feeds.feedburner.com/TheHackersNews'
limit = 12 * 3600 * 1000
current_time_millis = lambda: int(round(time.time() * 1000))
current_timestamp = current_time_millis()
def post_is_in_db(title):
with open(db, 'r') as database:
for line in database:
if title in line:
return True
return False
def post_is_in_db_with_old_timestamp(title):
with open(db, 'r') as database:
for line in database:
if title in line:
ts_as_string = line.split('|', 1)[1]
ts = long(ts_as_string)
if current_timestamp - ts > limit:
return True
return False
#
# get the feed data from the url
#
feed = feedparser.parse(url)
#
# figure out which posts to print
#
posts_to_print = []
posts_to_skip = []
for post in feed.entries:
# if post is already in the database, skip it
# TODO check the time
title = post.title
if post_is_in_db_with_old_timestamp(title):
posts_to_skip.append(title)
else:
posts_to_print.append(title)
#
# add all the posts we're going to print to the database with the current timestamp
# (but only if they're not already in there)
#
f = open(db, 'a')
for title in posts_to_print:
if not post_is_in_db(title):
f.write(title + "|" + str(current_timestamp) + "\n")
f.close
#
# output all of the new posts
#
count = 1
blockcount = 1
for title in posts_to_print:
if count % 5 == 1:
print("\n" + time.strftime("%a, %b %d %I:%M %p") + ' ((( ' + feed_name + ' - ' + str(blockcount) + ' )))')
print("-----------------------------------------\n")
blockcount += 1
print(title + "\n")
count += 1

sys.argv is a list in Python, which contains the command-line arguments passed to the script. sys.argv[0] contains the name of the script, sys.argv[1] contains the first argument and so on.
To prevent this error, you need to give command line arguments when starting the script. For example, you can start this script without any errors by
python rssfeedfour.py TRIBUNE http://chicagotribune.feedsportal.com/c/34253/f/622872/index.rss
You can also modify the script so that it works using the default arguments if you don't provide any command line arguments.
try:
feed_name = sys.argv[1]
except IndexError:
feed_name = 'TRIBUNE'
try:
url = sys.argv[2]
except IndexError:
url = 'http://chicagotribune.feedsportal.com/c/34253/f/622872/index.rss'
You can learn more about handling errors here.
Although it is much more convenient to use argparse library.

Reading .sql File in for Execution in Python (pymysql)

I'm attempting to create a multiscript tool, that will take an argument of a .sql file and execute it.
I've set up a simple test, just executing on one database, however the syntax is giving me issues every time.
DELIMITER $$
CREATE DEFINER=`a_user`#`%` PROCEDURE `a_procedure`(
IN DirectEmployeeID TEXT,
IN StartRange DATE,
IN EndRange DATE
)
BEGIN
SELECT aColumn
WHERE thisThing = 1;
END$$
DELIMITER ;
To be clear, this script has been tested, and works when passed like :
mysql -uuser -p -hhost -Pport databaseName < file.sql
and also works through mysql workbench.
I saw this type of solution on another site:
with conn.cursor() as cursor:
f = sys.argv[1]
file = open(f, 'r')
sql = " ".join(file.readlines())
cursor.execute(sql)
which gives me a MySQL syntax error:
pymysql.err.ProgrammingError: (1064, u"You have an error in your SQL syntax;
check the manual that corresponds to your MySQL server version for the right
syntax to use near 'DELIMITER $$\n CREATE DEFINER=`a_user`#`%` PROCEDURE
`MyCommissionsDirect`(\n \tIN ' at line 1")
as you can see, there are newline characters within the script that mysql isn't liking.
I then tried this:
with conn.cursor() as cursor:
f = sys.argv[1]
file = open(f, 'r')
sql = ''
line = file.readline()
while line:
sql += ' ' + line.strip('\n').strip('\t')
line = file.readline()
print sql
cursor.execute(sql)
and get another syntax issue, the print shows that this is all one line, which is not working in mysqlworkbench. doesn't even try to execute it, which is strange.
When I put the DELIMETER $$ on a separate line first, it executes in mysqlworkbench.
This is one of those situations where I feel like I may be making this more and more complicated. I'm very surprised pymysql doesn't have a way of simply executing a sql file directly. I'm weary of trying to do string manipulation and get this working for this particular file, because then the dream of making this tool ambiguous and reusable kind of goes out the door.
Am I going about this in the complete incorrect way?
Thanks!

Here is my solution for using an SQL file with PyMySQL. The files contain many requests ended by ; which is used to split requests in a list. So beware of the missing ; in the list.
I decided to add the missing ; not in the function to spar a for loop. Maybe there is a better way.
create-db-loff.sql :
DROP DATABASE IF EXISTS loff;
CREATE DATABASE loff CHARACTER SET 'utf8';
USE loff;
CREATE TABLE product(
`id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
`code` BIGINT UNSIGNED NOT NULL UNIQUE,
`name` VARCHAR(200),
`nutrition_grades` VARCHAR(1)
);
CREATE TABLE category(
`id`INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
`name` VARCHAR(200)
);
CREATE TABLE asso_prod_cat(
`category_id` INT UNSIGNED NOT NULL,
`product_id` INT UNSIGNED NOT NULL,
CONSTRAINT `fk_asso_prod_cat_category`
FOREIGN KEY(category_id)
REFERENCES category(id)
ON DELETE CASCADE,
CONSTRAINT `fk_asso_prod_cat_product`
FOREIGN KEY(product_id)
REFERENCES product(id)
ON DELETE CASCADE
);
db.py :
DB_CONFIG = {
'host': 'localhost',
'user': 'loff',
'pass': 'loff',
'db': 'loff',
'char': 'utf8',
'file': 'create-db-loff.sql'
}
def get_sql_from_file(filename=DB_CONFIG['file']):
"""
Get the SQL instruction from a file
:return: a list of each SQL query whithout the trailing ";"
"""
from os import path
# File did not exists
if path.isfile(filename) is False:
print("File load error : {}".format(filename))
return False
else:
with open(filename, "r") as sql_file:
# Split file in list
ret = sql_file.read().split(';')
# drop last empty entry
ret.pop()
return ret
request_list = self.get_sql_from_file()
if request_list is not False:
for idx, sql_request in enumerate(request_list):
self.message = self.MSG['request'].format(idx, sql_request)
cursor.execute(sql_request + ';')

DELIMITER is command used by a MySQL interpreter, such as the command line or Workbench, and not an actual MySQL command.
I ended up working in some logic in my Python application to disable execution of MySQL queries when DELIMITER has been defined, then to execute when DELIMITER has been defined again:
import MySQLdb
import re
file = 'somesql.sql'
conn = MySQLdb.Connection(mysqlserver, mysqluser, mysqlpass, mysqldb)
curs = conn.cursor()
ignorestatement = False # by default each time we get a ';' that's our cue to execute.
statement = ""
for line in open(file):
if line.startswith('DELIMITER'):
if not ignorestatement:
ignorestatement = True # disable executing when we get a ';'
continue
else:
ignorestatement = False # re-enable execution of sql queries on ';'
line = " ;" # Rewrite the DELIMITER command to allow the block of sql to execute
if re.match(r'--', line): # ignore sql comment lines
continue
if not re.search(r'[^-;]+;', line) or ignorestatement: # keep appending lines that don't end in ';' or DELIMITER has been called
statement = statement + line
else: # when you get a line ending in ';' then exec statement and reset for next statement providing the DELIMITER hasn't been set
statement = statement + line
# print "\n\n[DEBUG] Executing SQL statement:\n%s" % (statement)
try:
curs.execute(statement)
conn.commit()
statement = ""
except curs.Error, e:
print(file + " - Error applying (" + str(e) + ")\nTerminating.")
sys.exit(1)
It's a bit hacky, but seems to work well enough.

Most SQL files contain interpreter commands such as DELIMITER that make passing the commands through to pymysql somewhat difficult, this code snippet allows you to separate out the statements in the sql file into a list for sequential execution.
def parse_sql(filename):
data = open(filename, 'r').readlines()
stmts = []
DELIMITER = ';'
stmt = ''
for lineno, line in enumerate(data):
if not line.strip():
continue
if line.startswith('--'):
continue
if 'DELIMITER' in line:
DELIMITER = line.split()[1]
continue
if (DELIMITER not in line):
stmt += line.replace(DELIMITER, ';')
continue
if stmt:
stmt += line
stmts.append(stmt.strip())
stmt = ''
else:
stmts.append(line.strip())
return stmts
Usage example:
conn = pymysql.connect('test')
stmts = parse_sql('my_sql_file.sql')
with conn.cursor() as cursor:
for stmt in stmts:
cursor.execute(stmt)
conn.commit()

It's simple code
import pymysql
class ScriptRunner:
def __init__(self, connection, delimiter=";", autocommit=True):
self.connection = connection
self.delimiter = delimiter
self.autocommit = autocommit
def run_script(self, sql):
try:
script = ""
for line in sql.splitlines():
strip_line = line.strip()
if "DELIMITER $$" in strip_line:
self.delimiter = "$$"
continue
if "DELIMITER ;" in strip_line:
self.delimiter = ";"
continue
if strip_line and not strip_line.startswith("//") and not strip_line.startswith("--"):
script += line + "\n"
if strip_line.endswith(self.delimiter):
if self.delimiter == "$$":
script = script[:-1].rstrip("$") + ";"
cursor = self.connection.cursor()
print(script)
cursor.execute(script)
script = ""
if script.strip():
raise Exception("Line missing end-of-line terminator (" + self.delimiter + ") => " + script)
if not self.connection.get_autocommit():
self.connection.commit()
except Exception:
if not self.connection.get_autocommit():
self.connection.rollback()
raise
if __name__ == '__main__':
connection = pymysql.connect(host="127.0.0.1", user="root", password="root", db="test", autocommit=True)
sql = ""
ScriptRunner(connection).run_script(sql)

inserting data into mysql using python with external variables

I have a table where I need to populate email addresses from txt file, date and source from python variables. Down the code, a is the variable am getting from txt and b, c are the python variables I need to insert along
import csv,MySQLdb,os,os.path,datetime as dt
from sys import argv
path = argv
prompt = '>'
filename = raw_input(prompt)
fname = os.path.basename(filename)
source = fname
print "",source
date = dt.datetime.today().strftime("%d/%m/%y")
date_log = date
print "",date_log
db = MySQLdb.connect("localhost","root","passwd","testdb")
x = db.cursor()
csv_data = csv.reader(file(filename))
for row in csv_data:
print row
query = (" INSERT INTO test(a,b,c) VALUES (%s,%s,%s)",row)
x.execute(query, (date_log, source))
db.commit()
db.close()
Here am getting some error as follows:
File "csv2mysql.py", line 31, in <module>
x.execute(query, (date_log, source))
File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py",line159,in execute
query = query % db.literal(args)
TypeError: unsupported operand type(s) for %: 'tuple' and 'tuple'
any help is much appreciated

Replace query string
query = (" INSERT INTO test(a,b,c) VALUES %s") %(row,)
or
query = (" INSERT INTO test(a,b,c) VALUES %s") % str(row)
Hope it will work for you...

Can python sqlalchemy use a variable to construct an insert statement (for mysql database)

Thanks in advance for advice on this problem...
I am trying to create a python script to import a set of CSVs into a mysql database.
Each CSV filename matches the destination table. The first row of each CSV matches the fields of the table. Each CSV / table has a different number of fields, field names, etc.
The problem I am having is with this line (full code below)
ins = table_name.insert().values(temp_variable_name)
where I want to dynamically update the destination table (table_name) and the insert command (temp_variable_name).
So when reading the labels.csv file, this should produce
ins = labels.insert().values(id_label=d[0], label_name=d[1])
and when reading the company.csv file, this should produce
ins = company.insert().values(id_company=d[0], company_name=d[1], ticker=d[2])
The problem is if I generate a string,
temp_variable_name = 'id_company=d[0], company_name=d[1], ticker=d[2]'
I end up getting a 'str' object has no attribute 'items' error.
Is there any way to dynamically generate an insert command for an SQL statement?
Portion of the script below:
# files list contains a list of all of the files in the directory
# we read in CSVs, isolate the first row to determine table field names
# the rest of the data should then be imported into the table with the corresponding name as the CSV
for f in files:
if '.csv' in f :
# read in each CSV file
# these are a Class / Function I've set up to read files
x = Read_Files()
data = x.read_file_lines_strip(path, f)
temp = data[0].replace('"','') # get rid of quotation marks from the data
table_header_list = temp.split('|') # get the first row, which is the table field names
variable_name ='' # this is used to construct the insert into table string
for x in xrange (0, len(table_header_list)) :
if x == 0 :
variable_name = variable_name + table_header_list[0] + '=d[0]'
elif x == len(table_header_list) :
variable_name = variable_name + table_header_list[x] + '=d[' + str(x) + ']'
else :
variable_name = variable_name + ', ' + table_header_list[x] + '=d[' + str(x) + ']'
table_name = f.replace('.csv','') # remove the .csv from filename to isolate the file name, which is the same as table name
# data from file
for data_line in data[1:] :
data_line = data_line.replace('"', '') # remove quotation marks
d = data_line.split('|') # split the line which is delimited by a |
# used to construct the final insert string
for x in xrange(0, len(table_header_list)) :
if x == 0 :
temp_variable_name = variable_name.replace('d[0]', d[0])
else :
temp_variable_name = temp_variable_name.replace('d[' + str(x) + ']', d[x])
try:
# table name is the table to insert into, via the CSV filename
# temp_variable_name is the insert string, such as 'id_company=d[0], company_name=d[1], ticker=d[2]'
ins = table_name.insert().values(temp_variable_name)
result = conn.execute(ins)
except Exception, e :
print 'error : ' + str(e)

You can do this with Insert objects and the csv module which makes it easy with the DictReader class. Here's an example for the company table:
import csv
from sqlalchemy import create_engine
from sqlalchemy.sql import table, column
NULL_FIELD_VALUE = r'\N'
DB_CONNECT = 'sqlite:///company.db'
engine = create_engine(DB_CONNECT, echo=True)
conn = engine.connect()
with open('company.csv') as csvfile:
reader = csv.DictReader(csvfile, delimiter='|')
insert_table = table('company',
*[column(field) for field in reader.fieldnames])
insert_dict = [{k: None if v == NULL_FIELD_VALUE else v
for k,v in row.items()}
for row in reader]
conn.execute(insert_table.insert(), insert_dict)