We Keep Coding

Connecting with Athena using Python and pyathenajdbc

I am trying to connect to AWS Athena using python. I am trying to use pyathenajdbc to achieve this task. The issue I am having is obtaining a connection. When I run the code below, I receive an error message stating it cannot find the AthenaDriver. ( java.lang.RuntimeException: Class com.amazonaws.athena.jdbc.AthenaDriver not found). I did download this file from AWS and I have confirmed it is sitting in that directory.
from mdpbi.rsi.config import *
from mdpbi.tools.functions import mdpLog
from pkg_resources import resource_string
import argparse
import os
import pyathenajdbc
import sys
SCRIPT_NAME = "Athena_Export"
ATHENA_JDBC_CLASSPATH = "/opt/amazon/athenajdbc/AthenaJDBC41-1.0.0.jar"
EXPORT_OUTFILE = "RSI_Export.txt"
EXPORT_OUTFILE_PATH = os.path.join(WORKINGDIR, EXPORT_OUTFILE)
def get_arg_parser():
"""This function returns the argument parser object to be used with this script"""
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
return parser
def main():
args = get_arg_parser().parse_args(sys.argv[1:])
logger = mdpLog(SCRIPT_NAME, LOGDIR)
SQL = resource_string("mdpbi.rsi.athena.resources", "athena.sql")
conn = pyathenajdbc.connect(
s3_staging_dir="s3://athena",
access_key=AWS_ACCESS_KEY_ID,
secret_key=AWS_SECRET_ACCESS_KEY,
region_name="us-east-1",
log_path=LOGDIR,
driver_path=ATHENA_JDBC_CLASSPATH
)
try:
with conn.cursor() as cursor:
cursor.execute(SQL)
logger.info(cursor.description)
logger.info(cursor.fetchall())
finally:
conn.close()
return 0
if __name__ == '__main__':
rtn = main()
sys.exit(rtn)
Traceback (most recent call last): File
"/usr/lib64/python2.7/runpy.py", line 174, in _run_module_as_main
"main", fname, loader, pkg_name) File "/usr/lib64/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals File "/home/ec2-user/jason_testing/mdpbi/rsi/athena/main.py", line 53,
in
rtn = main() File "/home/ec2-user/jason_testing/mdpbi/rsi/athena/main.py", line 39,
in main
driver_path=athena_jdbc_driver_path File "/opt/mdpbi/Python_Envs/2.7.10/local/lib/python2.7/dist-packages/pyathenajdbc/init.py",
line 65, in connect
driver_path, **kwargs) File "/opt/mdpbi/Python_Envs/2.7.10/local/lib/python2.7/dist-packages/pyathenajdbc/connection.py",
line 68, in init
jpype.JClass(ATHENA_DRIVER_CLASS_NAME) File "/opt/mdpbi/Python_Envs/2.7.10/lib64/python2.7/dist-packages/jpype/_jclass.py",
line 55, in JClass
raise _RUNTIMEEXCEPTION.PYEXC("Class %s not found" % name)

The JDBC driver requires Java 8. I was currently running Java 7. I was able to install another version of Java on EC2 instance.
https://tecadmin.net/install-java-8-on-centos-rhel-and-fedora/#
I had to also set the java version in my code. With these changes, the code now runs as expected.
from mdpbi.rsi.config import *
from mdpbi.tools.functions import mdpLog
from pkg_resources import resource_string
import argparse
import os
import pyathenajdbc
import sys
SCRIPT_NAME = "Athena_Export"
def get_arg_parser():
"""This function returns the argument parser object to be used with this script"""
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
return parser
def main():
args = get_arg_parser().parse_args(sys.argv[1:])
logger = mdpLog(SCRIPT_NAME, LOGDIR)
SQL = resource_string("mdpbi.rsi.athena.resources", "athena.sql")
os.environ["JAVA_HOME"] = "/opt/jdk1.8.0_121"
os.environ["JRE_HOME"] = "/opt/jdk1.8.0_121/jre"
os.environ["PATH"] = "/opt/jdk1.8.0_121/bin:/opt/jdk1.8.0_121/jre/bin"
conn = pyathenajdbc.connect(
s3_staging_dir="s3://mdpbi.data.rsi.out/",
access_key=AWS_ACCESS_KEY_ID,
secret_key=AWS_SECRET_ACCESS_KEY,
schema_name="rsi",
region_name="us-east-1"
)
try:
with conn.cursor() as cursor:
cursor.execute(SQL)
logger.info(cursor.description)
logger.info(cursor.fetchall())
finally:
conn.close()
return 0
if __name__ == '__main__':
rtn = main()
sys.exit(rtn)

Try this :
pyathenajdbc.ATHENA_JAR = ATHENA_JDBC_CLASSPATH
You won't be needing to specify the driver_path argument in the connection method

File Too Large to import?

I just wrote this code to fetch the wireshark mac oui database, and I get the following error:
Traceback (most recent call last):
File "init.py", line 38, in <module>
main()
File "init.py", line 27, in main
import maclist
File "/home/synthetica/WiJam/maclist.py", line 23753
'FC:F6:4
however, this is NOT the contents of the file at that line. Is this a limit of the python intepreter, something I'm overlooking, or something else?
init.py:
def main():
#init
#Load config.
import localconfig
print localconfig.name
#update mac adress db, if at all possible:
try:
import maclist
except:
import urllib2
print "Fetching MAC adress db."
try:
maclist = urllib2.urlopen(localconfig.url)
else:
fl = open("maclist.py","w")
fl.write("#maclist.py generated by "+localconfig.name+"\n")
print "Generating maclist.py"
for line in maclist:
if "#" in line: line=line[:line.index("#")]
line = line.split()
if line and "-" not in line[0]:
line=[repr(part) for part in line]
line = "=".join(line)
fl.write("=".join(line.split())+"\n")
import maclist
#start browser
#start web interface
#handle web interface commands
#display web interface
if __name__=="__main__":
main()
localconfig.py
version = "0.3"
name = "Synth's WiJam (version "+version+")"
#maclist related:
url = "https://code.wireshark.org/review/gitweb?p=wireshark.git;a=blob_plain;f=manuf;hb=HEAD"
Any leads?
#bren
maclist.py: Not the full thing, heavens no. It's 20k+ lines.
'FC:E1:92'='SichuanJ'
'FC:E1:D9'='StableIm'
'FC:E2:3F'='ClayPaky'
'FC:E5:57'='Nokia'
'FC:E8:92'='Hangzhou'
'FC:ED:B9'='Arrayent'
'FC:F1:CD'='Optex-Fa'
'FC:F5:28'='ZyxelCom'
'FC:F6:47'='Fiberhom'
'FC:F8:AE'='IntelCor'
'FC:F8:B7'='TronteqE'
'FC:FA:F7'='Shanghai'
'FC:FB:FB'='Cisco'

Rewrite maclist.py to be proper python syntax, for example:
hosts={}
hosts['FC:FA:F7']='Shanghai'
hosts['FC:FB:FB']='Cisco'
and so on.

crontab with sudo python script

Alright, I've found something. Not sure how to tackle it. I've seen that this is a common error that comes up in google. The error seems to have something to do with the environment variables or something. Not sure how to handle this:
This is the code and it's the part where subprocess is called that leads to the error:
#!/usr/bin/python
import subprocess
import re
import sys
import time
import datetime
import gspread
# ===========================================================================
# Google Account Details
# ===========================================================================
# Account details for google docs
email = 'my_email#gmail.com'
password = 'my_password'
spreadsheet = 'my_spreadsheet'
# ===========================================================================
# Example Code
# ===========================================================================
# Login with your Google account
try:
gc = gspread.login(email, password)
except:
print "Unable to log in. Check your email address/password"
sys.exit()
# Open a worksheet from your spreadsheet using the filename
try:
worksheet = gc.open(spreadsheet).sheet1
# Alternatively, open a spreadsheet using the spreadsheet's key
# worksheet = gc.open_by_key('0BmgG6nO_6dprdS1MN3d3MkdPa142WFRrdnRRUWl1UFE')
except:
print "Unable to open the spreadsheet. Check your filename: %s" % spreadsheet
sys.exit()
# Continuously append data
while(True):
# Run the DHT program to get the humidity and temperature readings!
output = subprocess.check_output(["./Adafruit_DHT", "2302", "17"]);
print output
matches = re.search("Temp =\s+([0-9.]+)", output)
if (not matches):
time.sleep(3)
continue
temp1 = float(matches.group(1))
temp = temp1*9/5+32 # added the extra step to converto to fahrenheit
# search for humidity printout
matches = re.search("Hum =\s+([0-9.]+)", output)
if (not matches):
time.sleep(3)
continue
humidity = float(matches.group(1))
print "Temperature: %.1f F" % temp
print "Humidity: %.1f %%" % humidity
# Append the data in the spreadsheet, including a timestamp
try:
values = [datetime.datetime.now(), temp, humidity]
worksheet.append_row(values)
except:
print "Unable to append data. Check your connection?"
sys.exit()
# Wait 30 seconds before continuing or just exit
print "Wrote a row to %s" % spreadsheet
# time.sleep(60)
sys.exit()
that's basically it. It works fine using 'sudo python script.py' as long as the Adafruit_DHT program is in the same directory.
Here's the error I get:
Traceback (most recent call last):
File "/home/pi/Adafruit/dht/Ada_temp.py", line 44, in <module>
output = subprocess.check_output(["./home/pi/Adafruit/dht/Adafruit_DHT", "2302", "17"]);
File "/usr/lib/python2.7/subprocess.py", line 537, in check_output
process = Popen(stdout=PIPE, *popenargs, **kwargs)
File "/usr/lib/python2.7/subprocess.py", line 679, in __init__
errread, errwrite)
File "/usr/lib/python2.7/subprocess.py", line 1259, in _execute_child
raise child_exception
OSError: [Errno 2] No such file or directory
I've tried adding the full path of the c program (Adafruit_DHT), to no avail...

Locate the problem.
Does the script run at all?
Do something trivial in the first line of the script to see that it actually gets executed from cron (e.g.: write to a file in /tmp).
Once you managed to run it, look for other problems. Cron can be set up to send the a mail with the output of the script. One obvious thing I see is: ./Adafruit_DHT, relative path is used, that's a bad sign, the cron script is probably not executed in the directory you think it does. Fix it (= use absolute path).

Try:
output = subprocess.check_output(["PATH_TO_Adafruit_DHT/Adafruit_DHT", "2302", "17"]);
Oh and change your cron line to:
/usr/bin/python /home/pi/myscript.py

Python import error

I am trying to run a python file from the command line with a single parameter in Ubuntu 12.04. The program works if I simply run it from the IDE and pass the parameter in the code. However, if I call 'python readFromSerial1.py 3' in the command prompt, I get:
Traceback (most recent call last):
File "readFromSerial1.py", line 62, in <module>
main()
File "readFromSerial1.py", line 6, in main
readDataFromUSB(time)
File "readFromSerial1.py", line 9, in readDataFromUSB
import usb.core
ImportError: No module named usb.core
I'm a little confused as the module imports correctly if I run from the IDE. I download the pyUSB module and extracted it (its filename is pyusb-1.0.0a3). I then copied this file into
/usr/local/lib/python2.7/site-packages/. Is that the correct procedure? I have a feeling the issue is due to python simply not being able to find the usb module and I need to put it in the correct location. My code is below, and any help would be greatly appreciated:
readFromSerial1.py
import sys
def main():
time = sys.argv[1]
#time = 1
readDataFromUSB(time)
def readDataFromUSB(time):
import usb.core
#find device
dev = usb.core.find(idVendor=0x099e, idProduct=0x0001) #GPS info
#Was the device found?
if dev is None:
raise ValueError('Device not found')
else:
print "Device found!"
#Do this to avoid 'Errno16: Resource is busy'
if dev.is_kernel_driver_active(0):
try:
dev.detach_kernel_driver(0)
except usb.core.USBError as e:
sys.exit("Could not detach kernel driver: %s" % str(e))
#Sets default config
dev.set_configuration()
#Gets default endpoint
endpoint = dev[0][(0,0)][0]
writeObject = open("InputData.txt", "w")
#iterate for time purposes
for i in range(0, (time*6)): #sys.argv is command line variable for time input
data = dev.read(endpoint.bEndpointAddress, endpoint.wMaxPacketSize, 0, 100000)
sret = ''.join([chr(x) for x in data])
writeObject.write(sret);
print sret
'''
newStr = ''.join(sret[7:14])
compareStr = ",*4F"
if (newStr == compareStr):
print "The GPS is not reading in any values right now. Try going somewhere else with better reception."
else:
print sret[7:14]
'''
writeObject.close()
main()

NameError while handling command line arguments

I was trying out a few python test scripts with sqlite3.
Here is the script that I wrote
#!/usr/bin/env python
from sqlite3 import dbapi2 as sqlite
from sys import argv,exit
db_name = "filenames.db"
def define_db():
try:
conn = sqlite.connect(db_name)
except IOError as e:
print "problem while creating/connecting the db:",e.args[0]
exit(1)
return conn
def write_db(conn,cursor,fni):
conn.execute("CREATE TABLE IF NOT EXISTS file (filenames TEXT UNIQUE)")
query = "INSERT OR REPLACE INTO file VALUES($filenames)"
cursor.execute(query,[fni])
cursor.close()
conn.commit()
conn.close()
print fni,"should now be in the db"
exit(0)
if __name__ == "__main__":
if len(argv) == 2:
etag = argv[1]
else:
print "no argument given - stopping now"
exit(1)
conn = define_db()
cursor = conn.cursor()
write_db(conn,cursor,fni)
I keep getting this error and was not able to solve it.
Traceback (most recent call last):
File "blah.py", line 37, in <module>
write_db(conn,cursor,fni)
NameError: name 'fni' is not defined
Any idea what the problem is.
At this moment I use python 2.7.3

The last line of your script refers to a name fni that is not defined.

You have not defined the variable "fni", but you are using it.

Static analysis tools like pyflakes or pylint can be useful to catch silly errors like this
If you wrote the bulk of the code in a function (so it doesn't assume blub is a global variable, which don't make pyflakes/pylint complain):
def main():
if len(argv) == 2:
blub = argv[1]
else:
print "no argument given - stopping now"
exit(1)
conn = define_db()
cursor = conn.cursor()
write_db(conn,cursor,fni)
if __name__ == "__main__":
main()
...then you would get a pair of errors, which points out exactly what the error is (you stored the argument in blub, but tried to access it with fni):
$ pip install pyflakes
$ pyflakes example.py
example.py:30: local variable 'blub' is assigned to but never used
example.py:37: undefined name 'fni'