Related
is possible to handle all traceback/exception error code in django ?
i try using sys.excepthook = myfunction in the first line manage.py, which my function/method/class provides python functions/methods like send log to syslog and other functions, here's an example of my exception hook function:
#!/usr/bin/env python
# ehook.py
from __future__ import print_function
from make_colors import make_colors
from xnotify import notify
import os, sys
import socket
from configset import configset
import syslogx
import datetime
import re
import traceback
from pydebugger.debug import debug
PID = os.getpid()
class ehook:
CONFIGNAME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ehook.ini')
CONFIG = configset(CONFIGNAME)
def __init__(self, exc_type, exc_value, tb, **kwargs):
#local_vars = {}
#super(ehook, self)
self.exc_type = exc_type
self.exc_value = exc_value
self.tb = tb
data, data_color, raw_data, raw_data_color = self.tracert()
app = "ehook"
title = "Traceback"
event = 'Error'
icon = list(filter(lambda k: os.path.isfile(k), [os.path.splitext(os.path.realpath(__file__))[0] + "." + i for i in ['png', 'jpg']] + [os.path.join(os.path.dirname(os.path.realpath(__file__)), 'traceback') + "." + i for i in ['png', 'jpg']]))
#print("icon:", icon)
kwargs.update({'app': app,})
kwargs.update({'title': title,})
kwargs.update({'event': event,})
kwargs.update({'sticky': True,})
if self.CONFIG.get_config('syslog', 'severity'):
kwargs.update({'syslog_severity': self.CONFIG.get_config('syslog', 'severity').lower(),})
if self.CONFIG.get_config('syslog', 'facility'):
kwargs.update({'syslog_facility': self.CONFIG.get_config('syslog', 'facility').lower(),})
if icon: kwargs.update({'icon': icon[0],})
#icon = os.path.join(os.path.dirname(os.path.realpath(__file__)), icon)
syslog_server = re.split("\n|\t|,", self.CONFIG.get_config('syslog', 'host')) or kwargs.get('syslog_server')
syslog_server = [i.strip() for i in syslog_server]
debug(syslog_server = syslog_server)
if isinstance(syslog_server, list):
for i in syslog_server:
self.debug_server_client(data, data_color, i.split(":")[0])
else:
self.debug_server_client(data, data_color, syslog_server)
kwargs.update({'syslog_server': syslog_server,})
notify.send(message_color = data_color, message = "\n".join(raw_data[1:]), **kwargs)
def sent_to_syslog(self, message, severity=None, facility=None, host = None, port = None):
severity = severity or 3
facility = facility or 3
host = host or self.CONFIG.get_config('syslog', 'host') or '127.0.0.1'
port = port or self.CONFIG.get_config('syslog', 'port') or 514
if "," in host:
host = [i.strip() for i in host.split(",")]
else:
if not isinstance(host, list): host = [host]
if hasattr(message, 'decode'):
message = message.decode('utf-8')
for i in host:
if ":" in i:
HOST, PORT = str(i).split(":")
HOST = HOST.strip()
PORT = PORT.strip()
PORT = PORT or 514
print("sent to %s:%s" % (HOST, str(PORT)))
syslogx.syslog(message, severity, facility, HOST, int(PORT))
else:
syslogx.syslog(message, severity, facility, i, port)
def debug_server_client(self, msg, msg_color = None, host = None, port = None):
dtime = make_colors(datetime.datetime.strftime(datetime.datetime.now(), '%Y:%m:%d'), 'white', 'red') + \
make_colors('~', 'white') + \
make_colors(datetime.datetime.strftime(datetime.datetime.now(), '%H:%M:%S'), 'white', 'green') + \
make_colors(datetime.datetime.strftime(datetime.datetime.now(), '%f'), 'white', 'magenta') + \
"[%s]" % PID
if sys.version_info.major == 3:
msg = msg.encode('utf-8')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
host = host or self.CONFIG.get_config('server', 'host') or '127.0.0.1'
port = port or self.CONFIG.get_config('server', 'port') or 50000
if not host:
try:
if sys.version_info.major == 3:
msg_color = bytes(dtime + " " + msg_color, encoding = 'utf-8')
else:
msg_color = dtime + " " + msg_color
s.sendto((msg_color or msg), (host, port))
except:
traceback.format_exc()
print(msg)
self.sent_to_syslog(msg)
s.close()
#return False
if "," in host:
host = [i.strip() for i in host.split(",")]
else:
host = [host]
for i in host:
if ":" in i:
host1, port1 = str(i).strip().split(":")
port1 = int(port1.strip()) or port
host1 = host1.strip()
try:
if sys.version_info.major == 3:
msg_color = bytes(dtime + " " + msg_color, encoding = 'utf-8')
else:
msg_color = dtime + " " + msg_color
s.sendto((msg_color or msg), (host1, port1))
except:
traceback.format_exc()
print(msg)
self.sent_to_syslog(msg)
s.close()
#print "%s:%s 0" % (host, str(port))
break
else:
host = i.strip()
#print "%s:%s 1" % (host, str(port))
if sys.version_info.major == 3:
msg_color = bytes(dtime + " " + msg_color, encoding = 'utf-8')
else:
msg_color = dtime + " " + msg_color
s.sendto((msg_color or msg), (host, port))
s.close()
break
def tracert(self):
trace = ["Traceback: "]
trace_color = []
while self.tb:
filename = self.tb.tb_frame.f_code.co_filename
name = self.tb.tb_frame.f_code.co_name
line_no = self.tb.tb_lineno
data_color = make_colors("Traceback:", 'b', 'y') + " " + \
make_colors(self.exc_type.__name__, 'lw', 'r') + " (" + \
make_colors(self.exc_value, 'lw', 'm') + "), File " + \
make_colors(filename, 'b', 'lg') + ", line " + \
make_colors(line_no, 'lw', 'bl') + ", in " + \
make_colors(name, 'b', 'lc')
print(data_color)
trace_color.append(data_color)
data = str(self.exc_type.__name__) + ": " + str(self.exc_value) + "), File " + filename + ", line " + str(line_no) + ", in " + name
trace.append(data)
#print(f"File {filename} line {line_no}, in {name}")
#local_vars = tb.tb_frame.f_locals
#print(f"{exc_type.__name__}, Message: {exc_value}")
self.tb = self.tb.tb_next
#print(f"Local variables in top frame: {local_vars}")
return "\n".join(trace), "\n".join(trace_color), trace, trace_color
in manage.py
from ehook import ehook
sys.excepthook = ehook
import sys
import os
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tobi.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
but it doesn't work like what I expected
how to handle all exceptions in django ?
Edited
Q)Can someone help getting the values inserted into mysql database , just confused where place mydb function
Reason :Once I manually enter cntrl+c for .py , then only the values are getting inserted into mysql database
Used in the .py file
here is the complete code , where should i place the mydb function?
Table values not getting inserted into mysql database until cntrl+c is entered to close python file in linux
import os
import re
from builtins import len, Exception
import slack
import logging
from subprocess import check_output
import datetime
import mysql.connector
import time
import json
import requests
#user_threads_info = {}
#thread_ts = ""
#slack.RTMClient.run_on(event='message')
def say_hello(**payload):
try:
##0 get clients and payload
logging.info('msg received')
data = payload['data']
web_client = payload['web_client']
rtm_client = payload['rtm_client']
##0 - 1 Check if it is the first msg, not replied msg by me
# print(data)
if data.get('text') == None:
logging.info('This msg is my replied msg.')
return False
##0-2 Get channel info
channel_id = data['channel']
thread_ts = data['ts']
global user
user = data['user']
#user_info = get_userinfo(user)
#print(user_info)
msg = data['text']
##1 get scenario submsg
retVal = analysis_msg(msg)
# print(retVal)
response = web_client.users_list()
assert(response['ok'])
user_map = {x['id']: x['name'] for x in response['members']}
global user_name
user_name = user_map[user] if user in user_map else None
print(user_name)
if retVal[0] == False:
retMsg = retVal[1] + "\nI can create the following orders. \n" \
"a) spu - store pickup \n" \
"b) sth - ship to home \n" \
"c) del - delivery \n" \
"d) digitalAsGuest - Digital item \n" \
" \n" \
"Please provide information as mentioned in below example.\n" \
" \n" \
"Example: spu:3646989:sftqa3:AMEX\n" \
"\n" \
"Sample SKUS:\n" \
"spu - [3646989,8862011]\n" \
"sth - [2592015,6140094]\n" \
"del - [5592005,8862011]\n" \
"digitalAsGuest - [2810037,5057400]"
send_msg(web_client, channel_id, thread_ts, user, retMsg)
return False
##2 form cmd
retVal = form_cmd(retVal[1])
print(retVal)
if retVal == False:
return False
##3 execute cmd
# inform the start of test
retMsg = "Creating an order,Please wait for the result."
send_msg(web_client, channel_id, thread_ts, user, retMsg)
global res
try:
res1 = os.popen(retVal).read()
print("Printing result...")
print(res1)
print("end of print")
res = reg_result_new(res1)
if res == False:
print("reg_function failure")
retMsg = "The test order placement failed."
else:
retMsg = "Order Id - " + res['id'] + "\nFirst Name - " + res['firstName'] + "\nLast Name - " + res['lastName'] + "\n PhoneNumber - " + res['dayPhoneNumber'] + "\n Email - " + res['email'] + "\n"
except Exception as ee:
retMsg = "The test scenario has a failure. Please Check the feature file."
## 4 send result to slack
# retMsg = "Order Id - " + res['id'] + "\nFirst Name - " + res['firstName'] + "\nLast Name - " + res['lastName'] + "\n PhoneNumber - " + res['day PhoneNumber'] + "\n Email - " + res['email'] + "\n"
create_result_file(user, res)
send_msg(web_client, channel_id, thread_ts, user, retMsg)
print(retVal)
except Exception as e:
print("error")
logging.critical(str(e))
############################ My handlers ##############################
def create_result_file(user, res):
try:
cur_time = datetime.datetime.now()
file_name = user + str(cur_time.year) + str(cur_time.month) + str(cur_time.day) + str(cur_time.hour) + str(
cur_time.minute) + str(cur_time.second) + '.txt'
file = open(file_name, 'w')
file.write(res)
file.close()
except Exception as e:
print(str(e))
def send_msg(web_client, channel_id, thread_ts,user,mgs):
print("thread_ts value is:"+thread_ts)
web_client.chat_postMessage(
channel=channel_id,
text=f"```Hi <#{user}>! \n " + mgs + "```",
thread_ts=thread_ts
)
#def get_userinfo(user):
# payload = {'token': slack_token, 'user': user}
# r = requests.get('https://slack.com/api/users.info', params=payload)
# print(r.text)
# return json.loads(r.text)["user"]
# error code mgmt.
def error_code(code):
# reserved
print(code)
return [False, code]
# break down msg to the test scenario submsgs
def analysis_msg(msg):
global submsg
submsg = msg.split(":")
for value in submsg:
print(value)
if len(submsg) != 4:
logging.warning("This msg not test scenario")
return error_code("Please check the format")
res = {}
res["feature"] = submsg[0]
res["sku"] = submsg[1]
res["env"] = submsg[2]
res["payment"] = submsg[3]
###check
if validate_sku(res["sku"]) == False:
return error_code("INVALID_SKU \n")
if validate_env(res["env"]) == False:
return error_code("INVALID_ENV \n")
if validate_payment(res["payment"]) == False:
return error_code("INVALID_payment \n")
if check_specialCharacter(res["feature"]) == False:
return error_code("INVALID_PROFILE_WITH_SPECIAL_CHARACTER")
return [True, res]
# form cmd for test bat files ! reserved
def form_cmd(submsg):
cmd = 'sh /home/iptbot/iptautobot/test.sh ' + submsg['env'] + ' ' + submsg['feature'] + ' ' + submsg["sku"] + ' ' + submsg["payment"]
return cmd
#code to print user details
#code to print user details
def reg_result_new(res):
start = 'COP Order Response :'
end = 'isGuestMode'
start_index = res.find(start) + len(start)
res = res[start_index:]
end_index = res.find(end) + 22
global data
data = res[:end_index]
try:
print('Data -> ' + str(data))
data = json.loads(data.strip())
new_data = {}
new_data['id'] = data['id']
new_data['firstName'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['firstName']
new_data['lastName'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['lastName']
new_data['dayPhoneNumber'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['dayPhoneNumber']
new_data['email'] = data['lineItems'][0]['fulfillmentInfo']['storeInfo']['agentInfo']['email']
#new_data['firstName'] = data['paymentInfo']['billingAddressInfo']['firstName']
return new_data
except Exception as e:
print('Here error -> '+str(e))
return False
#def reg_result(res):
# "COP Order Response"
# lines = res.split('\n')
# for line in lines:
# pattern = "COP Order Response*"
# prog = re.compile(pattern)
# result = prog.search(line)
# if result == None:
# continue
# res1 = result.string.split('{')
# if len(res1) < 2:
# continue
# res2 = res1[1].split(',')
# if len(res2) < 2:
# continue
# res3 = res2[0].split(':')
# if len(res3) < 2:
# continue
# return res3[1]
# COP Order Response : {"id":"BBY01-200001878853"
# return False
# return val is Boolean
# True/False
# Input type: String
# for positive integer only
# alternative way: Handle exception for int(d)
def validate_sku(sku_val):
return sku_val.isnumeric()
# input val : string
# return val: Boolean
def validate_env(env_val):
env_list = [
"sftqa1" , "sftqa2" , "sftqa3" , "sftqa4"
]
if env_val in env_list:
return True
else:
return False
def validate_payment(payment_val):
env_payment = [
"AMEX","VISA"
]
if payment_val in env_payment:
return True
else:
return False
# input val : string
# return val: Boolean
def check_specialCharacter(s):
if s == "":
return False
if s.isspace():
return False
return s.isalnum()
slack_token = os.environ["SLACK_API_TOKEN"]
rtm_client = slack.RTMClient(token=slack_token)
rtm_client.start()
#database connction
mydb = mysql.connector.connect(
host="host",
user="user",
passwd="pass",
database="db"
)
mycursor = mydb.cursor()
for value in submsg:
print(value)
fulfilment=submsg[0]
sku=submsg[1]
environment=submsg[2]
payment=submsg[3]
ts = time.time()
date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
orderNumber=data['id']
username=user_name
print(fulfilment)
print(sku)
print(environment)
print(payment)
print(username)
print(orderNumber)
sqlformula = "INSERT INTO orderDetails (fulfilment,sku,environment,payment,orderNumber,date,user) VALUES (%s,%s,%s,%s,%s,%s,%s)"
#order=("sth",3643387,"sftqa2","AMEX")
#mycursor.execute(sqlformula,order)
mycursor.execute(sqlformula,(fulfilment,sku,environment,payment,orderNumber,date,username))
mydb.commit()
mydb.close()
Output
1 sh /home/iptbot/iptautobot/test.sh sftqa3 spu 3646989 AMEX
2 error
3 CRITICAL:root:'user'
4 error
5 CRITICAL:root:'user' // clicking Control+C values get inserted
6 ^CWARNING:slack.rtm.client:Websocket was closed.
7 3646989
8 sftqa3
9 AMEX
10 spu
11 3646989
12 sftqa3
13 AMEX
14 a6002043
15 BBY01-200002091354
You are stuck at this point because rtm_client.start() is a synchronous call.
If you want it to be asynchronous (non-blocking) then you should run:
rtm_client.start(run_async=True)
Here it is good walk-through on how to setup async usage of the library. Also have a look at the method signature for RTMClient to get an idea of how it works.
Here's a good example detailing a lot of what you would need in your case.
Then you will hit your db execution code where you will need to have a while loop to go through the data you want to add to the DB.
I would recommend that you use a Queue for this as it is synchronised and will be easier to manage than a global list which is overwritten on every order. Preferably you could use asyncio.Queue with an example of implementation here
When an order has passed the validation steps add it to the queue. Here is some pseudo code describing the flow with a basic (not asyncio) Queue:
import queue
q = queue.Queue()
def validate_order(order):
valid_order_data = ......
q.put(valid_order_data)
while True:
valid_order = q.get() # Will wait until there is a value on the queue
mycursor.execute(sqlformula, (valid_order))
I have the below Python script and it works very well, but I would like to introduce some fail safe options .. That fail safe options being ..
1) if I cannot find (in this example) Michael I would like to write to the file ERROR ..
2) If the database does not allow me to connect for whatever reason I would like to write to another file CONNECTION_ERROR
Here is my script:
#! /usr/bin/python
import pymssql
import sys
sys.path.insert(0, '/opt/mount/safe')
from secrets import password
conn = pymssql.connect(
server="server",
port=port,
user="user",
password=password,
database="database")
conn
cursor = conn.cursor()
cursor.execute("SELECT name, address FROM database WHERE name = 'michael'")
with open('safe/file.txt', 'w') as f:
for row in cursor.fetchall():
print ( "Person " + (row[0])),
print ( "has this address " + (row[1]))
f.write(str( "Person " + (row[0])))
f.write("%s\n" % str( " has this address " + (row[1])))
conn.close()
Took me a while but the below works really really well
import sys, pymssql, string, os, calendar, datetime, traceback, socket, platform
try:
d = datetime.datetime.now()
log = open("LogFile.txt","a")
log.write("----------------------------" + "\n")
log.write("----------------------------" + "\n")
log.write("Log: " + str(d) + "\n")
log.write("\n")
# Start process...
starttime = datetime.datetime.now()
log.write("Begin process:\n")
log.write(" Process started at "
+ str(starttime) + "\n")
log.write("\n")
xxxxxx
CODE HERE
XXXXXX
endtime = datetime.datetime.now()
# Process Completed...
log.write(" Completed successfully in "
+ str(endtime - starttime) + "\n")
log.write("\n")
log.close()
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
# Concatenate information together concerning
# the error into a message string
pymsg = "PYTHON ERRORS:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info())
# Return python error messages for use in
# script tool or Python Window
log.write("" + pymsg + "\n")
log.close()
I have 500+ units in the world that connects to my server and dump their data. Up to now i have been using a PHP script to act as a socket listener, but I need to go multi-threaded as the load is increasing and PHP can not keep up. I am quite new to Python and decided to use it for my new platform, over the past few days i have struggled and tried many examples to no avail. Through my search i came across some questions trying to answer this problem, but none did. I will attach my code.
The problem : as the units connect to the server and the server accepts it, the server creates a new thread to handle the connection, the problem comes when the unit drops the connection the thread stays open and active and the total thread count grows, and this is linked to the system limit : "number of open files", i can increase this limit but this only make it a time bomb , it does not solve this.
Please help.
#! /usr/bin/python
import multiprocessing
import socket
import sys
import pprint
import datetime
import MySQLdb
import time
import datetime
import re
import select
import resource
import threading
max_connections = 1024
max_connections_set = max_connections
resource.setrlimit(resource.RLIMIT_NOFILE, (max_connections_set, max_connections_set))
#the incomming port
the_global_port = xxxx #(any port)
#display id
the_global_id = "UNIT TYPE"
class ConnectionObject(object):
the_id = None
the_socket = None
the_socket_address = None
the_socket_address_ip = None
the_socket_address_port = None
the_server = None
the_process = None
the_process_id = None
the_process_name = None
the_imei = None
identifier = ""
# The class "constructor" - It's actually an initializer
def __init__(self, in_process_nr, in_process , in_socket , in_socket_address, in_server):
self.the_id = in_process_nr
self.the_process = in_process
self.the_process_id = self.the_process.exitcode
self.the_process_name = self.the_process.name
self.the_socket = in_socket
self.the_socket_address = in_socket_address
self.the_socket_address_ip = self.the_socket_address[0]
self.the_socket_address_port = self.the_socket_address[1]
self.the_server = in_server
self.identifier = str(self.the_id) + " " + str(self.the_process_name) + " " + str(self.the_socket_address_ip) + " " + str(self.the_socket_address_port) + " "
#end def init
#end def class
def processData(the_connection_object , the_data):
def mysql_open_connection_inside():
try:
the_conn = MySQLdb.connect(host= "127.0.0.1",
user="user",
passwd="password",
db="mydb")
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
time.sleep(30)
try:
the_conn = MySQLdb.connect(host= "127.0.0.1",
user="user",
passwd="password",
db="mydb")
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
print "Unexpected error:", sys.exc_info()[0]
raise
sys.exit(0)
#end time 2
#end try except
return the_conn
#end def mysql_open_connection
conn = mysql_open_connection_inside()
x = conn.cursor()
add_rawdata = ("INSERT INTO RawData"
"(ID,RawData,Type) "
"VALUES (%s, %s, %s)")
data_raw = ('123456', 'ABCD','')
records_inserted = 0
the_connection_object.the_imei = ""
#global clients
imei = ""
try:
thedata = ""
thedata = " ".join("{:02x}".format(ord(c)) for c in the_data)
record_to_save = ' '.join(thedata)
seqtoreply = ""
seqtoreply = "OK"
#reply part
if (seqtoreply != ""): #reply to unit
try:
the_connection_object.the_socket.sendall(seqtoreply)
#echoout(the_connection_object.identifier+"We Replyed With : " + seqtoreply)
except:
echoout(the_connection_object.identifier+"Send Reply Error : " + str(sys.exc_info()[1]))
#end except
#end of if
the_id = "some generated id"
data_raw = (the_id, werk_data, 'UNIT')
try:
x.execute(add_rawdata, data_raw)
conn.commit()
echoout(the_connection_object.identifier+"Raw Data Saved.")
except:
conn.rollback()
echoout(the_connection_object.identifier+" Raw Data NOT Saved : " + str(sys.exc_info()[1]))
#end of data save insert
#echoout("=============================")
endme = 1
echoout("")
conn.close()
#end try
except:
conn.close()
echoout(the_connection_object.identifier+"Error : " + str(sys.exc_info()[1]))
#end try except
#end def handel function
def handle_p(processnr, server, connection, address):
this_connection = ConnectionObject(processnr,multiprocessing.current_process(), connection, address, server)
thisprocess = multiprocessing.current_process()
this_connection.the_id = ""
the_address = this_connection.the_socket_address_ip
the_port = this_connection.the_socket_address_port
try:
echoout("New connection from : "+str(the_address)+" on port "+str(the_port))
close_the_socket = False
while True:
#--------------------- recive part -------------------------------------------------
data = connection.recv(512)
thedata = ""
thedata = " ".join("{:02x}".format(ord(c)) for c in data)
if ((thedata == "") or (thedata == " ") or (data == False)):
echoout("Socket Closed Remotely : No Data")
close_the_socket = True
break
#end - if data blank
else :
processData(this_connection, data)
#end there is data
echoout("=============================")
#end if while true
#end try
except:
print "handling request, Error : " + str(sys.exc_info()[1])
close_the_socket = True
connection.close()
finally:
close_the_socket = True
echoout("Closing socket")
connection.close()
#end try finally
#end def handel function
def mysql_update(update_statement, update_values):
conn_update = MySQLdb.connect(host= "127.0.0.1",
user="user",
passwd="password",
db="mydb")
x_update = conn_update.cursor(MySQLdb.cursors.DictCursor)
rawdata_data = (update_statement)
data_rawdata = (update_values)
allupdateok = False
#end if there is more
try:
x_update.execute(rawdata_data, data_rawdata)
conn_update.commit()
allupdateok = True
conn_update.close()
except:
conn_update.rollback()
allupdateok = False
conn_update.close()
print "Unexpected error:", sys.exc_info()[0]
raise
#end of data save insert
if (allupdateok == False):
echoout("Update Raw Data Table Error")
#end if update
return allupdateok
#end def mysqlupdate
def echoout(thestring):
datestring = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if (thestring != ""):
outstring = datestring + " " + thestring
print outstring
else :
outstring = thestring
print outstring
#end - def echoout
class Server(object):
threads = []
all_threads = []
high_proc = ""
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
def start(self):
echoout("Listening for conncetions")
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind((self.hostname, self.port))
self.socket.listen(10)
process_number = 1
inputs = [self.socket]
while True:
inready, outready, excready = select.select(inputs, [], [], 30);
for s in inready:
if s == self.socket:
conn, address = self.socket.accept()
high_processname = ""
echoout("Got a connection...")
process = threading.Thread(target=handle_p, args=(process_number,self, conn, address))
high_processname = process.name
self.high_proc = high_processname
process.daemon = True
process.start()
self.all_threads.append((process,conn))
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
self.threads.append((process_number,conn,st,0,process))
process_number = process_number + 1
print "ACTIVE Threads = " + str(threading.activeCount())
the_total_count = 0
dead_count = 0
alive_count = 0
for the_thread in self.all_threads :
if (the_thread[0].is_alive() == True):
alive_count = alive_count + 1
else :
dead_count = dead_count + 1
the_thread[1].close()
the_thread[0].join(0.3)
self.all_threads.pop(the_total_count)
#end if alive else
the_total_count = the_total_count + 1
#end for threads
print "LIVE Threads = " + str(alive_count)
print "DEAD Threads = " + str(dead_count)
print "TOTAL Threads = " + str(the_total_count)
print ""
#end if s = socke, new connection
#end for loop
#end while truw
self.socket.close()
#end def start
#main process part
if __name__ == "__main__":
start_ip = "0.0.0.0"
start_port = the_global_port
#start server
server = Server(start_ip, start_port)
try:
print "Listening on " , start_port
server.start()
except:
print "unexpected, Error : " + str(sys.exc_info()[1])
finally:
print "shutting down"
active_clients = 0
for process in multiprocessing.active_children():
try:
active_clients = active_clients + 1
process.terminate()
#process.join()
except:
print "Process not killed = " + str(sys.exc_info()[1])
#end try except
#close mysql connection
print "Active clients = " + str(active_clients)
#end try finally
server.socket.close()
server.threads = []
server = None
print "All done."
#end def main
First of all, it is silly to use threads when you can have 500+ connected clients, you should go asynchronous - look at gevent for example for
a very good library, or at least use select (see Python documentation).
Then, your code to close the socket in handle_p looks good, indeed when
the recv() call comes back with an empty string it means the remote
end is disconnected so you break the while, fine.
However, it looks like the remote closed the connection but it is not
detected on your side (recv() doesn't return). The best would be
then to have a kind of heartbeat to know when you can close the
connection.
Since COSM has become Xively, a nice device api has been added (or was always there- not sure). The flow is
create product batch with serial numbers
activate devices using some product batch identifiers (?)
start using the device with the obtained feed/api keys
I can't figure out how to do this via the python API- are there any pointers?
This should be added to the library, but for now you can use this code to implement device activation. I have used environment variables to store product secret and device serial, but change that for anything that suites your use case. The only tricky part is that you need to call a2b_hex().
import xively
from os import environ
from hashlib import sha1
from binascii import a2b_hex
import hmac
secret = environ['XIVELY_PRODUCT_SECRET']
serial = environ['XIVELY_DEVICE_SERIAL_NUMBER']
activation = hmac.new(a2b_hex(secret), serial, sha1).hexdigest()
creds = xively.Client(key=None).get('/v2/devices/'+activation+'/activate').json()
xi_feed = xively.XivelyAPIClient(creds['apikey']).feeds.get(creds['feed_id'])
You will also need take care to store the credentials into a file, as a device can be activated only once. You will notice 403 errors if you try to run this code again and again, so do use the Xively developer workbench for deactivating the device under test (you may need to refresh the page).
Here is a fully working example using config files or environment variables:
#!/usr/bin/python
from os import environ
from hashlib import sha1
from binascii import a2b_hex
import hmac
import sys, subprocess
import ConfigParser
import xively
CONFIG_FILE = 'xively.conf'
PROVISIONING = 'PROVISIONING'
PROVISIONING_PRODUCT_SECRET = 'PRODUCT_SECRET'
PROVISIONING_DEVICE_SERIAL = 'DEVICE_SERIAL'
PROVISIONING_FEED_ID = 'FEED_ID'
PROVISIONING_API_KEY = 'API_KEY'
def get_setting(config, section, key):
try:
value = config.get(section, key)
except:
print key + " not found in config file. Using environment variable " + key + " instead."
try:
value = environ[key]
except:
print key + " not found in environment."
raise
# value defined?
if not value:
raise
return value
def xively_activate_product(secret, serial):
activation = hmac.new(a2b_hex(secret), serial, sha1).hexdigest()
creds = xively.Client(key=None).get('/v2/devices/'+activation+'/activate').json()
return creds
# main
config = ConfigParser.RawConfigParser()
config.read(CONFIG_FILE)
try:
# see if we already have an api key and feed id
feed_id = config.get(PROVISIONING, PROVISIONING_FEED_ID)
api_key = config.get(PROVISIONING, PROVISIONING_API_KEY)
print "Provisioned product details:"
print "FEED_ID: " + str(feed_id)
print "API_KEY: " + api_key
# continue working with your activated product here
except:
print "FEED_ID and API_KEY not found. Activating product now."
# no error handling for secret- it _is_ needed
try:
secret = get_setting(config, PROVISIONING, PROVISIONING_PRODUCT_SECRET)
except:
print "Finding " + PROVISIONING_PRODUCT_SECRET + " failed. Giving up."
sys.exit(1)
try:
serial = get_setting(config, PROVISIONING, PROVISIONING_DEVICE_SERIAL)
except:
serial = subprocess.check_output("hostname", shell=True)
if not serial:
print "Fallback to hostname for " + PROVISIONING_DEVICE_SERIAL + " failed. Giving up."
sys.exit(1)
try:
creds = xively_activate_product(secret, serial)
# check if there were errors
try:
creds["errors"]
except:
pass
else:
print "Product activation failed (" + creds["title"] +": "+ creds["errors"] + ")."
sys.exit(1)
feed_id = creds['feed_id']
api_key = creds['apikey']
print "Product activation successful."
print "FEED_ID: " + str(feed_id)
print "API_KEY: " + api_key
if not config.has_section(PROVISIONING):
config.add_section(PROVISIONING)
config.set(PROVISIONING, PROVISIONING_FEED_ID, feed_id)
config.set(PROVISIONING, PROVISIONING_API_KEY, api_key)
# Writing our configuration file to 'example.cfg'
with open(CONFIG_FILE, 'wb') as configfile:
config.write(configfile)
except Exception as e:
print "Product activation failed (" + str(e) +")."
sys.exit(1)
This is another helpful class I wrote:
## Logging for debugging purposes
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
import os
from os import environ
from hashlib import sha1
from binascii import a2b_hex
import hmac
import sys, subprocess
import ConfigParser
import xively
PROVISIONING = 'PROVISIONING'
PROVISIONING_PRODUCT_SECRET = 'PRODUCT_SECRET'
PROVISIONING_FEED_ID = 'FEED_ID'
PROVISIONING_API_KEY = 'API_KEY'
class XivelyManager:
def __init__(self, settings="xively.conf"):
# main
self.settings=settings
self.config = ConfigParser.RawConfigParser()
self.config.read(settings)
try:
# see if we already have an api key and feed id
self.api_key = self.get_setting(PROVISIONING, PROVISIONING_API_KEY)
self.secret = self.get_setting(PROVISIONING, PROVISIONING_PRODUCT_SECRET)
# continue working with your activated product here
except:
logger.exception( "API KEY and SECRET NOT FOUND" )
def activate_sensor(self,serial):
try:
creds = self.xively_activate_product(str(serial))
# check if there were errors
try:
creds["errors"]
except:
pass
else:
logger.exception("Product activation failed (" + creds["title"] +": "+ creds["errors"] + ").")
return False
feed_id = creds['feed_id']
api_key = creds['apikey']
if not self.config.has_section(PROVISIONING):
self.config.add_section(PROVISIONING)
if not self.config.has_section(str(serial)):
self.config.add_section(str(serial))
self.config.set(PROVISIONING, PROVISIONING_API_KEY, api_key)
self.config.set(str(serial), PROVISIONING_FEED_ID , feed_id)
# Writing our configuration file to 'xively.cfg'
with open(self.settings, 'wb') as configfile:
self.config.write(configfile)
return True
except Exception as e:
logger.exception("Product activation failed (" + str(e) +").")
return False
def get_setting(self, section, key):
try:
value = self.config.get(section, key)
except:
logger.exception( key + " not found in config file. Using environment variable " + key + " instead.")
## try:
## value = environ[key]
## except:
## logger.exception( key + " not found in environment.")
## finally:
## pass
finally:
# value defined?
if not value:
raise
return value
def get_feed(self,serial):
try:
if self.config.has_section(str(serial)):
feed_id = self.get_setting(str(serial), PROVISIONING_FEED_ID)
else:
feed_id=False
except Exception, e:
feed_id=False
finally:
return feed_id
def xively_activate_product(self, serial):
activation = hmac.new(a2b_hex(self.secret), serial, sha1).hexdigest()
creds = xively.Client(key=None).get('/v2/devices/'+activation+'/activate').json()
return creds
if __name__ == "__main__":
print "Testing Xively Manager "
settings = os.path.join(os.path.dirname(sys.argv[0]), "config", "xively.conf")
print settings
testxive=XivelyManager(settings)
#print testxive.activate_sensor(10)
print testxive.get_feed(10)
This is helpful when your internet gateway is connected to several other devices.
Your config file will be updated with stuff like this:
[PROVISIONING]
product_secret = xxxxxxxxxxxxxxxxxxxxxxxxxxxx
api_key = xxxxxxxxxxxxxxxxxxxxxxxx
[productserial1]
feed_id = xxxxxxxx
[productserial2]
feed_id = xxxxxxxx