We Keep Coding

Python decorated function unable to return the correct value

I have written this code to decorate a function if the underlying sql scripts being called is too noisy. The gp_info_decorator(function) modifies the execute_gp_func() to suppress messages from the sql scripts. Instead, it just stores the result in the Status variable.
But, the decorated function is returning None, instead of current date.
import sys
import psycopg2
import json
import os
#import contextlib.contextmanager
import sys
# functions to train and predict
class GetExecutor:
#global props
# create instance variables from the properties file
def __init__(self,home_dir, os_type = "linux"):
if sys.platform in ('linux', 'linux2'):
self.home_dir = home_dir
self.os_type = 'linux'
self.dbname = None
self.user = None
self.host = None
self.passwd = None
self.conn = None
elif sys.platform in ('win32'):
self.home_dir = home_dir
self.os_type = 'windows'
self.dbname = None
self.user = None
self.host = None
self.passwd = None
self.conn = None
def read_properties(self):
with open('properties.json', 'r') as f1:
props = json.load(f1)
self.dbname = props['dbname']
self.user = props['user']
self.host = props['host']
self.passwd = props['passwd']
#self.start_date = props['start_date']
#self.end_date = props['end_date']
print ("Properties stored")
return props
# sql connector
def sql_connect(self):
try:
connect_string = "dbname = %s user = %s host = %s password = %s"%(self.dbname, self.user, self.host, self.passwd)
conn = psycopg2.connect(connect_string)
conn.set_isolation_level(0)
print ("Connecting --")
except:
print "Unable to connect to the database"
self.conn = conn
# suppress logger info from sql if it is too much noise during execution
def gp_info_decorator(function):
def wrapper(*args, **kwargs):
#print args[0].home_dir
#print args[0].query
#with open(sys.stdout, "w") as devnull:
function(*args, **kwargs)
"""
with open(os.devnull, "w") as devnull:
old_stdout = sys.stdout
sys.stdout = devnull
try:
function(*args, **kwargs)
finally:
sys.stdout = old_stdout
"""
print ("Debug 3--->", wrapper)
return wrapper
# Execute training
#gp_info_decorator
def execute_gp_func(self, home_dir, query):
print ("Function Name:", query)
with self.conn.cursor() as cursor:
cursor.execute(query)
print "Executing-->"
Status= cursor.fetchall()
print ("Status is ", Status)
for notice in self.conn.notices:
print notice
print ("Debug 2--->", Status)
#Status =
return Status
def main():
home_dir = os.getcwd()
print home_dir
obj = GetExecutor(home_dir)
print obj
props = obj.read_properties()
obj.sql_connect()
#status = obj.execute_gp_func(home_dir,"select get_analytic.fn_get_logistics_train_data(%s,%s);"%(props['start_date'],props['end_date']))
status = obj.execute_gp_func(home_dir,"select current_date")
print ("Status of load cost training:" , status)
# Bill process date date params
#status = obj.execute_gp_func(home_dir,"select get_analytic.fn_get_logistics_bill_date_train(%s,%s);"%(props['start_date'],props['end_date']))
print ("Status of bill process date training:" , status)
if __name__ == '__main__':
main()
On execution, I see that the print within function has correct value for date. But, it goes into the wrapper and then returns the None value.
<__main__.GetExecutor instance at 0x000000000AE8DE48>
Properties stored
Connecting --
('Function Name:', 'select current_date')
Executing-->
('Status is ', [(datetime.date(2017, 8, 29),)])
('Debug 2--->', [(datetime.date(2017, 8, 29),)])
('Debug 3--->', <function wrapper at 0x000000000AE56AC8>)
('Status of load cost training:', None)
('Status of bill process date training:', None)

Looks as if your wrapper only calls the function, but does nothing with the return value:
def gp_info_decorator(function):
def wrapper(*args, **kwargs):
#print args[0].home_dir
#print args[0].query
#with open(sys.stdout, "w") as devnull:
result = function(*args, **kwargs) # <-- remember result
"""
... # <-- removed for readability
"""
print ("Debug 3--->", wrapper)
return result # <-- return result
return wrapper
#gp_info_decorator
def fn1():
return 5
r = fn1()
print(r) # Output: 5

What could be the reason python says port is being used when its not?

No matter what port I set Python says its being used I don't understand why, I'm using twisted
Sat Aug 26 12:49:31 2017 - (/usr/lib/python2.7/dist-packages/twisted/internet/tcp.py:980) Couldn't listen on any:4444: [Errno 98] Address already in use.
I'm not sure what code portions you need or information so if you need anything let me know.
server.py
import glob
import uuid
from modules import util
from modules import db as _db
from modules import LOG
from objects.user import User
from objects.room import Room
from objects.message import Message
from objects.Table import Table
from autobahn.twisted.websocket import WebSocketServerProtocol, \
WebSocketServerFactory, \
listenWS
def hexc(e):
et, ev, tb = sys.exc_info()
if not tb:
return str(e)
while tb:
lineno = tb.tb_lineno
fn = tb.tb_frame.f_code.co_filename
tb = tb.tb_next
return "(%s:%i) %s" % (fn, lineno, str(e))
class oChat(WebSocketServerProtocol):
_rooms = []
_userlist = Table()
_commands = Table()
_commands.user = Table()
db = _db.db('/home/chat/database.db')
def onOpen(self):
self.loadUserCommands()
self.loadSysCommands()
def getLevel(self, user):
if user.mod:
return 1
elif user.owner:
return 2
else:
return 0
def add(self, object):
if object not in self._rooms:
self._rooms.append(object)
def get(self, name):
for room in self._rooms:
if room.name == name:
return room
def execFile(self, f, dict):
with open(f, "r") as file:
try:
exec (compile(file.read(), f, 'exec'), dict)
except:
execfile(f, dict)
file.close()
return dict
def config(self, value):
config = {}
self.execFile("configuration/config.conf", config)
return config[value]
def getCommand(self, name):
name = name.lower()
if self._commands.has_key(name):
if not self._commands[name].disabled:
return self._commands[name]
def getUserCommand(self, name):
name = name.lower()
if self._commands.user.has_key(name):
if not self._commands.user[name].disabled:
return self._commands.user[name]
def setCommand(self, name, desc, func, disabled=False):
name = name.lower()
self._commands[name] = Table()
self._commands[name].desc = desc
self._commands[name].func = func
self._commands[name].disabled = disabled
def setUserCommand(self, name, desc, func, disabled=False, level=0):
name = name.lower()
self._commands.user[name] = Table()
self._commands.user[name].desc = desc
self._commands.user[name].func = func
self._commands.user[name].level = level
self._commands.user[name].disabled = disabled
def reload(self):
try:
self.loadSysCommands()
self.loadUserCommands()
except Exception as e:
print hexc(e)
def make_user(self, *args):
return User(*args)
def make_room(self, *args):
return Room(*args)
def make_message(self, *args):
return Message(*args)
def loadUserCommands(self):
files = glob.glob("protocol/user/*.py")
for file in files:
b = self.execFile(file, {})
b['init'](self)
def loadSysCommands(self):
files = glob.glob("protocol/*.py")
for file in files:
b = self.execFile(file, {})
b['init'](self)
def joinRoom(self, room, user, args):
has_sym = util.has_symbles(args, False)
room.removeUser(user, self)
room._sendCommand("uc", str(room.getCount(self)))
if args in self.db.getRooms():
room.addUser(user, self)
user.setNewRoom(room.name)
self.add(room)
room._sendCommand("uc", str(room.getCount(self)))
return True
else:
args = args.replace(" ", "-")
if not has_sym and user.status == "logged_in":
self.db.addRoom(args, user.name)
room = Room(args, self)
self.add(room)
user.setNewRoom(args)
room.addUser(user, self)
self.db.setTitle(room.name, user.name, room.name)
room._sendCommand('title', room.name)
room._sendCommand("uc", str(room.getCount(self)))
return True
else:
return False
def onConnect(self, req):
self.id = uuid.uuid4().hex
User(self.id).setIdent(db._ident(str(self.peer.split(":", 2)[1])))
User(self.id).setConnection(self.id, self)
msg = "[CONNECT] IP(%s) IDENTITY(%s)" % (str(self.peer.split(":", 2)[1]), User(self.id).ident)
print(LOG.Log(msg))
def onMessage(self, payload, isBinary):
data = payload.decode('utf8').split("::", 1)
user = User(self.id).get()
room = self.get(user.roomname)
if not room: room = Room(user.roomname.lower(), self)
try: room.check(user, self.db)
except: pass
print LOG.Log(payload.decode("utf8"))
if len(data) > 1:
cmd, args = data[0], data[1]
else:
cmd, args = data[0], ""
if cmd == "bmsg":
if args.startswith(self.config("USER_PROTOCOL_SEP")):
data = args.split(self.config("USER_PROTOCOL_SEP"), 1)
data = data[1].split(" ", 1)
if len(data) > 1:
cmd, args = data[0], data[1]
else:
cmd, args = data[0], ""
key = cmd
cmd = self.getUserCommand(key)
msg = Message(room.name, user.name, args, ident=user.ident)
if cmd and self.getLevel(user) >= cmd.level: # user protocol
try: cmd.func(self, room, user, msg, args)
except Exception as e: user._sendCommand('sysmsg', 'er', hexc(e))
else:
if not user.banned:
key = cmd
msg = Message(room.name, user.name, args, ident=user.ident) # just storing the message the bmsg.py handles sending
msg.createMessage(self.db, True)
cmd = self.getCommand(key)
if cmd: # main protocol bmsg
if user.status == 'logged_in': cmd.func(self, room, user, msg, args)
else: user._sendCommand('sysmsg', 'er', 'login or register')
else:
user._sendCommand('sysmsg', 'banned', 'you are banned') # set on sending live msg only
else:
key = cmd
cmd = self.getCommand(key)
if cmd: # main protocol other
msg = Message(room.name, user.name, args, ident=user.ident, storeMsg=False)
try: cmd.func(self, room, user, msg, args)
except Exception as e: user._sendCommand("sysmsg", "er", hexc(e))
if __name__ == '__main__':
try:
import sys
from twisted.internet import reactor,ssl
contextFactory = ssl.DefaultOpenSSLContextFactory('/etc/letsencrypt/live/otku.ga/privkey.pem',
'/etc/letsencrypt/live/otku.ga/fullchain.pem')
factory = WebSocketServerFactory(u"wss://otku.ga:4444")
factory.protocol = oChat
listenWS(factory, contextFactory)
#log.startLogging(sys.stdout)
#factory.setProtocolOptions(maxConnections=2)
reactor.listenTCP(4444, factory)
reactor.run()
except KeyboardInterrupt:
print("[ERR] KBI")
except Exception as e:
LOG.Log(hexc(e), 'er')
I don't have any errors other then the port being in use when its not.
If you need the whole server ill provide a zip with requirements.txt

You’re setting the server up to listen twice – once with listenWS and once with reactor.listenTCP. Remove reactor.listenTCP, as you want listenWS to call reactor.listenSSL for you.

There are various reasons this might happen [General Solution],
Reason 1: You may tried running your application is one of the reserved ports [0-1024] so some applications might be using the port actually.
Reason 2: You may terminated the application, so the instruction for closing the socket (e.g socket.close()) has never gets called. so the socket is open somewhere
Reason 3: Is that your only error message? Does it says anything about admin permission? Have you tried running in admin permission?

Update client widget from server with rpyc callback

I want to edit all client QTextBrowser logs when any client sends some message to server.
my intended procedure is this:
[client]enter -> [client]chatUI.handleEnter -> (RFC)[server]exposed_send -> [server]broadcast -> (RFC)[clients]update.emit() -> [clients]listen -> log changed
When I run this code, other clients logs are not changed and only the client that give input to server has an updated log.
How I can solve this to update all clients properly?
chat_server.py
import rpyc
import random
import string
from threading import RLock
users = dict()
callbacks = dict()
user_num = 0
lock = RLock()
buf = dict()
class chatService(rpyc.Service):
def on_connect(self):
global user_num
with lock:
user_num = user_num+1
print ("connect user: %d" % user_num)
def on_disconnect(self):
global user_num
with lock:
user_num = user_num-1
print ("disconnect user: %d" % user_num)
def exposed_accept(self, idt, callback):
with lock:
global users
global callbacks
if not isinstance(idt, str) or len(idt) != 6:
return False
elif idt in users:
return -1
else:
pw = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
users[idt] = pw
callbacks[idt] = rpyc.async(callback)
return pw
def exposed_send(self, target, msg, idt, pw):
print ('here')
name = self.identify(idt, pw)
if name == False:
print ('here2')
return False
else:
print ('here3')
global callbacks
if target == None:
self.broadcast("[%s] %s" % (name, msg))
elif msg.target in callbacks:
self.send("[%s] %s" %(name, msg), target)
else:
return False
def exposed_order(self, msg, idt, pw):
pass
def identify(self, idt, pw):
global users
if users[idt] == pw:
return idt
else:
return False
def broadcast(self, msg):
with lock:
print("bloadcast calls")
global callbacks
global buf
for user, callback in callbacks.items():
if user not in buf or buf[user] == None:
buf[user] = (msg,)
else:
buf[user] = buf[user] + (msg,)
callback()
def send(self, msg, target):
global callbacks
global buf
if user not in buf or buf[user] == None:
buf[target] = (msg,)
else:
buf[target] = buf[target] + (msg,)
callbacks[target]()
def exposed_get_buf(self, user):
global buf
temp = buf[user]
buf[user] = None
return temp
if __name__ == '__main__':
from rpyc.utils.server import ThreadedServer
t = ThreadedServer(chatService, port = 3743)
t.start()
chat_client.py
from chatUI import *
import rpyc
import random
import string
if __name__ == '__main__':
service = rpyc.connect('floating.tk', 3743)
app, chat = UIReady(service)
while True:
idt = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
pw = service.root.accept(idt, chat.update.update.emit)
if pw != False and pw != -1:
break
chat.idt = idt
chat.pw = pw
sys.exit(app.exec_())
chatUI.py
import sys
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
class Updater(QObject):
update = pyqtSignal()
class Chat(QWidget):
log = None
enter = None
def __init__(self, service) :
super().__init__()
self.service = service
self.idt = None
self.pw = None
self.initUI()
self.update = Updater()
self.update.update.connect(self.listen)
def initUI(self):
logLabel = QLabel('chat log')
enterLabel = QLabel('enter')
self.log = QTextBrowser()
self.enter = QLineEdit()
self.enter.returnPressed.connect(self.handleEnter)
layout = QGridLayout()
layout.addWidget(logLabel, 0, 0)
layout.addWidget(self.log, 0, 1, 5, 1)
layout.addWidget(enterLabel, 6, 0)
layout.addWidget(self.enter, 6, 1)
self.setLayout(layout)
self.setWindowTitle('chat')
self.resize(600, 600)
self.show()
def handleEnter(self):
msg = self.enter.text()
self.enter.setText("")
self.service.root.send(None, msg, self.idt, self.pw)
print('get enter')
def listen(self):
msg = self.service.root.get_buf(self.idt)
for m in msg:
self.log.append(m)
def UIReady(service):
app = QApplication(sys.argv)
chat = Chat(service)
return app, chat

Still can't stop the Thread

I want to stop the thread with simple CTR+C, but data is keep coming and coming, so I have to close the window and start it again. Help me please to fix the issue. In order to see what I'm saying, just type your twitter username and password in user and pass
import threading
import random
import time
import Queue
import urllib2
import sys
import simplejson, pycurl
import sys, signal
queue = Queue.Queue()
keep_running = True
user = "" # username
pswd = "" # password
headers = [ "Content-Type: text/xml; charset: UTF-8; "]
class MyThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
self.buffer = ""
self.streaming_url = "https://stream.twitter.com/1/statuses/filter.json?track=justinbieber"
def start_crawl(self, user, pswd):
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (user, pswd))
self.conn.setopt(pycurl.URL, self.streaming_url)
#self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.HTTPHEADER, headers)
self.conn.setopt(pycurl.READFUNCTION, self.storage)
self.conn.setopt(pycurl.VERBOSE, 1)
self.conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
self.conn.perform()
def storage(self, data):
self.buffer += str(data)
if isinstance(self.buffer, str):
if self.buffer.endswith("\r\n") and self.buffer.strip():
content = json.loads(self.buffer)
self.buffer = ""
self.queue.put(content)
self.queue.task_done()
#if "text" in content:
#print u"{0[user][name]}: {0[text]}".format(content)
else:
return data
def run(self):
while keep_running:
self.start_crawl(user,pswd)
if signal.signal(signal.SIGINT, signal.SIG_DFL):
sys.exit()
#line = self.names[random.randint(0,len(self.names)-1)]
#queue.put(line)
class Starter():
def __init__(self):
self.queue = queue
t = MyThread(self.queue)
t.start()
self.next()
def next(self):
while True:
time.sleep(0.1)
if not self.queue.empty():
line = self.queue.get(timeout=0.2)
print line, self.queue.qsize()
else:
print 'waiting for queue'
def main():
try:
Starter()
queue.join()
except KeyboardInterrupt, e:
print 'Stopping'
global keep_running
keep_running = False
#Join all existing threads to main thread.
for thread in threading.enumerate():
if thread is not threading.currentThread():
thread.join()
sys.exit(1)
main()

Set the thread as daemon and it will be killed with your program
import threading
import random
import time
import Queue
import urllib2
import sys
import simplejson, pycurl
import sys, signal
queue = Queue.Queue()
keep_running = True
user = "" # username
pswd = "" # password
headers = [ "Content-Type: text/xml; charset: UTF-8; "]
class MyThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
self.buffer = ""
self.streaming_url = "https://stream.twitter.com/1/statuses/filter.json?track=justinbieber"
def start_crawl(self, user, pswd):
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (user, pswd))
self.conn.setopt(pycurl.URL, self.streaming_url)
#self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.HTTPHEADER, headers)
self.conn.setopt(pycurl.READFUNCTION, self.storage)
self.conn.setopt(pycurl.VERBOSE, 1)
self.conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
self.conn.perform()
def storage(self, data):
self.buffer += str(data)
if isinstance(self.buffer, str):
if self.buffer.endswith("\r\n") and self.buffer.strip():
content = json.loads(self.buffer)
self.buffer = ""
self.queue.put(content)
self.queue.task_done()
#if "text" in content:
#print u"{0[user][name]}: {0[text]}".format(content)
else:
return data
def run(self):
while keep_running:
self.start_crawl(user,pswd)
if signal.signal(signal.SIGINT, signal.SIG_DFL):
sys.exit()
#line = self.names[random.randint(0,len(self.names)-1)]
#queue.put(line)
class Starter():
def __init__(self):
self.queue = queue
t = MyThread(self.queue)
t.daemon = True
t.start()
self.next()
def next(self):
while True:
time.sleep(0.1)
if not self.queue.empty():
line = self.queue.get(timeout=0.2)
print line, self.queue.qsize()
else:
print 'waiting for queue'
def main():
try:
Starter()
queue.join()
except KeyboardInterrupt, e:
print 'Stopping'
raise
main()

Socket in python will only send data it receives

I must be missing something in the code. I've rewritten an 'echo server' example to do a bit more when it receives something.
This is how it currently looks:
#!/usr/bin/env python
import select
import socket
import sys
import threading
import time
import Queue
globuser = {}
queue = Queue.Queue()
class Server:
def __init__(self):
self.host = ''
self.port = 2000
self.backlog = 5
self.size = 1024
self.server = None
self.threads = []
def open_socket(self):
try:
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind((self.host,self.port))
self.server.listen(5)
except socket.error, (value,message):
if self.server:
self.server.close()
print "Could not open socket: " + message
sys.exit(1)
def run(self):
self.open_socket()
input = [self.server,sys.stdin]
running = 1
while running:
inputready,outputready,exceptready = select.select(input,[],[])
for s in inputready:
if s == self.server:
# handle the server socket
c = Client(self.server.accept(), queue)
c.start()
self.threads.append(c)
elif s == sys.stdin:
# handle standard input
junk = sys.stdin.readline()
running = 0
# close all threads
self.server.close()
for c in self.threads:
c.join()
class Client(threading.Thread):
initialized=0
def __init__(self,(client,address), queue):
threading.Thread.__init__(self)
self.client = client
self.address = address
self.size = 1024
self.queue = queue
self.threads = []
global globuser
print 'Client thread created!'
def run(self):
running = 1
while running:
print 'While running client'
data = self.client.recv(self.size)
print 'Dit we receive data?'
if data:
print 'Data received!'
print 'Fetching data from socket: ',
if data[0] == 'I':
print 'Initializing user: ' + data
user = {'uid': data[1:6] ,'x': data[6:9], 'y': data[9:12]}
globuser[user['uid']] = user
print globuser
initialized=1
m=updateClient(user['uid'], queue)
m.start()
self.threads.append(m)
self.client.send('Beginning - Initialized'+';')
elif data[0] == 'A':
print 'Client has received previous data: ' + data
#On deactivation, nothing works.
self.client.send(data+';')
#print 'Client Data sent: ' + data
else:
print 'Closing'
self.client.close()
running = 0
if self.queue.empty():
print 'Queue is empty'
else:
print 'Queue has information: ',
data2 = self.queue.get(1, 1)
isdata2 = 1
if data2 == 'Exit':
running = 0
print 'Client is being closed'
self.client.close()
if isdata2 == 1:
print 'Sending data to client: ' + data2,
self.client.send(data2)
self.queue.task_done()
isdata2 = 0
time.sleep(1)
class updateClient(threading.Thread):
def __init__(self,uid, queue):
threading.Thread.__init__(self)
self.uid = uid
self.queue = queue
global globuser
print 'updateClient thread started!'
def run(self):
running = 20
test=0
while running > 0:
test = test + 1
self.queue.put('Test Queue Data #' + str(test))
running = running - 1
time.sleep(1)
print 'Updateclient has stopped'
if __name__ == "__main__":
s = Server()
s.run()
This runs fine, although it's kind of silly to keep sending the same data back again along with other data.
In the middle of the code you'll see
#On deactivation, nothing works.
self.client.send(data+';')
#print 'Client Data sent: ' + data
When I DO deactive the self.client.send(data+';') or change it into self.client.send('something else;') it does not work! And the client receives nothing.
Is there something special about the "data" variable? Do I need to format the string in some way?

Here's a cleaned-up, functional version of your code! I tested it myself, though I didn't write unit tests.
There were some syntax errors and other miscellaneous problems with the
original code, so I took some liberties. I'm assuming that the protocol is
framed by using ; as a delimiter, since a ; is sent at the end of every
message to the client, though no framing was being done in the original code.
from twisted.internet import reactor, protocol, task
from twisted.protocols import basic
from twisted.python import log
import sys
class ServerProtocol(basic.LineOnlyReceiver):
delimiter = ';'
def lineReceived(self, line):
if line.startswith('I'):
user = dict(uid=line[1:6], x=line[6:9], y=line[9:12])
self.factory.users[user['uid']] = user
log.msg(repr(self.factory.users))
self.startUpdateClient()
self.sendLine('Beginning - Initialized')
elif line.startswith('A'):
self.sendLine(line)
else:
self.transport.loseConnection()
def _updateClient(self):
if self._running == 0:
self._looper.stop()
return
self._running -= 1
self._test += 1
self.sendLine('Test Queue Data #%d' % (self._test,))
def startUpdateClient(self):
self._running, self._test = 20, 0
self._looper = task.LoopingCall(self._updateClient)
self._looper.start(1, now=False)
class Server(protocol.ServerFactory):
protocol = ServerProtocol
def __init__(self):
self.users = {}
if __name__ == '__main__':
log.startLogging(sys.stderr)
reactor.listenTCP(2000, Server())
reactor.run()