We Keep Coding

How to fine tune the redundant code in python

I am trying to create two set of databases every time inside my python script for the same of which I have written the below set of code which looks redundant to me since I am initializing the variable ext 2 times and hence if anyone can suggest some better alternatives, that would be really helpful.
def create_datasets(database, ext):
try:
dataset = "bq --location=US mk -d " + database + ext
try:
return_cd, out, err = run_sys_command(dataset)
except Exception as e:
print(e)
except Exception as e:
print(e)
raise
ext = ''
create_datasets(database, ext)
ext = '_stg'
create_datasets(database, ext)

Use a loop?
for ext in ['', '_stg']:
create_datasets(database, ext)
About your function:
def create_datasets(database, ext):
try:
dataset = f"bq --location=US mk -d {database}{ext}"
return_cd, out, err = run_sys_command(dataset)
except Exception as e: # <- you should catch sub exception!
print(e)

Any exception Exception raised in your function is caught and handled by the inner try block. The outer therefore seems redundant.
def create_datasets(database, ext):
try:
dataset = "bq --location=US mk -d " + database + ext
return_cd, out, err = run_sys_command(dataset)
except Exception as e:
print(e)

Receiving and transmitting data with socket

I'm having trouble with receiving and sending data with Python's socket. In my script I need to listen to incoming data in socket and reading a FIFO file for a response and send it with socket when I find \n. I created separate thread for reading FIFO and it works but sometimes it is really slow. Is it possible to do both things in a main thread? My code:
#!/usr/bin/python
from __future__ import absolute_import, print_function, unicode_literals
from optparse import OptionParser, make_option
import os
import errno
import sys
import socket
import uuid
import dbus
import dbus.service
import dbus.mainloop.glib
import time
from threading import Thread
try:
from gi.repository import GObject
except ImportError:
import gobject as GObject
class ArduinoFifo:
fifofile = -1
OUT_PIPE_FILE = '/tmp/ble_pipe_out'
def removeFile(self, filename):
try:
os.remove(filename)
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
print(e)
raise # re-raise exception if a different error occured
def createFifo(self):
print('removing pipe file\n')
self.removeFile(self.OUT_PIPE_FILE)
print('making pipe\n')
try:
os.mkfifo(self.OUT_PIPE_FILE, 0777)
except OSError as err:
print (err)
raise
def openFifo(self):
print('waiting to open pipe\n')
try:
self.fifofile = os.open(self.OUT_PIPE_FILE, os.O_WRONLY) # | os.O_NONBLOCK)
except OSError as err:
print (err)
def writeFifo(self, data):
try:
if (self.fifofile == -1):
openFifo(self)
os.write(self.fifofile, data)
except OSError as err:
print (err)
class FIFOReader(Thread):
def __init__(self, server_sock):
super(FIFOReader, self).__init__()
self.server_sock = server_sock
self.daemon = True
self.received_msg = ""
self.cancelled = False
print('remove in fifo')
try:
os.remove("/tmp/ble_pipe_in")
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
print(e)
raise
print('create in fifo')
try:
os.mkfifo("/tmp/ble_pipe_in", 0777)
except OSError as err:
print (err)
raise
print('open in fifo')
try:
self.fifofile = os.open("/tmp/ble_pipe_in", os.O_RDWR)
except OSError as err:
print (err)
print('fifo in opened')
def run(self):
while not self.cancelled:
print("READING")
self.received_msg += os.read(self.fifofile, 1)
print("read: %s\n" % self.received_msg)
if "\n" in self.received_msg :
print("Sending Message...")
self.server_sock.send(self.received_msg)
self.received_msg = ""
def cancel(self):
self.cancelled = True
myfifo = ArduinoFifo()
class Profile(dbus.service.Object):
fd = -1
#dbus.service.method("org.bluez.Profile1",
in_signature="", out_signature="")
def Release(self):
print("Release")
mainloop.quit()
#dbus.service.method("org.bluez.Profile1",
in_signature="", out_signature="")
def Cancel(self):
print("Cancel")
#dbus.service.method("org.bluez.Profile1",
in_signature="oha{sv}", out_signature="")
def NewConnection(self, path, fd, properties):
global received_msg
self.fd = fd.take()
print("NewConnection(%s, %d)" % (path, self.fd))
server_sock = socket.fromfd(self.fd, socket.AF_UNIX, socket.SOCK_STREAM)
server_sock.setblocking(1)
myfifo.openFifo()
infifo = FIFOReader(server_sock)
infifo.start()
print('enter recv loop\n')
try:
while True:
data = server_sock.recv(1024)
#print("received: %s" % data)
if data:
myfifo.writeFifo(data)
#if data == "h":
#server_sock.send("Hello!\n")
except IOError as err:
print (err)
pass
server_sock.close()
print("all done")
os.kill(os.getpid(), 9)
#dbus.service.method("org.bluez.Profile1",
in_signature="o", out_signature="")
def RequestDisconnection(self, path):
print("RequestDisconnection(%s)" % (path))
if (self.fd > 0):
os.close(self.fd)
self.fd = -1
if __name__ == '__main__':
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez",
"/org/bluez"), "org.bluez.ProfileManager1")
option_list = [
make_option("-C", "--channel", action="store",
type="int", dest="channel",
default=None),
]
parser = OptionParser(option_list=option_list)
(options, args) = parser.parse_args()
options.uuid = "1101"
options.psm = "3"
options.role = "server"
options.name = "Edison SPP Loopback"
options.service = "spp char loopback"
options.path = "/foo/bar/profile"
options.auto_connect = False
options.record = ""
profile = Profile(bus, options.path)
mainloop = GObject.MainLoop()
opts = {
"AutoConnect" : options.auto_connect,
}
if (options.name):
opts["Name"] = options.name
if (options.role):
opts["Role"] = options.role
if (options.psm is not None):
opts["PSM"] = dbus.UInt16(options.psm)
if (options.channel is not None):
opts["Channel"] = dbus.UInt16(options.channel)
if (options.record):
opts["ServiceRecord"] = options.record
if (options.service):
opts["Service"] = options.service
if not options.uuid:
options.uuid = str(uuid.uuid4())
manager.RegisterProfile(options.path, options.uuid, opts)
myfifo.createFifo()
mainloop.run()
EDIT: I think the problem is in writing data to FIFO or receiving incoming data from socket because in my code in C I've got this delay when I want to read a data from the input FIFO using fgets function.
EIDT2: I use this to instantly receive a message and sends a response one after another

I doubt the issue has to do with the separate thread. "Threads" in Python aren't necessarily OS-level threads but could just be operations the main OS-level thread processes asynchronously. In cPython, which most people use, this is how they work. But I do see a couple of possible issues:
I'm not familiar with some of these libs, but os.read(self.fifofile, 1) stands out. If you use the builtin open() (not the one in os) or BufferedReader, this would be buffered and therefore ok. But os.open is a low-level call that doesn't buffer reads or writes, so you're actually reading 1 byte at a time from the file handle this way, which isn't a good idea as it can cause slowdowns for a variety of hard-to-trace reasons. You should either use a higher level library for this or do the buffering yourself.
Secondly, your +='ing of the read input to the message string repeatedly is going to be slow if your Python interpreter is creating a new string internally each time. So you could be looking at O(N^2) where N is message size time complexity for something that should be O(N). It depends on your interpreter, so to make things portable, you should be appending to a list instead.
Unrelated, but if you don't know whether your FIFO file is text, you shouldn't open it in text mode or else you'll run into errors. Strings only allow valid text bytes, UTF-8 if it's Py3 and I think ASCII if it's Py2, and you'll get an error if you receive, say, 0x00.
Hope this helps.

Exception not caught in multiprocessing

I'm using multiprocessing module for files processing in parallel, which works perfectly fine almost every time.
Also I've written that in try , except block to catch any exception.
I've come across a situation where except block doesn't catch the exception.
Since the code is huge I'm just putting relevant block which is giving problem.
def reader(que, ip, start, end, filename):
""" Reader function checks each line of the file
and if the line contains any of the ip addresses which are
being scanned, then it writes to its buffer.
If the line field doesn't match date string it skips the line.
"""
logging.info("Processing : %s" % os.path.basename(filename))
ip_pat = re.compile("(\d+\.\d+\.\d+\.\d+\:\d+)")
chunk = 10000000 # taking chunk of 10MB data
buff = ""
with bz2.BZ2File(filename,"rb", chunk) as fh: # open the compressed file
for line in fh:
output = []
fields = line.split()
try:
ts = fields[1].strip() + "/" +fields[0]+"/"+fields[3].split("-")[0]+" "+fields[2]
times = da.datetime.strptime(ts,"%d/%b/%Y %H:%M:%S")
if times < start:
continue
if times > end:
break
ips = re.findall(ip_pat,line)
if len(ips) < 3:
continue
if ips[0].split(":")[0] == ip:
output.append(times.strftime("%d/%m/%Y %H:%M:%S"))
status = "SESSION_OPEN" if "SESSION_OPEN" in line or "CREATE" in line else "SESSION_CLOSE"
protocol = "TCP" if "TCP" in line else "UDP"
output.append(status)
output.append(protocol)
ips[1], ips[2] = ips[2], ips[1]
output.extend(ips)
res = "|".join(output)
buff += res + "\n"
except IndexError, ValueError:
continue
logging.info("Processed : %s of size [ %d ]" % (os.path.basename(filename), os.path.getsize(filename)))
if buff:
que.put((ip,buff))
return buff
And this is what is received as error.
File "/usr/lib64/python2.7/multiprocessing/pool.py", line 554, in get
raise self._value
ValueError: time data '2/Dec/20 10:59:59' does not match format '%d/%b/%Y %H:%M:%S'
What I don't understand is why the exception is not caught, I've mentioned ValueError in except block.
What's the best way to get through this problem.

Provide the multiple exceptions as a tuple:
except (IndexError, ValueError):
continue
The relevant doc is https://docs.python.org/2/tutorial/errors.html#handling-exceptions
Snippet from the page:
Note that the parentheses around this tuple are required, because except ValueError, e: was the syntax used for what is normally written as except ValueError as e: in modern Python (described below). The old syntax is still supported for backwards compatibility. This means except RuntimeError, TypeError is not equivalent to except (RuntimeError, TypeError): but to except RuntimeError as TypeError: which is not what you want.

Python 'with' command

Is this code
with open(myfile) as f:
data = f.read()
process(data)
equivalent to this one
try:
f = open(myfile)
data = f.read()
process(f)
finally:
f.close()
or the following one?
f = open(myfile)
try:
data = f.read()
process(f)
finally:
f.close()
This article: http://effbot.org/zone/python-with-statement.htm suggests (if I understand it correctly) that the latter is true. However, the former would make more sense to me. If I am wrong, what am I missing?

According to the documentation:
A new statement is proposed with the syntax:
with EXPR as VAR:
BLOCK
The translation of the above statement is:
mgr = (EXPR)
exit = type(mgr).__exit__ # Not calling it yet
value = type(mgr).__enter__(mgr)
exc = True
try:
try:
VAR = value # Only if "as VAR" is present
BLOCK
except:
# The exceptional case is handled here
exc = False
if not exit(mgr, *sys.exc_info()):
raise
# The exception is swallowed if exit() returns true
finally:
# The normal and non-local-goto cases are handled here
if exc:
exit(mgr, None, None, None)
And this is an extended version of your second code snippet. Initialization goes before try ... finaly block.

It's equivalent to the latter one, because until open() successfully returns, f has no value, and should not be closed.

Using urandom in windows

What happens when you use os.urandom(256) in python in windows ?
The code shows :
def urandom(n):
"""urandom(n) -> str
Return a string of n random bytes suitable for cryptographic use.
"""
try:
_urandomfd = open("/dev/urandom", O_RDONLY)
except (OSError, IOError):
raise NotImplementedError("/dev/urandom (or equivalent) not found")
try:
bs = b""
while n > len(bs):
bs += read(_urandomfd, n - len(bs))
finally:
close(_urandomfd)
return bs
http://docs.python.org/2/library/os.html#os.urandom says when it is run in Windows it automatically uses CryptGenRandom(). I cannot find references to this implementation anywhere.