We Keep Coding

How to create a high load trap receiver?

I need to process a very large number of traps (10,000 per second). I have the simplest linux server. I tried to implement through threads, but cpu clogs up very quickly. Please tell me how to minimize the load on memory and processor, but at the same time process a large number of traps?
There is also work with the database. Writing to the database of taps
from pysnmp.entity import engine, config
from pysnmp.carrier.asyncore.dgram import udp
from pysnmp.entity.rfc3413 import ntfrcv
import psycopg2
from pysnmp.hlapi import SnmpEngine as Sm, CommunityData, UdpTransportTarget,\
ContextData, ObjectType, ObjectIdentity, getCmd
from datetime import datetime
import logging.config
from os import getpid, system, stat, path, chdir, listdir, remove
from threading import Thread
snmpEngine = engine.SnmpEngine()
config.addTransport(
snmpEngine,
udp.domainName + (1,),
udp.UdpTransport().openServerMode(('localhost', 162))
)
config.addV1System(snmpEngine, '', 'public')
class cbFun(Thread):
def __init__(self, snmpEngine, stateReference, contextEngineId, contextName,
varBinds, cbCtx):
Thread.__init__(self)
self.snmpEngine = snmpEngine
self.stateReference = stateReference
self.contextEngineId = contextEngineId
self.contextName = contextName
self.varBinds = varBinds
self.cbCtx = cbCtx
self.localConnected = False
self.localDb = None
self.errorFlag = False
self.start()
def run(self):
print('\n{0}New trap message received on {1} {0}'.format(
'-' * 7,
datetime.now().strftime('%d-%b-%Y at %H:%M:%S')))
execContext = self.snmpEngine.observer.getExecutionContext(
'rfc3412.receiveMessage:request')
print('Trap is coming from %s:%s' % execContext['transportAddress'])
dict_traps = {}
for name, val in self.varBinds:
oid = name.prettyPrint()
value = val.prettyPrint()
print(f'{oid} = {value}')
dict_traps.update({oid: value})
connectDB(dict_traps)
def connectDB(self, values):
connect = psycopg2.connect(dbname="test", user="test",
password="test",
host="test")
cursor = connect.cursor()
for key,value in values:
command = f"insert into TRAPS VALUES ({key}, {value})"
cursor.execute(command)
connect.commit()
connect.close()
ntfrcv.NotificationReceiver(snmpEngine, cbFun)
snmpEngine.transportDispatcher.jobStarted(1)
try:
snmpEngine.transportDispatcher.runDispatcher()
except:
snmpEngine.transportDispatcher.closeDispatcher()
raise

Pysnmp Command Responder with GetNext or Walk operation

I am implementing a Pysnmp responder that receives SNMP GET/SET requests currently and would like to extend with Walk, getNext and getBulk operations. All the OID and its values are stored in key-value in a file.
What I have tried is to use readNextVars() method from instrum.AbstractMibInstrumController class where I iterate over OID's list by calling self.readVars() within readNextVars()
Below is a code snippet showing only GET request and SET request is similar but it writes value to its respective OID in oid.json file.
from pysnmp.entity import engine, config
from pysnmp.entity.rfc3413 import cmdrsp, context
from pysnmp.carrier.asynsock.dgram import udp
from pysnmp.smi import instrum, error
from pysnmp.proto.api import v2c
import json
class SnmpData:
def __init__(self, host, port):
self.snmpEngine = engine.SnmpEngine()
config.addSocketTransport(
self.snmpEngine,
udp.domainName,
udp.UdpTransport().openServerMode((host, port))
)
config.addV1System(self.snmpEngine, 'my-area', 'public', contextName='my-context')
config.addVacmUser(self.snmpEngine, 2, 'my-area', 'noAuthNoPriv', (1, 3, 6), (4, 5, 7))
self.snmpContext = context.SnmpContext(self.snmpEngine)
def snmp_run_command(self):
self.snmpContext.registerContextName(
v2c.OctetString('my-context'),
FileInstrumController()
)
cmdrsp.GetCommandResponder(self.snmpEngine, self.snmpContext)
cmdrsp.SetCommandResponder(self.snmpEngine, self.snmpContext)
cmdrsp.NextCommandResponder(self.snmpEngine, self.snmpContext)
cmdrsp.BulkCommandResponder(self.snmpEngine, self.snmpContext)
self.snmpEngine.transportDispatcher.jobStarted(1)
try:
self.snmpEngine.transportDispatcher.runDispatcher()
except:
self.snmpEngine.transportDispatcher.closeDispatcher()
return "yes"
def main(self):
self.snmp_run_command()
class FileInstrumController(instrum.AbstractMibInstrumController):
def readVars(self, vars, acInfo=(None, None)):
try:
data = None
final_data = None
with open('oid.json') as f:
data = json.load(f)
if str(vars[0][0]) in data.keys())):
final_data = data[str(vars[0][0])]
return [(vars[0][0], v2c.OctetString(str(final_data)))]
else:
return [(vars[0][0], v2c.OctetString(str("Not a Valid OID")))]
except IOError:
raise error.SmiError
def readNextVars(self, vars, acInfo=(None, None))
# get oid & split and match if in file than return its value
# else return invalid oid and break but its not breaking
# and sending continuously requests of next oid (OID + 1)
Here is the oid file (oid.json)
{
"1.3.6.1.1.999.1.1.0": 1,
"1.3.6.1.1.999.1.2.0": 2,
"1.3.6.1.1.999.1.3.0": 3,
"1.3.6.1.1.999.1.4.0": 4,
"1.3.6.1.1.999.1.5.0": 5,
"1.3.6.1.1.999.1.6.0": 100,
"1.3.6.1.1.999.1.7.0": 200,
"1.3.6.1.1.999.1.8.0": 300,
"1.3.6.1.1.999.1.9.0": 400,
"1.3.6.1.1.999.1.10.0": 500
}

Using same data in different states of SMACH Concurrent-container

Let's say we have a concurrent SMACH container sm_con which includes two state machines SM1 and SM2. I need to find a way for SM1 to continuously update some data and for SM2 to access (and eventually also modify) the same data. I thought about solving this by passing the userdata of sm_con to SM1 and SM2 as input- and output-keys hoping that if SM1 modifies the data it would automatically overwrite sm_cons userdata (kind of like working with pointers in c++) but this doesn't work.
The corresponding code would look like this:
import smach
import smach_ros
import rospy
class st1(smach.State):
def __init__(self, outcomes=['successful', 'preempted']):
smach.State.__init__(self, outcomes, input_keys=['in_test'], output_keys=['out_test'])
def execute(self, userdata):
if self.preempt_requested():
self.service_preempt()
return 'preempted'
rospy.logerr('test 1: '+str(userdata.in_test))
userdata.out_test=userdata.in_test+1
return 'successful'
class st2(smach.State):
def __init__(self, outcomes=['successful', 'preempted']):
smach.State.__init__(self, outcomes, input_keys=['in_test'])
def execute(self, userdata):
#time.sleep(2)
if self.preempt_requested():
self.service_preempt()
return 'preempted'
rospy.logerr('test 2: ' + str(userdata.in_test))
return 'successful'
if __name__=="__main__":
rospy.init_node('test_state_machine')
sm_con = smach.Concurrence(outcomes=['success'],
default_outcome='success'
)
sm_con.userdata.testdata = 0
with sm_con:
sm_1 = smach.StateMachine(outcomes=['success', 'preempted'], input_keys=['testdata'], output_keys=['testdata'])
with sm_1:
smach.StateMachine.add('ST1', st1(),
remapping={'in_test': 'testdata', 'out_test': 'testdata'},
transitions={'successful': 'ST1'})
sm_2 = smach.StateMachine(outcomes=['success', 'preempted'], input_keys=['testdata'])
with sm_2:
smach.StateMachine.add('ST2', st2(),
remapping={'in_test':'testdata'},
transitions={'successful': 'ST2'})
smach.Concurrence.add('SM1', sm_1)
smach.Concurrence.add('SM2', sm_2)
# Execute SMACH plan
outcome = sm_con.execute()
print('exit-outcome:' + outcome)
# Wait for ctrl-c to stop the application
rospy.spin()
Running this code, the output 'test 1: ...' shows that inside SM1 the userdata gets incremented while the output 'test 2: ...' shows that SM2 doesn't access the incremented data as the output remains 0.
How can I modify some data in SM1 and access the modified data in SM2?

I found a workaround for this using mutable objects like described here.
Applied on the code above, it would look like the following:
import smach
import smach_ros
import rospy
class st1(smach.State):
def __init__(self, outcomes=['successful', 'preempted']):
smach.State.__init__(self, outcomes, input_keys=['in_test'])
def execute(self, userdata):
if self.preempt_requested():
self.service_preempt()
return 'preempted'
rospy.logerr('test 1: '+str(userdata.in_test))
userdata.in_test[0]=userdata.in_test[0]+1
return 'successful'
class st2(smach.State):
def __init__(self, outcomes=['successful', 'preempted']):
smach.State.__init__(self, outcomes, input_keys=['in_test'])
def execute(self, userdata):
#time.sleep(2)
if self.preempt_requested():
self.service_preempt()
return 'preempted'
rospy.logerr('test 2: ' + str(userdata.in_test[0]))
return 'successful'
if __name__=="__main__":
rospy.init_node('test_state_machine')
sm_con = smach.Concurrence(outcomes=['success'],
default_outcome='success'
)
sm_con.userdata.testdata = [0]
with sm_con:
sm_1 = smach.StateMachine(outcomes=['success', 'preempted'], input_keys=['testdata'])
with sm_1:
smach.StateMachine.add('ST1', st1(),
remapping={'in_test': 'testdata'},
transitions={'successful': 'ST1'})
sm_2 = smach.StateMachine(outcomes=['success', 'preempted'], input_keys=['testdata'])
with sm_2:
smach.StateMachine.add('ST2', st2(),
remapping={'in_test':'testdata'},
transitions={'successful': 'ST2'})
smach.Concurrence.add('SM1', sm_1)
smach.Concurrence.add('SM2', sm_2)
# Execute SMACH plan
outcome = sm_con.execute()
print('exit-outcome:' + outcome)
# Wait for ctrl-c to stop the application
rospy.spin()
Since this is only a workaround, refer to my corresponding issue-post here for more information.

Serial Communication breaks down when threading

I am using a USB microwave source, which communicates via a virtual COM port.
Communication is done via python. Everything works fine, as long as I am executing the code blockingly.
However, as soon as any communication is done in a thread
I get a SerialException: Attempting to use a port that is not open. Is there an obvious reason, why this is happening? Nothing else, at least originating from my software, is trying to communicate with the port during that time.
The highest level script:
from serial import SerialException
import deer
import windfreak_usb
try:
deer.mw_targets = windfreak_usb.WindfreakSynthesizer()
except SerialException:
deer.mw_targets.port.close()
deer.mw_targets = windfreak_usb.WindfreakSynthesizer()
deer_mes = deer.DeerMeasurement(f_start=0.9e9,
f_end=1.2e9,
df=3e6,
f_nv=1.704e9,
seq=["[(['mw'],28),([],tau),(['mw', 'mwy'],56),([],tau),(['mw'],28),([],100)]",
"[(['mw'],28),([],tau),(['mw', 'mwy'],56),([],tau),(['mw'],84),([],100)]"],
power_nv=10,
power_targets=3,
tau=700
)
deer_mes.run(10e6) # <- this works perfectly, as it is the blocking version
# deer_mes.start(10e6) # <- raises the SerialException at the line indicated below
deer.mw_targets.port.close()
A reduced form of the microwave source (windfreak_usb.py):
import serial
import synthesizer
class WindfreakSynthesizer(synthesizer.Synthesizer):
def __init__(self):
synthesizer.Synthesizer.__init__(self)
self.port = serial.Serial(
port='COM14',
baudrate=9600,
timeout=10
)
self.off()
def __del__(self):
self.port.close()
def off(self):
self.port.write('o0')
def power(self, p):
p = int(p)
self.port.write('a{}'.format(p))
A reduced form of the measurement class (deer.py):
import threading
import time
import numpy
import os
from PyQt4 import QtCore
from PyQt4.QtCore import QObject
import matplotlib.pyplot as plt
import hardware
import matpickle
import pulsed
import pulser
import savepath
import synthesizer
if 'pg' not in globals():
pg = pulser.Pulser()
if 'mw_targets' not in globals():
mw_targets = synthesizer.Synthesizer()
timeout = 30
CurrentMeasurement = None # global variable pointing to the currently active measurement
class DeerMeasurement(QObject):
update = QtCore.pyqtSignal()
def __init__(self, f_start, f_end, df, f_nv, seq, power_nv, power_targets, tau, sweeps_per_iteration=50e3,
switching_time=300e-6):
super(QObject, self).__init__()
""" setting all parameters as self.parameter """
self.power_targets = power_targets
self.fs = numpy.arange(f_start, f_end + df, df)
self.abort = threading.Event()
self.save_deer()
def run(self, sweeps):
global CurrentMeasurement
if CurrentMeasurement is not None:
print('Deer Warning: cannot start measurement while another one is already running. Returning...')
return
CurrentMeasurement = self
# Start measurement
print('Deer measurement started.')
mw_targets.power(self.power_targets) # <- This causes the SerialException.
""" Here comes the actual measurement, that is never executed, as the line above kills the thread already with the SerialException. """
def start(self, sweeps, monitor=None):
"""Start Measurement in a thread."""
if monitor is not None:
monitor.register(self)
if not hasattr(self, 'mes_thread'):
# noinspection PyAttributeOutsideInit
self.mes_thread = threading.Thread(target=self.run, args=(sweeps,))
self.mes_thread.start()
else:
print('Already threading')
Any help is highly appreciated, as running the measurement outside a thread is not an option.
Best regards!

Non blocking python sockets

I'd like to write a small Bluetooth server application to my Nokia phone in PyS60. It needs to be able to send response to the client's request and be able to push data to the client as well.
option 1:
if I use socket.recv(1024), the program waits until something is received, therefore the server can't push data to the client. The Python for S60 implementation is missing the socket.settimeout() method, so I couldn't write a proper non-blocking code.
oprion 2:
The socket.makefile() approach was looking good, but couldn't make it work. When I replaced the conn.recv(1024) to fd = socket.makefile() fd.readline(), it didn't read a thing.
option 3:
Looked into the select() function, but had no luck with it. When I changed the conn.recv() to the r,w,e = select.select([conn],[],[]) like it's been suggested the client doesn't even connect. It hangs at "Waiting for the client...". Strange...
I know that there are pretty nice server implementations and asynchronous API-s as well, but I only need a really basic stuff here. Thanks in advance!
here's what I have:
sock = btsocket.socket(btsocket.AF_BT, btsocket.SOCK_STREAM)
channel = btsocket.bt_rfcomm_get_available_server_channel(sock)
sock.bind(("", channel))
sock.listen(1)
btsocket.bt_advertise_service(u"name", sock, True, btsocket.RFCOMM)
print "Waiting for the client..."
conn, client_mac = sock.accept()
print "connected: " + client_mac
while True:
try:
data = conn.recv(1024)
if len(data) != 0:
print "received [%s]" % data
if data.startswith("something"): conn.send("something\r\n")
else:
conn.send("some other data \r\n")
except:
pass
It's obviously blocking, so the "some other data" is never sent, but it's the best I've got so far. At least I can send something in reply to the client.

Found the solution finally!
The select function wasn't working with the btsocket module of the newer PyS60 ports.
Someone wrote a new_btsocket (available here) with a working select function.

Here is a simple example based on an echo server
#!/usr/bin/python
import socket
import select
server = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
server.bind( ('localhost', 12556) )
server.listen( 5 )
toread = [server]
running = 1
# we will shut down when all clients disconenct
while running:
rready,wready,err = select.select( toread, [], [] )
for s in rready:
if s == server:
# accepting the socket, which the OS passes off to another
# socket so we can go back to selecting. We'll append this
# new socket to the read list we select on next pass
client, address = server.accept()
toread.append( client ) # select on this socket next time
else:
# Not the server's socket, so we'll read
data = s.recv( 1024 )
if data:
print "Received %s" % ( data )
else:
print "Client disconnected"
s.close()
# remove socket so we don't watch an invalid
# descriptor, decrement client count
toread.remove( s )
running = len(toread) - 1
# clean up
server.close()
That said, I still find socketserver cleaner and easier. Implement handle_request and call serve_forever

Here's an Epoll Server Implementation (non-blocking)
http://pastebin.com/vP6KPTwH (same thing as below, felt this might be easier to copy)
use python epollserver.py to start the server.
Test it using wget localhost:8888
import sys
import socket, select
import fcntl
import email.parser
import StringIO
import datetime
"""
See:
http://docs.python.org/library/socket.html
"""
__author__ = ['Caleb Burns', 'Ben DeMott']
def main(argv=None):
EOL1 = '\n\n'
EOL2 = '\n\r\n'
response = 'HTTP/1.0 200 OK\r\nDate: Mon, 1 Jan 1996 01:01:01 GMT\r\n'
response += 'Content-Type: text/plain\r\nContent-Length: 13\r\n\r\n'
response += 'Hello, world!'
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Tell the server socket file descriptor to destroy itself when this program ends.
socketFlags = fcntl.fcntl(serversocket.fileno(), fcntl.F_GETFD)
socketFlags |= fcntl.FD_CLOEXEC
fcntl.fcntl(serversocket.fileno(), fcntl.F_SETFD, socketFlags)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversocket.bind(('0.0.0.0', 8888))
serversocket.listen(1)
# Use asynchronous sockets.
serversocket.setblocking(0)
# Allow a queue of up to 128 requests (connections).
serversocket.listen(128)
# Listen to socket events on the server socket defined by the above bind() call.
epoll = select.epoll()
epoll.register(serversocket.fileno(), select.EPOLLIN)
print "Epoll Server Started..."
try:
#The connection dictionary maps file descriptors (integers) to their corresponding network connection objects.
connections = {}
requests = {}
responses = {}
while True:
# Ask epoll if any sockets have events and wait up to 1 second if no events are present.
events = epoll.poll(1)
# fileno is a file desctiptor.
# event is the event code (type).
for fileno, event in events:
# Check for a read event on the socket because a new connection may be present.
if fileno == serversocket.fileno():
# connection is a new socket object.
# address is client IP address. The format of address depends on the address family of the socket (i.e., AF_INET).
connection, address = serversocket.accept()
# Set new socket-connection to non-blocking mode.
connection.setblocking(0)
# Listen for read events on the new socket-connection.
epoll.register(connection.fileno(), select.EPOLLIN)
connections[connection.fileno()] = connection
requests[connection.fileno()] = b''
responses[connection.fileno()] = response
# If a read event occured, then read the new data sent from the client.
elif event & select.EPOLLIN:
requests[fileno] += connections[fileno].recv(1024)
# Once we're done reading, stop listening for read events and start listening for EPOLLOUT events (this will tell us when we can start sending data back to the client).
if EOL1 in requests[fileno] or EOL2 in requests[fileno]:
epoll.modify(fileno, select.EPOLLOUT)
# Print request data to the console.
epoll.modify(fileno, select.EPOLLOUT)
data = requests[fileno]
eol = data.find("\r\n") #this is the end of the FIRST line
start_line = data[:eol] #get the contents of the first line (which is the protocol information)
# method is POST|GET, etc
method, uri, http_version = start_line.split(" ")
# re-used facebooks httputil library (works well to normalize and parse headers)
headers = HTTPHeaders.parse(data[eol:])
print "\nCLIENT: FD:%s %s: '%s' %s" % (fileno, method, uri, datetime.datetime.now())
# If the client is ready to receive data, sent it out response.
elif event & select.EPOLLOUT:
# Send response a single bit at a time until the complete response is sent.
# NOTE: This is where we are going to use sendfile().
byteswritten = connections[fileno].send(responses[fileno])
responses[fileno] = responses[fileno][byteswritten:]
if len(responses[fileno]) == 0:
# Tell the socket we are no longer interested in read/write events.
epoll.modify(fileno, 0)
# Tell the client we are done sending data and it can close the connection. (good form)
connections[fileno].shutdown(socket.SHUT_RDWR)
# EPOLLHUP (hang-up) events mean the client has disconnected so clean-up/close the socket.
elif event & select.EPOLLHUP:
epoll.unregister(fileno)
connections[fileno].close()
del connections[fileno]
finally:
# Close remaining open socket upon program completion.
epoll.unregister(serversocket.fileno())
epoll.close()
serversocket.close()
#!/usr/bin/env python
#
# Copyright 2009 Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""HTTP utility code shared by clients and servers."""
class HTTPHeaders(dict):
"""A dictionary that maintains Http-Header-Case for all keys.
Supports multiple values per key via a pair of new methods,
add() and get_list(). The regular dictionary interface returns a single
value per key, with multiple values joined by a comma.
>>> h = HTTPHeaders({"content-type": "text/html"})
>>> h.keys()
['Content-Type']
>>> h["Content-Type"]
'text/html'
>>> h.add("Set-Cookie", "A=B")
>>> h.add("Set-Cookie", "C=D")
>>> h["set-cookie"]
'A=B,C=D'
>>> h.get_list("set-cookie")
['A=B', 'C=D']
>>> for (k,v) in sorted(h.get_all()):
... print '%s: %s' % (k,v)
...
Content-Type: text/html
Set-Cookie: A=B
Set-Cookie: C=D
"""
def __init__(self, *args, **kwargs):
# Don't pass args or kwargs to dict.__init__, as it will bypass
# our __setitem__
dict.__init__(self)
self._as_list = {}
self.update(*args, **kwargs)
# new public methods
def add(self, name, value):
"""Adds a new value for the given key."""
norm_name = HTTPHeaders._normalize_name(name)
if norm_name in self:
# bypass our override of __setitem__ since it modifies _as_list
dict.__setitem__(self, norm_name, self[norm_name] + ',' + value)
self._as_list[norm_name].append(value)
else:
self[norm_name] = value
def get_list(self, name):
"""Returns all values for the given header as a list."""
norm_name = HTTPHeaders._normalize_name(name)
return self._as_list.get(norm_name, [])
def get_all(self):
"""Returns an iterable of all (name, value) pairs.
If a header has multiple values, multiple pairs will be
returned with the same name.
"""
for name, list in self._as_list.iteritems():
for value in list:
yield (name, value)
def items(self):
return [{key: value[0]} for key, value in self._as_list.iteritems()]
def get_content_type(self):
return dict.get(self, HTTPHeaders._normalize_name('content-type'), None)
def parse_line(self, line):
"""Updates the dictionary with a single header line.
>>> h = HTTPHeaders()
>>> h.parse_line("Content-Type: text/html")
>>> h.get('content-type')
'text/html'
"""
name, value = line.split(":", 1)
self.add(name, value.strip())
#classmethod
def parse(cls, headers):
"""Returns a dictionary from HTTP header text.
>>> h = HTTPHeaders.parse("Content-Type: text/html\\r\\nContent-Length: 42\\r\\n")
>>> sorted(h.iteritems())
[('Content-Length', '42'), ('Content-Type', 'text/html')]
"""
h = cls()
for line in headers.splitlines():
if line:
h.parse_line(line)
return h
# dict implementation overrides
def __setitem__(self, name, value):
norm_name = HTTPHeaders._normalize_name(name)
dict.__setitem__(self, norm_name, value)
self._as_list[norm_name] = [value]
def __getitem__(self, name):
return dict.__getitem__(self, HTTPHeaders._normalize_name(name))
def __delitem__(self, name):
norm_name = HTTPHeaders._normalize_name(name)
dict.__delitem__(self, norm_name)
del self._as_list[norm_name]
def get(self, name, default=None):
return dict.get(self, HTTPHeaders._normalize_name(name), default)
def update(self, *args, **kwargs):
# dict.update bypasses our __setitem__
for k, v in dict(*args, **kwargs).iteritems():
self[k] = v
#staticmethod
def _normalize_name(name):
"""Converts a name to Http-Header-Case.
>>> HTTPHeaders._normalize_name("coNtent-TYPE")
'Content-Type'
"""
return "-".join([w.capitalize() for w in name.split("-")])
if(__name__ == '__main__'):
sys.exit(main(sys.argv))