How to mock a TCPSocket wrapper for the socket from the Python's standard libary via the mock library (unittest.mock in case of Python 3)?
This is my wrapper:
import socket
import utils
class TCPSocket:
def __init__(self):
self.buf = ''
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def connect(self, host, port):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((host, port))
def close(self):
self.sock.close()
def send(self, data):
self.sock.send(data)
def sendall(self, data):
self.sock.sendall(data)
# For best match with hardware and network realities,
# the value of limit should be a relatively small power of 2, for example, 4096
def recv_some(self, limit=4096):
return self.sock.recv(limit)
def recv_bytes(self, count):
while len(self.buf) < count:
self.buf += self.recv_some()
parts = utils.split_first_n(self.buf, count)
self.buf = parts[1]
return parts[0]
def recv_until(self, delim):
while delim not in self.buf:
self.buf += self.recv_some()
parts = self.buf.split(delim, maxsplit=1)
self.buf = parts[1]
return parts[0]
I want to test whether functions like recv_until and recv_bytes do what they really need.
It can be as simple as
import mock # or from unittest import mock
mock_socket = mock.Mock()
mock_socket.recv.return_value = data
then use mock_socket where you would use the real socket. You can also mock whatever creates the socket to return a mock value like the one configured here, depending on your needs.
For your case, you can mock socket.socket so that it returns something whose method you can configure. Note that mock_socket in this example is a function that returns a Socket object, not a Socket object itself.
with mock.patch('socket.socket') as mock_socket:
mock_socket.return_value.recv.return_value = some_data
t = TCPSocket()
t.connect('example.com', 12345) # t.sock is a mock object, not a Socket
self.assertEqual(t.recv_bytes(), whatever_you_expect)
t.sock.connect.assert_called_with(('example.com', 12345))
Related
I'm having a strange phenomena in Python with callback functions and handlers.
I use ZMQ to handle my communication and use a stream for the socket. I have the base class:
import multiprocessing
import zmq
from concurrent.futures import ThreadPoolExecutor
from zmq.eventloop import ioloop, zmqstream
from zmq.utils import jsonapi as json
# Types of messages
TYPE_A = 'type_a'
TYPE_B = 'type_b'
class ZmqProcess(multiprocessing.Process):
def __init__(self):
super(ZmqProcess, self).__init__()
self.context = None
self.loop = None
self.handle_stream = None
def setup(self):
self.context = zmq.Context()
self.loop = ioloop.IOLoop.instance()
def send(self, msg_type, msg, host, port):
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://%s:%s' % (host, port))
sock.send_json([msg_type, msg])
def stream(self, sock_type, addr):
sock = self.context.socket(sock_type)
if isinstance(addr, str):
addr = addr.split(':')
host, port = addr if len(addr) == 2 else (addr[0], None)
if port:
sock.bind('tcp://%s:%s' % (host, port))
else:
port = sock.bind_to_random_port('tcp://%s' % host)
stream = zmqstream.ZMQStream(sock, self.loop)
return stream, int(port)
class MessageHandler(object):
def __init__(self, json_load=-1):
self._json_load = json_load
self.pool = ThreadPoolExecutor(max_workers=10)
def __call__(self, msg):
i = self._json_load
msg_type, data = json.loads(msg[i])
msg[i] = data
if msg_type.startswith('_'):
raise AttributeError('%s starts with an "_"' % msg_type)
getattr(self, msg_type)(*msg)
And I have a class that inherits from it:
import zmq
import zmq_base
class ZmqServerMeta(zmq_base.ZmqProcess):
def __init__(self, bind_addr, handlers):
super(ZmqServerMeta, self).__init__()
self.bind_addr = bind_addr
self.handlers = handlers
def setup(self):
super(ZmqServerMeta, self).setup()
self.handle_stream, _ = self.stream(zmq.PAIR, self.bind_addr)
self.handle_stream.on_recv(StreamHandler(self.handle_stream, self.stop,
self.handlers))
def run(self):
self.setup()
self.loop.start()
def stop(self):
self.loop.stop()
class StreamHandler(zmq_base.MessageHandler):
def __init__(self, handle_stream, stop, handlers):
super(StreamHandler, self).__init__()
self._handle_stream = handle_stream
self._stop = stop
self._handlers = handlers
def type_a(self, data):
if zmq_base.TYPE_A in self._handlers:
if self._handlers[zmq_base.TYPE_A]:
for handle in self._handlers[zmq_base.TYPE_A]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def type_b(self, data):
if zmq_base.TYPE_B in self._handlers:
if self._handlers[zmq_base.TYPE_B]:
for handle in self._handlers[zmq_base.TYPE_B]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def endit(self):
self._stop()
Additionally, I have a class that I want to use as storage. And here is where the trouble starts:
import threading
import zmq_server_meta as server
import zmq_base as base
class Storage:
def __init__(self):
self.list = []
self.list_lock = threading.RLock()
self.zmq_server = None
self.host = '127.0.0.1'
self.port = 5432
self.bind_addr = (self.host, self.port)
def setup(self):
handlers = {base.TYPE_A: [self. remove]}
self.zmq_server = server.ZmqServerMeta(handlers=handlers, bind_addr=self.bind_addr)
self.zmq_server.start()
def add(self, data):
with self.list_lock:
try:
self.list.append(data)
except:
print "Didn't work"
def remove(self, msg):
with self.list_lock:
try:
self.list.remove(msg)
except:
print "Didn't work"
The idea is that that class stores some global information that it receives.
It is all started in a file to test:
import sys
import time
import storage
import zmq_base as base
import zmq_server_meta as server
def printMsg(msg):
print msg
store = storage.Storage()
store.setup()
handlers = {base.TYPE_B: [printMsg]}
client = server.ZmqServerMeta(handlers=handlers, bind_addr=('127.0.0.1', 5431))
client.start()
message = "Test"
store.add(message)
client.send(base.TYPE_A, message, '127.0.0.1', 5432)
I simplified it to reduce clutter. Instead of just adding it, it is usually send and then a response comes back. The response, the client sending, should be processed by the correct callback, remove(), and it should remove something out of the list. The problem that occurs is, that the remove() function sees an empty list, although there should be an element in the list. If I check from the testing file, I can see the element after it was added, and if I call remove() from there, I see a non-empty list and can remove it. My question is, why does the callback sees an empty list and how can I make sure it does see the correct elements in the list?
Kind regards
Patrick
I believe the problem lays in the fact that the ZmqProcess class inherits from multiprocessing.Process. Multiprocessing does not allow to share objects among different processes, except by using a shared memory map using Value or Array ( as can be seen in the documentation: https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes )
If you want to use your custom object, you can use a Server process / proxy object, which can be found in on the same page of the documentation.
So you can, for instance, define a manager in the init function of the Storage class like: self.manager = Manager() Afterwards you put self.list = self.manager.list(). This should do the trick.
I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?
You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.
i am trying to receive about 1000 connections to my server but it cannot receive more than 512. What can i do to increase the amount of open connections? I am running windows 8.1
Not: I am very new to this stuff so, thanks for help
Here is my code;
import asyncore
import socket
import uuid
import time
import threading
class statistics(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
entry = raw_input("")
zaman = int(time.time())
cmd = receivedCmd
print "calculating.."
time.sleep(1)
if entry == 'istatistik':
print str(receivedCmd-cmd) + " command/second"
print "total received commands: " + str(receivedCmd)
entry = ""
class tcpClient:
def __init__(self):
self.clientid = uuid.uuid1(int(time.time()))
self.buffer = ""
self.buffer_size = 0
self.conn_time = time.time()
self.overflow = 0
#print str(self.clientid) + " assingned"
def recv_msg(self, msg):
global receivedCmd
self.buffer = msg
self.buffer_size = len(self.buffer)
receivedCmd = receivedCmd + 1
if self.buffer_size >= 1024:
self.overflow = 1
def __del__(self):
print str(self.clientid) + " has left."
class TCPHandler(asyncore.dispatcher_with_send):
global clist
def handle_read(self):
data = self.recv(1024)
if data:
if clist[self].overflow:
self.send("overflow")
self.handle_close()
else:
self.send(data)
clist[self].recv_msg(data)
def handle_close(self):
del clist[self]
self.close()
def handle_error(self):
del clist[self]
self.close()
class TCPServer(asyncore.dispatcher):
global clist
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
self.clist = clist
pair = self.accept()
if pair is None:
pass
else:
sock, addr = pair
#print 'Connection : %s' % repr(addr)
clist[TCPHandler(sock)] = tcpClient()
if __name__ == '__main__':
clist = {}
receivedCmd = 0
server = TCPServer('', 5000)
server2 = TCPServer('',5001)
StaticsThread = statistics()
StaticsThread.start()
asyncore.loop()
Note: I still cannot receive more than 512 connections with the Twisted Framework, i don't know what to do. There have to be thousands of connected clients. Please help.
The asyncore module relies in the select OS function, which only supports a limited number of file descriptors.
As an alternative use a multi-threading server (I won't recommend this) or, better, the Twisted framework which is event-driven (highly recommended!).
Hope this helps!
Since Twisted's default reactor under Windows is also select-based then you should consider using the IOCP reactor instead.
from twisted.internet import iocpreactor
iocpreactor.install()
from twisted.internet import reactor
But also take into account that Twisted prefers Linux systems (where the default reactor is epoll-based) rather than Windows. Maybe switching to Linux is a better choice.
My Problem looks like this:
I have a Twisted Server, which sends values every 2 seconds over TCP with callLater. And I have a twisted client, which recieves the values and should handle the recieving value with a deferred.
My Server looks like this:
from twisted.internet.protocol import ServerFactory
from twisted.protocols.basic import NetstringReceiver
from random import randint
class ServerProtocol(NetstringReceiver):
def __init__(self, factory):
self.factory = factory
self.sendObject = None
def connectionMade(self):
self.sendValue()
def connectionLost(self, reason):
sendObject, self.sendObject = self.sendObject, None
sendObject.cancel()
def sendValue(self):
data = randint(2,20)
self.sendString(str(data))
print('send: {0}'.format(data))
from twisted.internet import reactor
self.sendObject = reactor.callLater(2, self.sendValue)
class MyServerFactory(ServerFactory):
def __init__(self):
self.protsa = []
def buildProtocol(self, addr):
return ServerProtocol(self)
def setCallback(self, callback):
self.callback = callback
def serverMain():
factory = MyServerFactory()
# factory.setCallback(generateVal)
from twisted.internet import reactor
port = reactor.listenTCP(2345, factory, interface='127.0.0.1')
print 'Serving on %s.' % (port.getHost())
reactor.run()
if __name__ == '__main__':
serverMain()
My Client looks like this:
from twisted.internet.protocol import ClientFactory
from twisted.protocols.basic import NetstringReceiver
from twisted.internet import defer
class ClientProtocol(NetstringReceiver):
def stringReceived(self, string):
print("recieved")
self.factory.printValue(string)
def connectionMade(self):
print("Made Connection")
def connetionLost(self):
print("Connection Lost")
class MyClientFactory(ClientFactory):
protocol = ClientProtocol
def __init__(self, deferred):
self.deferred = deferred
def clientConnectionFailed(self, connector, reason):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
def printValue(self, value):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.callback(value)
def OutputValue(host, port):
d = defer.Deferred()
from twisted.internet import reactor
factory = MyClientFactory(d)
reactor.connectTCP(host, port, factory)
return d
def clientMain():
def writeError(err):
print("Deferred Error!\n")
print("Error: {0}".format(err.__str__))
def writeValue(value):
print("Value revieved: {0}".format(value))
from twisted.internet import reactor
d = OutputValue('127.0.0.1', 2345)
d.addCallbacks(writeValue, writeError)
reactor.run()
if __name__ == '__main__':
clientMain()
Output:
Made Connection
recieved
Value revieved: 11
recieved
recieved
recieved
For the first time everything works fine. After connection the server sends a value which the client recieves. The client handels the with a deferred as wished. The second value are not handled by the deferred. I also expected that because the deferred are only fired once.
Now I would like to have a possibility to handle every recieved value by a deferred, so I can also do error handling.
I had a look at this:
Twisted: deferred that fires repeatedly
Re-using deferred objects in Twisted
but I cannot get hold of a solution for my problem. This can't be so unusual.
Or do I have to disconnect and connect every time?
A Deferred is an object which represents the result of a specific request that you are making. But since a request can only have one associated response, in the same way that a function can only return once, a Deferred can only be fired once.
You already have a function - printValue - which is run when a value is received and should be printed. Why not just print the value in that function right away, rather than going looking for a Deferred which your caller set up?
As we can see, send method is not overloaded.
from socket import socket
class PolySocket(socket):
def __init__(self,*p):
print "PolySocket init"
socket.__init__(self,*p)
def sendall(self,*p):
print "PolySocket sendall"
return socket.sendall(self,*p)
def send(self,*p):
print "PolySocket send"
return socket.send(self,*p)
def connect(self,*p):
print "connecting..."
socket.connect(self,*p)
print "connected"
HOST="stackoverflow.com"
PORT=80
readbuffer=""
s=PolySocket()
s.connect((HOST, PORT))
s.send("a")
s.sendall("a")
Output:
PolySocket init
connecting...
connected
PolySocket sendall
I am sure you don't actually need it and there are other ways to solve your task (not subclassing but the real task).
If you really need to mock object, go with proxy object:
from socket import socket
class PolySocket(object):
def __init__(self, *p):
print "PolySocket init"
self._sock = socket(*p)
def __getattr__(self, name):
return getattr(self._sock, name)
def sendall(self, *p):
print "PolySocket sendall"
return self._sock.sendall(*p)
def send(self, *p):
print "PolySocket send"
return self._sock.send(*p)
def connect(self, *p):
print "connecting..."
self._sock.connect(*p)
print "connected"
HOST = "stackoverflow.com"
PORT = 80
readbuffer = ""
s = PolySocket()
s.connect((HOST, PORT))
s.send("a")
s.sendall("a")
Here's the output:
% python foo.py
PolySocket init
connecting...
connected
PolySocket send
PolySocket sendall