We Keep Coding

Attempting secure socket communication between client and server and getting An operation was attempted on something that is not a socket error

I am attempting to piece together a secure socket client server communication solution. I do not have experience in doing so, so have cobbled together what I believe are relevant sections. The idea is that the Server waits for connections, the client creates a connection that is secure and then communication can take place.
The code also utilizes secure communication in authorization with client and server keys and certificates.
client code:
class Client:
def __init__(self):
try:
self.host, self.port = "127.0.0.1", 65416
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self._context = ssl.SSLContext()
self._context.load_cert_chain(self.client_cert, self.client_key)
self._sock = None
self._ssock = None
except Exception as e:
print("Error in Initializing")
def checkvalidclient(self):
# ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock,)
self._ssock.connect((HOST, PORT))
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
Response = self._ssock.recv(1024)
while True:
Input = input('Say Something: ')
# s.send(str.encode(Input))
send_msg(self._ssock, str.encode(Input))
# Response = s.recv(1024)
Response = recv_msg(self._ssock)
if Response is not None:
print(Response.decode('utf-8'))
def closesockconnection(self):
self._ssock.close()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
client = Client()
client.checkvalidclient()
Server code:
import socket
import os
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
try:
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
print("Exception in recvall : " + str(e))
# ---- Server Communication Setup
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_verify_locations(self.client_cert)
self.sock = None
def connect(self):
try: # create socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
print(str(e))
print('Waiting for a Connection..')
self.sock.listen(3)
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
print(data)
if data is not None:
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, str.encode(reply))
#conn.close()
def waitforconnection(self):
while True:
Client, addr = self.sock.accept()
self._context.wrap_socket(Client, server_side=True)
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
start_new_thread(self.threaded_client, (Client, )) # Calling threaded_client() on a new thread
self.ThreadCount += 1
print('Thread Number: ' + str(self.ThreadCount))
#self.sock.close()
server = Server()
server.connect()
server.waitforconnection()
The lines:
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
result in the error:
[WinError 10038] An operation was attempted on something that is not a socket
When I removed the certificate related lines in client:
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self._context = ssl.SSLContext()
self._context.load_cert_chain(self.client_cert, self.client_key)
and the certificate related lines in the server:
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_verify_locations(self.client_cert)
self.sock = None
and a couple of small changes to remove the certificate related functionality, everything seemed to work, the client could send messages to the server and the server could respond (and the client displayed the response).
When however I added the context related certificates I start getting the error:
An operation was attempted on something that is not a socket
The server waits at:
Client, addr = self.sock.accept()
and continues to run once the client has called (in the client.py file):
self._ssock.connect((HOST, PORT))
The server then reaches the lines:
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
where it fails on this error.
Printing the terminal, a traceback and exception error results in:
Socket successfully created
Waiting for a Connection..
Connected to: 127.0.0.1:57434
Thread Number: 1
Traceback (most recent call last):
File "c:\testcode\Server.py", line 71, in threaded_client
conn.send(str.encode('Welcome to the Server'))
OSError: [WinError 10038] An operation was attempted on something that is not a socket
My knowledge is limited and I cannot find more examples of secure multi threaded two way communication client to server socket code. The idea is to ensure the client is authorized to communicate with the server before transmission happens.
Any ideas on where I am failing?
Thanks

Ok, It seems like I was close, but had a couple of tweaks to do.
The solution of:
SSL/TLS client certificate verification with Python v3.4+ SSLContext
and the commenters here, helped me get over the finish line.
Server code:
import socket
import os
from socket import AF_INET, SOCK_STREAM, SO_REUSEADDR, SOL_SOCKET, SHUT_RDWR
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
import traceback
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
try:
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
print("Exception in recvall : " + str(e))
# ---- Server Communication Setup
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(certfile=self.server_cert, keyfile=self.server_key)
self._context.load_verify_locations(cafile=self.client_cert)
self.sock = None
def connect(self):
try: # create socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) ###<-- socket.socket() ???
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
print(str(e))
print('Waiting for a Connection..')
self.sock.listen(3)
def threaded_client(self, conn: socket):
try:
conn.send(str.encode('Welcome to the Server'))
while True:
data = recv_msg(conn)
print("data")
print(data)
if data is not None:
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
send_msg(conn, str.encode(reply))
except Exception as e:
print(traceback.format_exc())
print(str(e))
finally:
print("Closing connection")
conn.shutdown(socket.SHUT_RDWR)
conn.close()
#conn.close()
def waitforconnection(self):
while True:
Client, addr = self.sock.accept()
conn = self._context.wrap_socket(Client, server_side=True)
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
print("SSL established. Peer: {}".format(conn.getpeercert()))
start_new_thread(self.threaded_client, (conn, )) # Calling threaded_client() on a new thread
self.ThreadCount += 1
print('Thread Number: ' + str(self.ThreadCount))
#self.sock.close()
server = Server()
server.connect()
server.waitforconnection()
Client code:
import socket
import struct # Here to convert Python data types into byte streams (in string) and back
import sys
import ssl
import socket
import selectors
import types
import io
import os
import time
import requests
from pathlib import Path
import mysql.connector as mysql
from loguru import logger as log
from utils.misc import read_py_config
import json
import rsa
import base64
class Client:
def __init__(self):
self.host, self.port = "127.0.0.1", 65416
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self.server_crt = os.path.join(os.path.dirname(__file__), "server.crt")
self.sni_hostname = "example.com"
self._context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=self.server_crt)
self._context.load_cert_chain(certfile=self.client_cert, keyfile=self.client_key)
self._sock = None
self._ssock = None
def checkvalidclient(self):
# ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock, server_side=False, server_hostname=self.sni_hostname)
self._ssock.connect((HOST, PORT))
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
Response = self._ssock.recv(1024)
if Response is not None:
print(Response.decode('utf-8'))
while True:
Input = input('Say Something: ')
send_msg(self._ssock, str.encode(Input))
Response = recv_msg(self._ssock)
if Response is not None:
print(Response.decode('utf-8'))
def closesockconnection(self):
self._ssock.close()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
Also ensure (as per the link) that the certificate creation is correct.
There is also another useful link at:
Exploring HTTPS With Python
Which covers HTTPS, specifically the Wireshark section allows you to monitor the traffic from client to server. After completing the above and deploying Wireshark I see that the data is encrypted. Any editing of the certificates (manually) causes the app to fail.
There still needs to be additions of try and except if the communication is halted midway etc. But hoping it will smooth the journey for others.
Thanks to the commenters, helped lead me on the way to solution.

Sending and receiving using sockets python

I am trying to create a function to send and receive information over a socket client & server. It appears that my code is somehow blocking. In the code the first command iteration in my for loop is carried out but then the process becomes blocked. Does anyone have any suggestions how to do this using threading or multithreading?
My code is below:
import socket
import json
import sys
import time
import select
import queue
Ni_Rio_IP= "172.22.11.2"
Ni_Base_IP= "172.22.11.1"
class AliceRio:
def __init__(self, ip_rio, ip_pc):
self.ip_rio = ip_rio
AliceRio.udp_port_rio = 60006
self.ip_pc = ip_pc
AliceRio.udp_port_pc = 50005
AliceRio.json= '{"Dest":"","Name":"","Time":"","Val":{"Str":[],"Pos":[[]],"Data":[[]]},"IP":0,"Port":0,"RT error":{"status":false,"code":0,"source":""}}'
AliceRio.dict= json.loads(self.json)
def PrintUDP(self):
print("RIO IP: %s" % self.ip_rio)
print("RIO UDP port: %s" % self.udp_port_rio)
print("PC IP: %s" % self.ip_pc)
print("PC UDP port: %s" % self.udp_port_pc)
def SendRec(self, send_str):
# Set up socket for sending
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # Internet, UDP
sock.sendto(bytes(send_str, 'utf-8'), (self.ip_rio, self.udp_port_rio))
sock.close()
print('got here')
# Set up socket for receiving
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # Internet, UDP
sock.bind((self.ip_pc, self.udp_port_pc))
rec_str, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
print('got here2')
sock.close()
return rec_str
def Receive(self, rec_str):
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # Internet, UDP
sock.bind((self.ip_pc, self.udp_port_pc))
rec_str, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
sock.close()
return rec_str
def Send(self, send_str):
# Set up socket for sending
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # Internet, UDP
sock.sendto(bytes(send_str, 'utf-8'), (self.ip_rio, self.udp_port_rio))
sock.close()
#return rec_str
def Aim(self, aim_perc):
if aim_perc < 0 or aim_perc > 100: return "aim_perc out of range"
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='Laser Control'
Laser_Mode=1
Simmer_A=0
Pulse_A= 0
Pulse_ms= 20
send_dict["Val"]["Str"]=[str(Laser_Mode), str(aim_perc), str(Simmer_A), str(Pulse_A), str(Pulse_ms)]
send_json=json.dumps(send_dict)
# send it out
self.SendRec(send_json)
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
return "Aim laser now at " + rec_dict["Val"]["Str"][1] +'%'
def PWM_Laser_Fan(self, fan_perc):
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='PWM Laser'
send_dict["Val"]["Str"][0]=str(fan_perc)
send_json=json.dumps(send_dict)
# send it out
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
return rec_dict["Val"]["Str"][0]
def Poll(self):
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='Poll'
send_json=json.dumps(send_dict)
# send it out
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
if rec_dict["Val"]["Data"][0][0]==0: pid_mode='off'
else: pid_mode='PID'
print('PID mode:', pid_mode)
print('Pos X:', rec_dict["Val"]["Data"][0][1])
print('Pos Y:', rec_dict["Val"]["Data"][0][2])
print('Home:', rec_dict["Val"]["Data"][0][3])
print('Enabled:', rec_dict["Val"]["Data"][0][4])
def PIDControl(self, pid_mode,pid_center):
if pid_mode=="off": mode= 0
elif pid_mode=="PID":mode =1
else: return "pid_mode not valid"
if pid_center[0] not in range(-2048,2048): return "center x-pos not in range"
if pid_center[1] not in range(-2048,2048): return "center y-pos not in range"
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='PID Control'
send_dict["Val"]["Str"]=[str(mode), str(pid_center[0]), str(pid_center[1])]
send_json=json.dumps(send_dict)
# send it out
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
return "PID mode now at " + rec_dict["Val"]["Str"][0]
Alice1 = AliceRio(Ni_Rio_IP, Ni_Base_IP)
Alice1.PrintUDP()
for i in range(10):
Alice1.Aim((i*10)+10)
time.sleep(0.2)

I would suggest learning to use Pdb and trace through the execution of your program to find where it is getting caught.
Also when learning/developing with sockets I've found that it helps to have separate programs for your client and server in the beginning so you can see how both sides are handling exchanges instead of going the threading route to start since the logging can get confusing, best of luck!

Module threading does help in this scenario.
We can create a thread to receiving incoming messages. And when new message received the thread trigger an event to notify the waiting method SendRec.
import sys
import socket
import json
import threading
import time
class AliceRio:
def __init__(self, .....):
# .........
self.s_in = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.s_in.bind((self.ip_pc, self.udp_port_pc))
self.evt = threading.Event()
self.last_msg = None
def _recv(self):
while True:
msg, _ = self.s_in.recvfrom(1024)
self.last_msg = msg
self.evt.set()
def SendRec(self, send_str):
if not hasattr(self, 'th_recv'):
th = threading.Thread(target=self._recv)
th.setDaemon(True)
th.start()
self.th_recv = th
self.evt.clear()
rio_endpoint = (self.ip_rio, self.udp_port_rio)
s_out = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s_out.sendto(bytes(send_str, 'utf-8'), rio_endpoint)
s_out.close()
if self.evt.wait(timeout=15.0) and self.last_msg:
return self.last_msg
raise Exception('timeout waiting for response.')

Python subprocess connection refused in ros melodic

I want to communicate using subprocess in ros melodic package.
Because ros uses python2 as default, I had some python version problems with python3 libraries.
That's why I decided to use a subprocess.
Here are two of my codes: first is a node of ros(as a client), second is a python3 file as a server.
- Client(image_bus.py)
#!/usr/bin/env python
import rospy
import cv2
import numpy as np
import os
import subprocess
from cv_bridge import CvBridge, CvBridgeError
from std_msgs.msg import String
from sensor_msgs.msg import Image
from json_socket import Client
HOST_IP = "127.0.0.1"
HOST_PORT = 9989
class ImageBus:
def __init__(self):
self.sub_video_capture = rospy.Subscriber('/video_capture', Image, self.send_to_face_detector)
self.cvb = CvBridge()
self.rate = rospy.Rate(2)
self.client = Client()
def send_to_face_detector(self, image):
try:
cv_image = self.cvb.imgmsg_to_cv2(image, "bgr8")
except CvBridgeError as e:
print(e)
filename = os.path.dirname(os.path.abspath(__file__)) + "/face_detector.py"
p = subprocess.Popen(["python3", filename])
print("p : ", p)
self.rate.sleep()
# client = Client()
c = self.client.connect(HOST_IP, HOST_PORT)
send_data = {}
send_data['video_cap'] = cv_image
self.client.send(send_data)
recv_data = self.client.recv()
self.rate.sleep()
self.client.close()
if __name__ == '__main__':
rospy.init_node('ImageBus', anonymous=False)
ib = ImageBus()
try:
rospy.spin()
except rospy.ROSInterruptException:
pass
Server(face_detector.py)
import cv2
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import math
from sklearn.decomposition import PCA
import os
import glob
from jsonsocket import Server
BIND_IP = "127.0.0.1"
BIND_PORT = 9989
class FaceDetector:
def __init__(self):
self.socket = Server(BIND_IP, BIND_PORT)
# Some Codes Here
def check_face(self, frame):
# Some codes Here
return boxed_frame
if __name__=="__main__":
fd = FaceDetector()
fd.socket.accept()
while True:
print("SERVER")
recv_image = fd.socket.recv()
if not recv_image:
break
video_cap = recv_data['video_cap']
face_detected_image = fd.check_face(video_cap)
send_image = {}
send_image['face_detected_image'] = face_detected_image
fd.socket.send(send_image)
fd.socket.close()
And below code is my jsonsocket.py.
import socket
import json
def _send(socket, send_data):
json_data = json.JSONEncoder().encode(send_data)
socket.sendall(json_data.encode())
def _recv(socket):
recv_data = socket.recv(4096)
json_data = json.loads(recv_data.decode())
return json_data
class Server(object):
backlog =1
client =None
def __init__(self, host, port):
self.socket = socket.socket()
self.socket.bind((host, port))
self.socket.listen(self.backlog)
def __del__(self):
self.close()
def accept(self):
if self.client:
self.client.close()
self.client, self.client_addr =self.socket.accept()
return self
def send(self, data):
if not self.client:
raise Exception('Cantnot send data, no client is connected.')
_send(self.client, data)
return self
def recv(self):
if not self.client:
raise Exception('Cannot receive data, no client is connected.')
return _recv(self.client)
def close(self):
if self.client:
self.client.close()
self.client =None
if self.socket:
self.socket.close()
self.socket =None
class Client(object):
socket =None
def __del__(self):
self.close()
def connect(self, host, port):
self.socket = socket.socket()
self.socket.connect((host, port))
return self
def send(self, data):
if not self.socket:
raise Exception('You have to connect first before sending data.')
_send(self.socket, data)
return self
def recv(self):
if not self.socket:
raise Exception('You have to connect first before receving data.')
return _recv(self.socket)
def close(self):
if self.socket:
self.socket.close()
self.socket =None
When I roslaunch this package with my launch file, then It outputs:
File "/catkin_ws/src/jetson/src/image_bus.py", line 37, in send_to_face_detector
c = self.client.connect(HOST_IP, HOST_PORT)
File "/catkin_ws/src/jetson/src/json_socket.py", line 67, in connect
self.socket.connect((host, port))
File "/usr/lib/python2.7/socket.py", line 228, in meth
return getattr(self._sock,name)(*args)
error: [Errno 111] Connection refused
How can I solve this problem?
I tried to change the port number, change the variable name, check where the problem occurs exactly. However, I couldn't find out.
Help me, please.
Thank you.

Stripped down, this looks purely like an issue with your socket code: you're trying to connect() and the other side isn't allowing it.
I notice your socket() calls, however, are empty; try socket.socket(socket.AF_INET, socket.SOCK_DGRAM) instead of the empty call.
Try isolating/testing just the server/client, there are other questions that address issues with socket connections.

Why h.getresponse() is needed in python logging HTTPHandler?

I overrided the method emit of python logging httphandler to adapt it to my needs, and I noticed the line
h.getresponse() #can't do anything with the result
Why is this line necessary?
I noticed that removing this line has no effect when using unsecure logging, but makes the logs fail when using secure connection.
def emit(self, record):
"""
Emit a record.
Send the record to the Web server as a percent-encoded dictionary
"""
try:
import http.client, urllib.parse
host = self.host
if self.secure:
h = http.client.HTTPSConnection(host, context=self.context)
else:
h = http.client.HTTPConnection(host)
url = self.url
data = urllib.parse.urlencode(self.mapLogRecord(record))
if self.method == "GET":
if (url.find('?') >= 0):
sep = '&'
else:
sep = '?'
url = url + "%c%s" % (sep, data)
h.putrequest(self.method, url)
# support multiple hosts on one IP address...
# need to strip optional :port from host, if present
i = host.find(":")
if i >= 0:
host = host[:i]
# See issue #30904: putrequest call above already adds this header
# on Python 3.x.
# h.putheader("Host", host)
if self.method == "POST":
h.putheader("Content-type",
"application/x-www-form-urlencoded")
h.putheader("Content-length", str(len(data)))
if self.credentials:
import base64
s = ('%s:%s' % self.credentials).encode('utf-8')
s = 'Basic ' + base64.b64encode(s).strip().decode('ascii')
h.putheader('Authorization', s)
h.endheaders()
if self.method == "POST":
h.send(data.encode('utf-8'))
h.getresponse() #can't do anything with the result
except Exception:
self.handleError(record)

The getresponse() call guarantees that the request is actually sent to the server by getting the response to the request.

Buildbot force build via python

Just a little question : it's possible to force a build in Buildbot via a python script or command line (and not via the web interface) ?
Thank you!

If you have a PBSource configured in your master.cfg, you can send a change from the command line:
buildbot sendchange --master {MASTERHOST}:{PORT} --auth {USER}:{PASS}
--who {USER} {FILENAMES..}

You can make a python script using the urlib2 or requests library to simulate a POST to the web UI
import urllib2
import urllib
import cookielib
import uuid
import unittest
import sys
from StringIO import StringIO
class ForceBuildApi():
MAX_RETRY = 3
def __init__(self, server):
self.server = server
cookiejar = cookielib.CookieJar()
self.urlOpener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
def login(self, user, passwd):
data = urllib.urlencode(dict(username=user,
passwd=passwd))
url = self.server + "login"
request = urllib2.Request(url, data)
res = self.urlOpener.open(request).read()
if res.find("The username or password you entered were not correct") > 0:
raise Exception("invalid password")
def force_build(self, builder, reason, **kw):
"""Create a buildbot build request
several attempts are created in case of errors
"""
reason = reason + " ID="+str(uuid.uuid1())
kw['reason'] = reason
data_str = urllib.urlencode(kw)
url = "%s/builders/%s/force" % (self.server, builder)
print url
request = urllib2.Request(url, data_str)
file_desc = None
for i in xrange(self.MAX_RETRY):
try:
file_desc = self.urlOpener.open(request)
break
except Exception as e:
print >>sys.stderr, "error when doing force build", e
if file_desc is None:
print >>sys.stderr, "too many errors, giving up"
return None
for line in file_desc:
if 'alert' in line:
print >>sys.stderr, "invalid arguments", url, data_str
return None
if 'Authorization Failed' in line:
print >>sys.stderr, "Authorization Failed"
return
return reason
class ForceBuildApiTest(unittest.TestCase):
def setUp(self):
from mock import Mock # pip install mock for test
self.api = ForceBuildApi("server/")
self.api.urlOpener = Mock()
urllib2.Request = Mock()
uuid.uuid1 = Mock()
uuid.uuid1.return_value = "myuuid"
sys.stderr = StringIO()
def test_login(self):
from mock import call
self.api.login("log", "pass")
self.assertEquals(len(self.api.urlOpener.open.call_args_list), 1)
req = urllib2.Request.call_args_list
self.assertEquals([call('server/login', 'passwd=pass&username=log')], req)
def test_force(self):
from mock import call
self.api.urlOpener.open.return_value = ["blabla"]
r = self.api.force_build("builder1", reason="reason", param1="foo", param2="bar")
self.assertEquals(len(self.api.urlOpener.open.call_args_list), 1)
req = urllib2.Request.call_args_list
self.assertEquals([call('server//builders/builder1/force', 'reason=reason+ID%3Dmyuuid&param2=bar&param1=foo')], req)
self.assertEquals(r, "reason ID=myuuid")
def test_force_fail1(self):
from mock import call
self.api.urlOpener.open.return_value = ["alert bla"]
r = self.api.force_build("builder1", reason="reason", param1="foo", param2="bar")
self.assertEquals(len(self.api.urlOpener.open.call_args_list), 1)
req = urllib2.Request.call_args_list
self.assertEquals([call('server//builders/builder1/force', 'reason=reason+ID%3Dmyuuid&param2=bar&param1=foo')], req)
self.assertEquals(sys.stderr.getvalue(), "invalid arguments server//builders/builder1/force reason=reason+ID%3Dmyuuid&param2=bar&param1=foo\n")
self.assertEquals(r, None)
def test_force_fail2(self):
from mock import call
def raise_exception(*a, **kw):
raise Exception("oups")
self.api.urlOpener.open = raise_exception
r = self.api.force_build("builder1", reason="reason", param1="foo", param2="bar")
req = urllib2.Request.call_args_list
self.assertEquals([call('server//builders/builder1/force', 'reason=reason+ID%3Dmyuuid&param2=bar&param1=foo')], req)
self.assertEquals(sys.stderr.getvalue(), "error when doing force build oups\n"*3 + "too many errors, giving up\n")
self.assertEquals(r, None)
def test_force_fail3(self):
from mock import call
self.api.urlOpener.open.return_value = ["bla", "blu", "Authorization Failed"]
r = self.api.force_build("builder1", reason="reason", param1="foo", param2="bar")
req = urllib2.Request.call_args_list
self.assertEquals([call('server//builders/builder1/force', 'reason=reason+ID%3Dmyuuid&param2=bar&param1=foo')], req)
self.assertEquals(sys.stderr.getvalue(), "Authorization Failed\n")
self.assertEquals(r, None)
if __name__ == '__main__':
unittest.main()