We Keep Coding

"This socket is already used by another greenlet" using locust to report response times into graphite

Hello I managed to get locust to work with graphite with the following code:
class MyLocust(HttpUser):
tasks = [MyTask]
sock = None
def exit_handler(self):
self.sock.shutdown(socket.SHUT_RDWR)
self.sock.close()
def hook_request_success(self, request_type, name, response_time, response_length):
# print(response_time)
message="%s %d %d\n" % ("client." + name.replace('.', '-'), response_time, time.time())
self.sock.send(message.encode())
def hook_request_fail(self, request_type, name, response_time, exception):
self.request_fail_stats.append([name, request_type, response_time, exception])
def __init__(self, parent):
super().__init__(parent)
self.sock = socket.socket()
self.sock.connect( ("192.168.XX.YYYY", 2003) )
events.request_success.add_listener(self.hook_request_success)
events.request_failure.add_listener(self.hook_request_fail)
atexit.register(self.exit_handler)
Adn this works quiet good while I use a small amout of users in locust, around 100.
But when the users start to go up, I get the following exception:
[2021-05-31 18:40:48,430] HMG28/ERROR/root: Uncaught exception in event handler:
Traceback (most recent call last):
File "c:\python39\lib\site-packages\gevent\_socketcommon.py", line 722, in send
return self._sock.send(data, flags)
BlockingIOError: [WinError 10035] A non-blocking socket operation could not be completed immediately
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\python39\lib\site-packages\locust\event.py", line 40, in fire
handler(**kwargs)
File "C:\Users\AAA\Desktop\AAA\locust_files\locustfile.py", line 60, in hook_request_success
self.sock.send(message.encode())
File "c:\python39\lib\site-packages\gevent\_socketcommon.py", line 727, in send
self._wait(self._write_event)
File "src\\gevent\\_hub_primitives.py", line 317, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src\\gevent\\_hub_primitives.py", line 322, in gevent._gevent_c_hub_primitives.wait_on_socket
File "src\\gevent\\_hub_primitives.py", line 297, in gevent._gevent_c_hub_primitives._primitive_wait
gevent.exceptions.ConcurrentObjectUseError: This socket is already used by another greenlet: <bound method Waiter.switch of <gevent._gevent_c_waiter.Waiter object at 0x00000222BA8FB4F0>>
Is there a way to make the socket connections wait until they can send the data?

Searching SO for your error, I found this possibly helpful answer:
Non-Blocking error when adding timeout to python server
If you continue to have issues, you can try using a gevent compatible socket and see if that gets you the concurrency you want more easily.

Pyro4 [Errno -2]

I am trying out pyro4 connection between my PC and Raspberry Pi 4.
Code on my PC is:
# saved as server.py
import Pyro4, Pyro4.naming
import socket, threading
# Define an object that will be accessible over the network.
# This is where all your code should go...
#Pyro4.expose
class MessageServer(object):
def show_message(self, msg):
print("Message received: {}".format(msg))
# Start a Pyro nameserver and daemon (server process) that are accessible
# over the network. This has security risks; see
# https://pyro4.readthedocs.io/en/stable/security.html
hostname = socket.gethostname()
ns_thread = threading.Thread(
target=Pyro4.naming.startNSloop, kwargs={'host': hostname}
)
ns_thread.daemon = True # automatically exit when main program finishes
ns_thread.start()
main_daemon = Pyro4.Daemon(host=hostname)
# find the name server
ns = Pyro4.locateNS()
# register the message server as a Pyro object
main_daemon_uri = main_daemon.register(MessageServer)
# register a name for the object in the name server
ns.register("example.message", main_daemon_uri)
# start the event loop of the main_daemon to wait for calls
print("Message server ready.")
main_daemon.requestLoop()
And code on my Raspberry is:
import Pyro4
import sys
print("Message:")
msg=sys.stdin.readline().strip()
message_server = Pyro4.Proxy("PYRONAME:192.168.1.5")
message_server.show_message(msg)
Code on my PC doesn t show any errors, but when I try to send a message from raspberry i get this:
What s your message?
test
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/socketutil.py", line 102, in getIpAddress
return getaddr(config.PREFER_IP_VERSION) if ipVersion is None else getaddr(ipVersion)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/socketutil.py", line 94, in getaddr
ip = socket.getaddrinfo(hostname or socket.gethostname(), 80, family, socket.SOCK_STREAM, socket.SOL_TCP)[0][4][0]
File "/usr/lib/python3.7/socket.py", line 748, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 515, in connect_and_handshake
sslContext=sslContext)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/socketutil.py", line 266, in createSocket
if getIpVersion(connect[0]) == 4:
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/socketutil.py", line 68, in getIpVersion
address = getIpAddress(hostnameOrAddress)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/socketutil.py", line 106, in getIpAddress
return getaddr(0)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/socketutil.py", line 94, in getaddr
ip = socket.getaddrinfo(hostname or socket.gethostname(), 80, family, socket.SOCK_STREAM, socket.SOL_TCP)[0][4][0]
File "/usr/lib/python3.7/socket.py", line 748, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/pi/Desktop/client.py", line 10, in <module>
message_server.show_message(msg)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 275, in __getattr__
self._pyroGetMetadata()
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 615, in _pyroGetMetadata
self.__pyroCreateConnection()
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 588, in __pyroCreateConnection
uri = _resolve(self._pyroUri, self._pyroHmacKey)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 1915, in _resolve
return nameserver.lookup(uri.object)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 275, in __getattr__
self._pyroGetMetadata()
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 615, in _pyroGetMetadata
self.__pyroCreateConnection()
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 596, in __pyroCreateConnection
connect_and_handshake(conn)
File "/home/pi/.local/lib/python3.7/site-packages/Pyro4/core.py", line 549, in connect_and_handshake
raise ce
Pyro4.errors.CommunicationError: cannot connect to ('JAKOB-PC', 9090): [Errno -2] Name or service not known
My PC has its firewall disabled, so there shouldn t be any problem with that. My local ip is 192.168.1.5.
I am using a headless Raspberry and write code on it with puTTY and VNC.
I have googled this error but couldn t find any answers. Any help would be appreciated.

I did this
#
# Server.py
#
from __future__ import print_function
import Pyro4
#Pyro4.expose
#Pyro4.behavior(instance_mode="single")
class Messenger(object):
# This is a constructor
def __init__(self):
pass
# This method will be called on the server
def send_message(self, name, message):
print("[{0}] {1}".format(name, message))
def main():
Pyro4.Daemon.serveSimple(
{
Messenger: "example.messenger"
},
ns=True)
if __name__ == "__main__":
main()
#
# Client.py
#
# This is the code that visits the warehouse.
import sys
import Pyro4
import Pyro4.util
sys.excepthook = Pyro4.util.excepthook
messenger = Pyro4.Proxy("PYRONAME:example.messenger#192.168.1.5")
messenger.send_message("Tim", "Hello!")
Then ran
python -m Pyro4.naming -n 192.168.1.5
python Server.py
python Client.py

In short I couldn't solve the problem with Pyro and (almost) no one helped so I decided to use 'websockets' instead.
You can read the documentation here but I'll explain it here anyway.
First of all you need two devices with network connection. You also have to run python 3.6.1 on both of them. After that you also need to install websockets if you don't have them already with pip install websockets or as I had to do it with pip3 install websockets.
Code below runs on the server and handles messages you send to it from client. Function 'hello' is a simple example of processing request and sending back a response. 'request' is the data server receives, that data must be bytes, string on iterable. Response is made by converting request to integer, squaring it and converting it back to string. This response is then sent back to client.
'start_server' defines the server, function that will define its behavior(hello), ip address on witch the server is running on(192.168.1.117) and port on witch it will receive requests(8765).
!/usr/bin/env python
import asyncio
import websockets
print("Running...")
async def hello(websocket, path):
request = await websocket.recv()
print("Request: " + request)
response = str(int(request)*int(request))
await websocket.send(response)
print("Response:" + response)
start_server = websockets.serve(hello, "192.168.1.117", 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Next bit is code on the client. 'uri' is ip and port of the server. Function 'tellServ' asks you to input some data('tell' variable) and sends it to the server. After that it waits for reply and once it gets it it prints it out. In this case if I would input number "6" server would reply with "36". Function loop is in a while loop so I can send multiple numbers without having to restart the script.
#!/usr/bin/env python
import asyncio
import websockets
uri = "ws://192.168.1.117:8765"
async def tellServ():
async with websockets.connect(uri) as websocket:
tell = input("Podatek ki ga posles: ")
await websocket.send(tell)
reply = await websocket.recv()
print("Odgovor:")
print(reply)
while 1:
asyncio.get_event_loop().run_until_complete(tellServ())

How to fix invalid URL error while using http.client?

I am trying to hit an api using http.client of pyhton3 to simulate how I would do the same in a web browser.
However, http.client feels that url is inappropriate.
This is what I am trying to do.
import http.client
connection = http.client.HTTPSConnection("https://analyticsapi.zoho.com/api/EmailAddress/WorkspaceName/TableName?ZOHO_ACTION=IMPORT&ZOHO_OUTPUT_FORMAT=XML&ZOHO_ERROR_FORMAT=XML&ZOHO_API_VERSION=1.0&authtoken=************&ZOHO_IMPORT_TYPE=APPEND&ZOHO_AUTO_IDENTIFY=TRUE&ZOHO_ON_IMPORT_ERROR=ABORT&ZOHO_CREATE_TABLE=TRUE&ZOHO_FILE=/home/dev1/Desktop/Zoho/temporary.csv")
connection.request("GET", "/")
response = connection.getresponse()
print("Status: {} and reason: {}".format(response.status, response.reason))
connection.close()
And this is the error I am getting.
$ python3 pyToTestPushingCSV.py
Traceback (most recent call last):
File "/usr/lib/python3.5/http/client.py", line 798, in _get_hostport
port = int(host[i+1:])
ValueError: invalid literal for int() with base 10: '//analyticsapi.zoho.com/api/usename/ATable/InsideTable?ZOHO_ACTION=IMPORT&ZOHO_OUTPUT_FORMAT=XML&ZOHO_ERROR_FORMAT=XML&ZOHO_API_VERSION=1.0&authtoken=****
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "pyToTestPushingCSV.py", line 3, in <module>
connection = http.client.HTTPSConnection("https://analyticsapi.zoho.com/api/usename/ATable/InsideTable?ZOHO_ACTION=IMPORT&ZOHO_OUTPUT_FORMAT=XML&ZOHO_ERROR_FORMAT=XML&ZOHO_API_VERSION=1.0&authtoken=****************&ZOHO_IMPORT_TYPE=APPEND&ZOHO_AUTO_IDENTIFY=TRUE&ZOHO_ON_IMPORT_ERROR=ABORT&ZOHO_CREATE_TABLE=TRUE&ZOHO_FILE=/home/dev1/Desktop/Zoho/temporary.csv")
File "/usr/lib/python3.5/http/client.py", line 1233, in __init__
source_address)
File "/usr/lib/python3.5/http/client.py", line 762, in __init__
(self.host, self.port) = self._get_hostport(host, port)
File "/usr/lib/python3.5/http/client.py", line 803, in _get_hostport
raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
http.client.InvalidURL: nonnumeric port: '//analyticsapi.zoho.com/api/usename/ATable/InsideTable?ZOHO_ACTION=IMPORT&ZOHO_OUTPUT_FORMAT=XML&ZOHO_ERROR_FORMAT=XML&ZOHO_API_VERSION=1.0&authtoken=*************&ZOHO_IMPORT_TYPE=APPEND&ZOHO_AUTO_IDENTIFY=TRUE&ZOHO_ON_IMPORT_ERROR=ABORT&ZOHO_CREATE_TABLE=TRUE&ZOHO_FILE=/home/dev1/Desktop/Zoho/temporary.csv'
When I hit the URL with my Browser it gives a good response back in XML which in short, succeeds.
This is where I referred for documentation
Can you pin point where I went wrong?

As per documentation HTTPS support is only available if Python was compiled with SSL support (through the ssl module).
Also the default port for https connection is 443. The module seems to be hitting that port by default.

Why am I getting a "Connection refused" in python? Can't resolve using existing solutions

import ssl
import socket
port = 3001
while True:
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
ssl_socket = ssl.wrap_socket(s, cert_reqs=ssl.CERT_REQUIRED, ca_certs='scert.pem')
ssl_socket.connect(('127.0.0.1', 3001))
ssl_socket.write(str(input("Enter two numbers to add:")).encode())
z = ssl_socket.recv(1024)
print(z.decode())
ssl_socket.close()
The error I'm getting is:
Traceback (most recent call last):
ssl_socket.connect(('127.0.0.1', 3001))
self._real_connect(addr, False)
socket.connect(self, addr)
File "/usr/lib/python2.7/socket.py", line 228, in meth
return getattr(self._sock,name)(*args)
error: [Errno 111] Connection refused
I am confused as to what to do. Already tried the methods on the other thread on socket.

Connection to remote MySQL db from Python 3.4

I'm trying to connect to two MySQL databases (one local, one remote) at the same time using Python 3.4 but I'm really struggling. Splitting the problem into three:
Step 1: connect to the local DB. This is working fine
using PyMySQL. (MySQLdb isn't compatible with Python 3.4, of
course.)
Step 2: connect to the remote DB (which needs to
use SSH). I can get it to work from the Linux command prompt but not
from Python... see below.
Step 3: connect to both at the
same time. I think I'm supposed to use a different port for the
remote database so that I can have both connections at the same time
but I'm out of my depth here! If it's relevant then the two DBs will
have different names. And if this question isn't directly related,
please tell me and I'll post it separately.
Unfortunately I'm not really starting in the right place for a newbie... once I can get this working I can happily go back to basic Python and SQL but hopefully someone will take pity on me and give me a hand to get started!
For Step 2, my code is below. It seems to be quite close to the sshtunnel example which answers this question Python - SSH Tunnel Setup and MySQL DB Access - though that uses MySQLdb. For the moment I'm embedding the connection parameters – I'll move them to the config file once it's working properly.
import dropbox, pymysql, shlex, shutil, subprocess
from sshtunnel import SSHTunnelForwarder
import iot_config as cfg
def CloseLocalDB():
localcur.close()
localdb.close()
def CloseRemoteDB():
# Disconnect from the database
# remotecur.close()
# remotedb.close()
# Close the SSH tunnel
# ssh.close()
print("end of CloseRemoteDB function")
def OpenLocalDB():
global localcur, localdb
localdb = pymysql.connect(host=cfg.localdbconn['host'], user=cfg.localdbconn['user'], passwd=cfg.localdbconn['passwd'], db=cfg.localdbconn['db'])
localcur = localdb.cursor()
def OpenRemoteDB():
global remotecur, remotedb
with SSHTunnelForwarder(
('my_remote_site', 22),
ssh_username = "my_ssh_username",
ssh_private_key = "/etc/ssh/my_private_key.ppk",
ssh_private_key_password = "my_private_key_password",
remote_bind_address = ('127.0.0.1', 3308)) as server:
remotedb = None
#Following line gives an error if uncommented
# remotedb = pymysql.connect(host='127.0.0.1', user='remote_db_user', passwd='remote_db_password', db='remote_db_name', port=server.local_bind_port)
#remotecur = remotedb.cursor()
# Main program starts here
OpenLocalDB()
CloseLocalDB()
OpenRemoteDB()
CloseRemoteDB()
This is the error I'm getting:
2016-04-21 19:13:33,487 | ERROR | Secsh channel 0 open FAILED: Connection refused: Connect failed
2016-04-21 19:13:33,553 | ERROR | In #1 <-- ('127.0.0.1', 60591) to ('127.0.0.1', 3308) failed: ChannelException(2, 'Connect failed')
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 60591)
Traceback (most recent call last):
File "/usr/local/lib/python3.4/dist-packages/sshtunnel.py", line 286, in handle
src_address)
File "/usr/local/lib/python3.4/dist-packages/paramiko/transport.py", line 834, in open_channel
raise e
paramiko.ssh_exception.ChannelException: (2, 'Connect failed')
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.4/socketserver.py", line 613, in process_request_thread
self.finish_request(request, client_address)
File "/usr/lib/python3.4/socketserver.py", line 344, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib/python3.4/socketserver.py", line 669, in __init__
self.handle()
File "/usr/local/lib/python3.4/dist-packages/sshtunnel.py", line 296, in handle
raise HandlerSSHTunnelForwarderError(msg)
sshtunnel.HandlerSSHTunnelForwarderError: In #1 <-- ('127.0.0.1', 60591) to ('127.0.0.1', 3308) failed: ChannelException(2, 'Connect failed')
----------------------------------------
Traceback (most recent call last):
File "/home/pi/Documents/iot_pm2/iot_ssh_example_for_help.py", line 38, in <module>
OpenRemoteDB()
File "/home/pi/Documents/iot_pm2/iot_ssh_example_for_help.py", line 32, in OpenRemoteDB
remotedb = pymysql.connect(host='127.0.0.1', user='remote_db_user', passwd='remote_db_password', db='remote_db_name', port=server.local_bind_port)
File "/usr/local/lib/python3.4/dist-packages/pymysql/__init__.py", line 88, in Connect
return Connection(*args, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/pymysql/connections.py", line 678, in __init__
self.connect()
File "/usr/local/lib/python3.4/dist-packages/pymysql/connections.py", line 889, in connect
self._get_server_information()
File "/usr/local/lib/python3.4/dist-packages/pymysql/connections.py", line 1190, in _get_server_information
packet = self._read_packet()
File "/usr/local/lib/python3.4/dist-packages/pymysql/connections.py", line 945, in _read_packet
packet_header = self._read_bytes(4)
File "/usr/local/lib/python3.4/dist-packages/pymysql/connections.py", line 981, in _read_bytes
2013, "Lost connection to MySQL server during query")
pymysql.err.OperationalError: (2013, 'Lost connection to MySQL server during query')
Thanks in advance.

Answering my own question because, with a lot of help from J.M. Fernández on Github, I have a solution: the example that I copied at the beginning uses port 3308 but port 3306 is the standard. Once I'd changed this it started working.