I'm working on a microservice endpoint which "only" consumes messages from RabbitMq and then serves those messages as SSE events.
This is my code for the endpoint:
data = queue.Queue()
def sseRouteHandler(reqId):
def consume():
connection = pika.BlockingConnection(pika.ConnectionParameters(host=rmqHostName,
port=rmqPort,
virtual_host='/',
credentials=pika.PlainCredentials(username=rmqUserName, password=rmqPassword),
connection_attempts=retryCount,
retry_delay=retryInterval))
channel = connection.channel()
channel.queue_declare(queue=consumerQueues, auto_delete=False, exclusive=False, arguments=None)
channel.queue_bind(queue=consumerQueues, exchange="eis.ds", routing_key=consumerQueues)
def callback(ch, method, properties, body):
# print(body)
data.put(body)
# ch.basic_ack(delivery_tag = method.delivery_tag)
channel.basic_consume(queue=consumerQueues, on_message_callback=callback, exclusive=False, arguments=None)
channel.start_consuming()
thread = Thread(target=consume)
thread.start()
# Check if queue is empty, if not then pop the element else continue. Not working... I only get values after I close the server using keyboard interrupt
def xcallback():
while True:
if not data.empty():
yield data.get(block=False)
else:
continue
return Response(xcallback(), mimetype="text/event-stream")
Check if queue is empty, if not then get the element else continue. This is not working... I only get values after I close the server using keyboard interrupt
^C^CTraceback (most recent call last):
File "python3.6/site-packages/waitress/server.py", line 307, in run
use_poll=self.adj.asyncore_use_poll,
File "python3.6/site-packages/waitress/wasyncore.py", line 222, in loop
poll_fun(timeout, map)
File "python3.6/site-packages/waitress/wasyncore.py", line 152, in poll
r, w, e = select.select(r, w, e, timeout)
KeyboardInterrupt
curl -X GET 'http://localhost:9080/v1/req/1ljhzckm3'
curl: (18) transfer closed with outstanding read data remaining
{"requestId": "1212122", "message": "Dat"}
What could be the solution for this?
Related
I'm using this example provided in python docs on asyncio streams, to write data and read from a socket.
I am consuming rabbitmq and sending the messages through the socket and waiting for a response. I've setup the reader and writer in __init__() :
self.reader, self.writer = await asyncio.open_connection(self.host, self.port, loop=self.loop)
and in consuming messages, I just send the message to the socket and read the response, then publish the response to another queue (after some processing):
async def process_airtime(self, message: aio_pika.IncomingMessage):
async with message.process():
logger.info('Send: %r' % message.body)
self.writer.write(message.body)
data = await self.reader.read(4096)
logger.info('Received: %r' % data)
await self.publish(data) # publishing to some other queue
The problem is when i try to consume like say 10 messages, all other messges raise this error, although the last message successfully gets a response..
RuntimeError: read() called while another coroutine is already waiting for incoming data
This is the response i get(i've truncated some response..):
2022-02-05 10:59:17,123 INFO [__main__:130] request_consumer [*] waiting for messages...
Send: b'\x00\x00\x00"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
Send: b'\x00\x00\x00"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
Send: b'\x00\x00\x00"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
...
2022-02-05 10:59:17,194 ERROR [asyncio:1707] base_events Task exception was never retrieved
future: <Task finished name='Task-53' coro=<consumer() done, defined at /home/steven/workspace/python/onfon/cheka/venv/lib/python3.8/site-packages/aio_pika/queue.py:25> exception=RuntimeError('read() called while another coroutine is already waiting for incoming data')>
Traceback (most recent call last):
File "/home/steven/workspace/python/onfon/cheka/venv/lib/python3.8/site-packages/aio_pika/queue.py", line 27, in consumer
return await create_task(callback, message, loop=loop)
File "request_consumer.py", line 122, in process_airtime
data = await self.reader.read(4096)
File "/usr/lib/python3.8/asyncio/streams.py", line 684, in read
await self._wait_for_data('read')
File "/usr/lib/python3.8/asyncio/streams.py", line 503, in _wait_for_data
raise RuntimeError(
RuntimeError: read() called while another coroutine is already waiting for incoming data
2022-02-05 10:59:17,194 ERROR [asyncio:1707] base_events Task exception was never retrieved
future: <Task finished name='Task-54' coro=<consumer() done, defined at /home/steven/workspace/python/onfon/cheka/venv/lib/python3.8/site-packages/aio_pika/queue.py:25> exception=RuntimeError('read() called while another coroutine is already waiting for incoming data')>
Traceback (most recent call last):
File "/home/steven/workspace/python/onfon/cheka/venv/lib/python3.8/site-packages/aio_pika/queue.py", line 27, in consumer
return await create_task(callback, message, loop=loop)
File "request_consumer.py", line 122, in process_airtime
data = await self.reader.read(4096)
File "/usr/lib/python3.8/asyncio/streams.py", line 684, in read
await self._wait_for_data('read')
File "/usr/lib/python3.8/asyncio/streams.py", line 503, in _wait_for_data
raise RuntimeError(
RuntimeError: read() called while another coroutine is already waiting for incoming data
Received: b'\xf2>D\x95\n\xe0\x80 \x00\x00\x00\x00\x00\x00\x00"0000000000000000000000000000000000000000000000'
My question is, what should i, probably, do to make read() be called again only after a current coroutine reading has finished. Will that affect performance or is there some way i can read on, say, different threads?
I will appreciate if someone points me the right direction.
I use python3.8 on linux
Simple answer is to only read() from one task.
It sounds like you are using a callback to consume RMQ messages. If so, then aio_pika will consume messages asynchronously (ie. concurrently) if it has a multiple messages. That is, it will create a new task for each callback/message and leave it to its own devices.
Given that you have a read() whilst processing a message, that doesn't really make sense for your read calls. How will you know which read is for which message. You need to find some way to sync your
reads to each message. There are a few ways you can do this:
Put a lock around calls to read()
Creating a separate task that is solely responsible for calling read(), it
puts the results onto a queue, from which any task can read. asyncio
queues are task safe (unlike read()).
Or perhaps most simply, by using the queue as an iterator (and
not spawning a new task to handle the message).
I am assuming you current code looks something a bit like:
async def init_rmq_consumer():
# connect to RMQ and create a queue
...
queue = ...
# start consuming messages off of the queue
await queue.consume(process_airtime)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(init_rmq_consumer())
loop.run_forever()
Using the queue as an iterator might look something like:
async def main():
# connect to RMQ and create a queue
...
queue = ...
# start consuming messages off of the queue *serially*
async with queue.iterator() as queue_iter:
async for message in queue_iter:
# we will not fetch a new message from the queue until we are
# finished with this one.
await process_airtime(message)
if __name__ == '__main__':
asyncio.run(main())
We all know that using asyncio substantially improves the performance of a socket server, and obviously things get even more awesome if we could take advantage of all cores in our cpu (maybe via multiprocessing module or os.fork() etc.)
I'm now trying to build a multicore socket server demo, with a asynchronous socket server listening on each core and all binding to one port. Simply by creating a async server and then use os.fork(), let processes work competitively.
However the single-core-fine code runs into some trouble when I'm trying to fork. Seems like there's some problem with registering same filedescriptors from different processes in epoll selector module.
I'm showing some code below, can anyone help me out?
Here's a simple, logically clear code of echo server using asyncio:
import os
import asyncio #,uvloop
from socket import *
# hendler sends back incoming message directly
async def handler(loop, client):
with client:
while True:
data = await loop.sock_recv(client, 64)
if not data:
break
await loop.sock_sendall(client, data)
# create tcp server
async def create_server(loop):
sock = socket(AF_INET ,SOCK_STREAM)
sock.setsockopt(SOL_SOCKET , SO_REUSEADDR ,1)
sock.bind(('',25000))
sock.listen()
sock.setblocking(False)
return sock
# whenever accept a request, create a handler task in eventloop
async def serving(loop, sock):
while True:
client ,addr = await loop.sock_accept(sock)
loop.create_task(handler(loop ,client))
loop = asyncio.get_event_loop()
sock = loop.run_until_complete(create_server(loop))
loop.create_task(serving(loop, sock))
loop.run_forever()
It works fine until I'm trying to fork, after the socket was bounl and before server starts serving. (This logic works fine in synchronous -- threading based code.)
When I'm trying this:
loop = asyncio.get_event_loop()
sock = loop.run_until_complete(create_server(loop))
from multiprocessing import cpu_count
for num in range(cpu_count() - 1):
pid = os.fork()
if pid <= 0: # fork process as the same number as
break # my cpu cores
loop.create_task(serving(loop, sock))
loop.run_forever()
Theoretically forked process are bounl to a same socket? And run in a same event loop? then work just fine?
However I'm getting these error messages:
Task exception was never retrieved
future: <Task finished coro=<serving() done, defined at /home/new/LinuxDemo/temp1.py:21> exception=FileExistsError(17, 'File exists')>
Traceback (most recent call last):
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 262, in _add_reader
key = self._selector.get_key(fd)
File "/usr/local/lib/python3.7/selectors.py", line 192, in get_key
raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '6 is not registered'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/test/temp1.py", line 23, in serving
client ,addr = await loop.sock_accept(sock)
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 525, in sock_accept
self._sock_accept(fut, False, sock)
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 538, in _sock_accept
self.add_reader(fd, self._sock_accept, fut, True, sock)
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 335, in add_reader
return self._add_reader(fd, callback, *args)
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 265, in _add_reader
(handle, None))
File "/usr/local/lib/python3.7/selectors.py", line 359, in register
self._selector.register(key.fd, poller_events)
FileExistsError: [Errno 17] File exists
Python version 3.7.3,
I'm totally confused about what's going on.
Could anybody help? thanks
According to the tracker issue, it is not supported to fork an existing asyncio event loop and attempt to use it from multiple processes. However, according to Yury's comment on the same issue, multi-processing can be implemented by forking before starting a loop, therefore running fully independent asyncio loops in each child.
Your code actually confirms this possibility: while create_server is async def, it doesn't await anything, nor does it use the loop argument. So we can implement Yury's approach by by making create_server a regular function, removing the loop argument, and calling it before os.fork(), and only running event loops after forking:
import os, asyncio, socket, multiprocessing
async def handler(loop, client):
with client:
while True:
data = await loop.sock_recv(client, 64)
if not data:
break
await loop.sock_sendall(client, data)
# create tcp server
def create_server():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(('', 25000))
sock.listen()
sock.setblocking(False)
return sock
# whenever accept a request ,create a handler task in eventloop
async def serving(loop, sock):
while True:
client, addr = await loop.sock_accept(sock)
loop.create_task(handler(loop, client))
sock = create_server()
for num in range(multiprocessing.cpu_count() - 1):
pid = os.fork()
if pid <= 0: # fork process as the same number as
break # my cpu cores
loop = asyncio.get_event_loop()
loop.create_task(serving(loop, sock))
loop.run_forever()
I have a task queue in RabbitMQ with multiple producers (12) and one consumer for heavy tasks in a webapp. When I run the consumer it starts dequeuing some of the messages before crashing with this error:
Traceback (most recent call last):
File "jobs.py", line 42, in <module> jobs[job](config)
File "/home/ec2-user/project/queue.py", line 100, in init_queue
channel.start_consuming()
File "/usr/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 1822, in start_consuming
self.connection.process_data_events(time_limit=None)
File "/usr/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 749, in process_data_events
self._flush_output(common_terminator)
File "/usr/lib/python2.7/site-packages/pika/adapters/blocking_connection.py", line 477, in _flush_output
result.reason_text)
pika.exceptions.ConnectionClosed: (-1, "error(104, 'Connection reset by peer')")
The producers code is:
message = {'image_url': image_url, 'image_name': image_name, 'notes': notes}
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='tasks_queue')
channel.basic_publish(exchange='', routing_key=queue_name, body=json.dumps(message))
connection.close()
And the only consumer's code (the one is clashing):
def callback(self, ch, method, properties, body):
"""Callback when receive a message."""
message = json.loads(body)
try:
image = _get_image(message['image_url'])
except:
sys.stderr.write('Error getting image in note %s' % note['id'])
# Crop image with PIL. Not so expensive
box_path = _crop(image, message['image_name'], box)
# API call. Long time function
result = long_api_call(box_path)
if result is None:
sys.stderr.write('Error in note %s' % note['id'])
return
# update the db
db.update_record(result)
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='tasks_queue')
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback_obj.callback, queue='tasks_queue', no_ack=True)
channel.start_consuming()
As you can see, there are 3 expensive functions for message. One crop task, one API call and one database update. Without the API call, que consumer runs smoothly.
Thanks in advance
Your RabbitMQ log shows a message that I thought we might see:
missed heartbeats from client, timeout: 60s
What's happening is that your long_api_call blocks Pika's I/O loop. Pika is a very lightweight library and does not start threads in the background for you so you must code in such a way as to not block Pika's I/O loop longer than the heartbeat interval. RabbitMQ thinks your client has died or is unresponsive and forcibly closes the connection.
Please see my answer here which links to this example code showing how to properly execute a long-running task in a separate thread. You can still use no_ack=True, you will just skip the ack_message call.
NOTE: the RabbitMQ team monitors the rabbitmq-users mailing list and only sometimes answers questions on StackOverflow.
Starting with RabbitMQ 3.5.5, the broker’s default heartbeat timeout
decreased from 580 seconds to 60 seconds.
See pika: Ensuring well-behaved connection with heartbeat and blocked-connection timeouts.
The simplest fix is to increase the heartbeat timeout:
rabbit_url = host + "?heartbeat=360"
conn = pika.BlockingConnection(pika.URLParameters(rabbit_url))
# or
params = pika.ConnectionParameters(host, heartbeat=360)
conn = pika.BlockingConnection(params)
I am using Kombu in Python to consume a durable RabbitMQ queue.
There is only one consumer consuming the queue in Windows. This consumer produces the below error:
Traceback (most recent call last):
File ".\consumer_windows.py", line 66, in <module>
message.ack()
File "C:\Users\Administrator\Anaconda2\lib\site-packages\kombu\message.py", line 88, in ack
self.channel.basic_ack(self.delivery_tag)
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\channel.py", line 1584, in basic_ack
self._send_method((60, 80), args)
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\abstract_channel.py", line 56, in _send_method
self.channel_id, method_sig, args, content,
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\method_framing.py", line 221, in write_method
write_frame(1, channel, payload)
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\transport.py", line 182, in write_frame
frame_type, channel, size, payload, 0xce,
File "C:\Users\Administrator\Anaconda2\lib\socket.py", line 228, in meth
return getattr(self._sock,name)(*args)
error: [Errno 10054] An existing connection was forcibly closed by the remote host
There are at most 500 messages in the queue at any one time. Each message is small in size however it is a task and takes up to 10 minutes to complete (although it usually takes less then 5 mins per message).
I have tried restarting the consumer, RabbitMQ server and deleting the queue however the error still persists.
I've seen this question however the answer is from 2010 and my rabbitmq.log has different entries:
=ERROR REPORT==== 24-Apr-2016::08:26:20 ===
closing AMQP connection <0.6716.384> (192.168.X.X:59602 -> 192.168.Y.X:5672):
{writer,send_failed,{error,timeout}}
There were no recent events in the rabbitmq-sasl.log.
Why is this error happening and how can I prevent it from occurring?
I'm still looking for an answer. In the meantime I restart the connection to my rabbit server:
while True:
try:
connection = pika.BlockingConnection(params)
channel = connection.channel() # start a channel
channel.queue_declare(queue=amqp_q, durable=True) # Declare a queue
...
except pika.exceptions.ConnectionClosed:
print('connection closed... and restarted')
I had the same issue with MySQL server which was hosted...
I came to understand that it happened if we open the connection for a long time or unmodified for a long time..
If your program opens the DB or anything until the whole program runs make it in a such a way that it opens the DB writes everything and closes and repeats
I don't know what exactly rabbitmq is but I think the error you wrote as title may be for this reason
I had the same error (using pure PIKA library) and trying to connect to a Rabbitmq broker through Amazon MQ.
The problem resolved when setting up correctly the ssl configuration.
Please check full blog post here: https://docs.aws.amazon.com/amazon-mq/latest/developer-guide/amazon-mq-rabbitmq-pika.html
Core snippets that I used:
Define Pika Client:
import ssl
import pika
class BasicPikaClient:
def __init__(self, rabbitmq_broker_id, rabbitmq_user, rabbitmq_password, region):
# SSL Context for TLS configuration of Amazon MQ for RabbitMQ
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.set_ciphers('ECDHE+AESGCM:!ECDSA')
url = f"amqps://{rabbitmq_user}:{rabbitmq_password}#{rabbitmq_broker_id}.mq.{region}.amazonaws.com:5671"
parameters = pika.URLParameters(url)
parameters.ssl_options = pika.SSLOptions(context=ssl_context)
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
Producer:
from basicClient import BasicPikaClient
class BasicMessageSender(BasicPikaClient):
def declare_queue(self, queue_name, durable):
print(f"Trying to declare queue({queue_name})...")
self.channel.queue_declare(queue=queue_name, durable=durable)
def send_message(self, exchange, routing_key, body):
channel = self.connection.channel()
channel.basic_publish(exchange=exchange,
routing_key=routing_key,
body=body)
print(f"Sent message. Exchange: {exchange}, Routing Key: {routing_key}, Body: {body}")
def close(self):
self.channel.close()
self.connection.close()
Calling Producer:
# Initialize Basic Message Sender which creates a connection
# and channel for sending messages.
basic_message_sender = BasicMessageSender(
credentials["broker_id"],
credentials["username"],
credentials['password'],
credentials['region']
)
# Declare a queue
basic_message_sender.declare_queue("q_name", durable=True)
# Send a message to the queue.
basic_message_sender.send_message(exchange="", routing_key="q_name", body=b'Hello World 2!')
# Close connections.
basic_message_sender.close()
Define Consumer:
class BasicMessageReceiver(BasicPikaClient):
def get_message(self, queue):
method_frame, header_frame, body = self.channel.basic_get(queue)
if method_frame:
print(method_frame, header_frame, body)
self.channel.basic_ack(method_frame.delivery_tag)
return method_frame, header_frame, body
else:
print('No message returned')
def close(self):
self.channel.close()
self.connection.close()
Calling Consumer:
# Create Basic Message Receiver which creates a connection
# and channel for consuming messages.
basic_message_receiver = BasicMessageReceiver(
credentials["broker_id"],
credentials["username"],
credentials['password'],
credentials['region']
)
# Consume the message that was sent.
basic_message_receiver.get_message("q_name")
# Close connections.
basic_message_receiver.close()
I hope the above helps.
Thanks
I'm trying to send a python dictionary from a python producer to a python consumer using RabbitMQ. The producer first establishes the connection to local RabbitMQ server. Then it creates a queue to which the message will be delivered, and finally sends the message. The consumer first connects to RabbitMQ server and then makes sure the queue exists by creating the same queue. It then receives the message from producer within the callback function, and prints the 'id' value (1). Here are the scripts for producer and consumer:
producer.py script:
import pika
import sys
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_queue', durable=True)
message = {'id': 1, 'name': 'name1'}
channel.basic_publish(exchange='',
routing_key='task_queue',
body=message,
properties=pika.BasicProperties(
delivery_mode = 2, # make message persistent
))
print(" [x] Sent %r" % message)
connection.close()
consumer.py script:
import pika
import time
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_queue', durable=True)
print(' [*] Waiting for messages. To exit press CTRL+C')
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
print(body['id'])
print(" [x] Done")
ch.basic_ack(delivery_tag = method.delivery_tag)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback,
queue='task_queue')
channel.start_consuming()
But, when I run the producer.py, I get this error:
line 18, in <module>
delivery_mode = 2, # make message persistent
File "/Library/Python/2.7/site-packages/pika/adapters/blocking_connection.py", line 1978, in basic_publish
mandatory, immediate)
File "/Library/Python/2.7/site-packages/pika/adapters/blocking_connection.py", line 2064, in publish
immediate=immediate)
File "/Library/Python/2.7/site-packages/pika/channel.py", line 338, in basic_publish
(properties, body))
File "/Library/Python/2.7/site-packages/pika/channel.py", line 1150, in _send_method
self.connection._send_method(self.channel_number, method_frame, content)
File "/Library/Python/2.7/site-packages/pika/connection.py", line 1571, in _send_method
self._send_message(channel_number, method_frame, content)
File "/Library/Python/2.7/site-packages/pika/connection.py", line 1596, in _send_message
content[1][s:e]).marshal())
TypeError: unhashable type
Could anybody help me? Thanks!
You can't send native Python types as your payload, you have to serialize them first. I recommend using JSON:
import json
channel.basic_publish(exchange='',
routing_key='task_queue',
body=json.dumps(message),
properties=pika.BasicProperties(
delivery_mode = 2, # make message persistent
))
and
def callback(ch, method, properties, body):
print(" [x] Received %r" % json.loads(body))