I'm trying to send a python dictionary from a python producer to a python consumer using RabbitMQ. The producer first establishes the connection to local RabbitMQ server. Then it creates a queue to which the message will be delivered, and finally sends the message. The consumer first connects to RabbitMQ server and then makes sure the queue exists by creating the same queue. It then receives the message from producer within the callback function, and prints the 'id' value (1). Here are the scripts for producer and consumer:
producer.py script:
import pika
import sys
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_queue', durable=True)
message = {'id': 1, 'name': 'name1'}
channel.basic_publish(exchange='',
routing_key='task_queue',
body=message,
properties=pika.BasicProperties(
delivery_mode = 2, # make message persistent
))
print(" [x] Sent %r" % message)
connection.close()
consumer.py script:
import pika
import time
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_queue', durable=True)
print(' [*] Waiting for messages. To exit press CTRL+C')
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
print(body['id'])
print(" [x] Done")
ch.basic_ack(delivery_tag = method.delivery_tag)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback,
queue='task_queue')
channel.start_consuming()
But, when I run the producer.py, I get this error:
line 18, in <module>
delivery_mode = 2, # make message persistent
File "/Library/Python/2.7/site-packages/pika/adapters/blocking_connection.py", line 1978, in basic_publish
mandatory, immediate)
File "/Library/Python/2.7/site-packages/pika/adapters/blocking_connection.py", line 2064, in publish
immediate=immediate)
File "/Library/Python/2.7/site-packages/pika/channel.py", line 338, in basic_publish
(properties, body))
File "/Library/Python/2.7/site-packages/pika/channel.py", line 1150, in _send_method
self.connection._send_method(self.channel_number, method_frame, content)
File "/Library/Python/2.7/site-packages/pika/connection.py", line 1571, in _send_method
self._send_message(channel_number, method_frame, content)
File "/Library/Python/2.7/site-packages/pika/connection.py", line 1596, in _send_message
content[1][s:e]).marshal())
TypeError: unhashable type
Could anybody help me? Thanks!
You can't send native Python types as your payload, you have to serialize them first. I recommend using JSON:
import json
channel.basic_publish(exchange='',
routing_key='task_queue',
body=json.dumps(message),
properties=pika.BasicProperties(
delivery_mode = 2, # make message persistent
))
and
def callback(ch, method, properties, body):
print(" [x] Received %r" % json.loads(body))
Related
I'm working on a microservice endpoint which "only" consumes messages from RabbitMq and then serves those messages as SSE events.
This is my code for the endpoint:
data = queue.Queue()
def sseRouteHandler(reqId):
def consume():
connection = pika.BlockingConnection(pika.ConnectionParameters(host=rmqHostName,
port=rmqPort,
virtual_host='/',
credentials=pika.PlainCredentials(username=rmqUserName, password=rmqPassword),
connection_attempts=retryCount,
retry_delay=retryInterval))
channel = connection.channel()
channel.queue_declare(queue=consumerQueues, auto_delete=False, exclusive=False, arguments=None)
channel.queue_bind(queue=consumerQueues, exchange="eis.ds", routing_key=consumerQueues)
def callback(ch, method, properties, body):
# print(body)
data.put(body)
# ch.basic_ack(delivery_tag = method.delivery_tag)
channel.basic_consume(queue=consumerQueues, on_message_callback=callback, exclusive=False, arguments=None)
channel.start_consuming()
thread = Thread(target=consume)
thread.start()
# Check if queue is empty, if not then pop the element else continue. Not working... I only get values after I close the server using keyboard interrupt
def xcallback():
while True:
if not data.empty():
yield data.get(block=False)
else:
continue
return Response(xcallback(), mimetype="text/event-stream")
Check if queue is empty, if not then get the element else continue. This is not working... I only get values after I close the server using keyboard interrupt
^C^CTraceback (most recent call last):
File "python3.6/site-packages/waitress/server.py", line 307, in run
use_poll=self.adj.asyncore_use_poll,
File "python3.6/site-packages/waitress/wasyncore.py", line 222, in loop
poll_fun(timeout, map)
File "python3.6/site-packages/waitress/wasyncore.py", line 152, in poll
r, w, e = select.select(r, w, e, timeout)
KeyboardInterrupt
curl -X GET 'http://localhost:9080/v1/req/1ljhzckm3'
curl: (18) transfer closed with outstanding read data remaining
{"requestId": "1212122", "message": "Dat"}
What could be the solution for this?
Hy, I'm created a AmazonMQ using broker as RabbitMQ. Now I want to publish a message and read that message from the queue using python. So I followed the steps given in the AWS docs.
Link: Using Python Pika with Amazon MQ for RabbitMQ
I follow the same steps in docs but when I tried to send a message to the queue it gives me this error!
Traceback (most recent call last):
File "E:/Axcer/Using-Python-Pika-with-Amazon-MQ-for-RabbitMQ/publisher.py", line 26, in
basic_message_sender = BasicMessageSender(
File "E:\Axcer\Using-Python-Pika-with-Amazon-MQ-for-RabbitMQ\basicClient.py", line 15, in init
parameters = pika.URLParameters(url)
File "E:\Axcer\Using-Python-Pika-with-Amazon-MQ-for-RabbitMQ\Intern\lib\site-packages\pika\connection.py", line 757, in init
if parts.port is not None:
File "C:\Users\Yomal\Python38-32\lib\urllib\parse.py", line 174, in port
raise ValueError(message) from None
ValueError: Port could not be cast to integer value as 'xxxxxyyyy'
I have really no Idea about this issue! I hope that someone can help me with this. Thank you!
basicClient.py
import ssl
import pika
class BasicPikaClient:
def __init__(self, rabbitmq_broker_id, rabbitmq_user, rabbitmq_password, region):
# SSL Context for TLS configuration of Amazon MQ for RabbitMQ
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.set_ciphers('ECDHE+AESGCM:!ECDSA')
url = f"amqps://{rabbitmq_user}:{rabbitmq_password}#{rabbitmq_broker_id}.mq.{region}.amazonaws.com:5671"
parameters = pika.URLParameters(url)
parameters.ssl_options = pika.SSLOptions(context=ssl_context)
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
publisher.py
from basicClient import BasicPikaClient
class BasicMessageSender(BasicPikaClient):
def declare_queue(self, queue_name):
print(f"Trying to declare queue({queue_name})...")
self.channel.queue_declare(queue=queue_name)
def send_message(self, exchange, routing_key, body):
channel = self.connection.channel()
channel.basic_publish(exchange=exchange,
routing_key=routing_key,
body=body)
print(f"Sent message. Exchange: {exchange}, Routing Key: {routing_key}, Body: {body}")
def close(self):
self.channel.close()
self.connection.close()
if __name__ == "__main__":
# Initialize Basic Message Sender which creates a connection
# and channel for sending messages.
basic_message_sender = BasicMessageSender(
"*******************",
"xxxxx",
"xxxxxyyyy#zzzzzzz",
"********"
)
# Declare a queue
basic_message_sender.declare_queue("hello world queue")
# Send a message to the queue.
basic_message_sender.send_message(exchange="", routing_key="hello world queue", body=b'Hello World!')
# Close connections.
basic_message_sender.close()
I have installed pip install pika==0.11.0, as it is shown on the readme file, and after that I tried python receiver.py, and it gave me the following error:
Traceback (most recent call last):
File "receive.py", line 9, in <module>
channel.queue_declare(queue='hello')
File "C:\Python36\lib\site-packages\pika\adapters\blocking_connection.py", line 2400, in queue_declare
self._flush_output(declare_ok_result.is_ready)
File "C:\Python36\lib\site-packages\pika\adapters\blocking_connection.py", line 1258, in _flush_output
raise exceptions.ChannelClosed(method.reply_code, method.reply_text)
pika.exceptions.ChannelClosed: (406, "PRECONDITION_FAILED - inequivalent arg 'durable' for queue 'hello' in vhost '/': received 'false' but current is 'true'")
Even if I didn't modify anything. I tried also the spring-amqp example, and these works. Can you gave me a hint about what should I change?
These is the py fille:
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='hello')
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
channel.basic_consume(callback,
queue='hello',
no_ack=True)
print(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()
The error is at thee following line:
channel.queue_declare(queue='hello')
The queue hello in the default vhost / already exists but it is declared as durable. Either delete this queue before you execute your code or declare it as durable from Python:
channel.queue_declare(queue='hello', durable=True)
See the section 'Message durability' of the tutorial.
# This script will publish MQ message to my_exchange MQ exchange
import pika
#credentials = pika.PlainCredentials('the_user', 'the_pass')
#connection = #pika.BlockingConnection(pika.ConnectionParameters('localhost', 5672, '/', pika.PlainCredentials('guest', 'guest')))
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost', 5672, 'test', pika.PlainCredentials('admin', 'password1243')))
channel = connection.channel()
#channel.queue_declare(queue='items-queue', durable=Ture)
channel.queue_declare(queue='dx-naas', durable=True)
channel.basic_publish(exchange='my-exchange', routing_key='my-routing-key', body='Hello World!')
print("[x] Sent 'Hello World!'")
connection.close()
I am using Kombu in Python to consume a durable RabbitMQ queue.
There is only one consumer consuming the queue in Windows. This consumer produces the below error:
Traceback (most recent call last):
File ".\consumer_windows.py", line 66, in <module>
message.ack()
File "C:\Users\Administrator\Anaconda2\lib\site-packages\kombu\message.py", line 88, in ack
self.channel.basic_ack(self.delivery_tag)
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\channel.py", line 1584, in basic_ack
self._send_method((60, 80), args)
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\abstract_channel.py", line 56, in _send_method
self.channel_id, method_sig, args, content,
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\method_framing.py", line 221, in write_method
write_frame(1, channel, payload)
File "C:\Users\Administrator\Anaconda2\lib\site-packages\amqp\transport.py", line 182, in write_frame
frame_type, channel, size, payload, 0xce,
File "C:\Users\Administrator\Anaconda2\lib\socket.py", line 228, in meth
return getattr(self._sock,name)(*args)
error: [Errno 10054] An existing connection was forcibly closed by the remote host
There are at most 500 messages in the queue at any one time. Each message is small in size however it is a task and takes up to 10 minutes to complete (although it usually takes less then 5 mins per message).
I have tried restarting the consumer, RabbitMQ server and deleting the queue however the error still persists.
I've seen this question however the answer is from 2010 and my rabbitmq.log has different entries:
=ERROR REPORT==== 24-Apr-2016::08:26:20 ===
closing AMQP connection <0.6716.384> (192.168.X.X:59602 -> 192.168.Y.X:5672):
{writer,send_failed,{error,timeout}}
There were no recent events in the rabbitmq-sasl.log.
Why is this error happening and how can I prevent it from occurring?
I'm still looking for an answer. In the meantime I restart the connection to my rabbit server:
while True:
try:
connection = pika.BlockingConnection(params)
channel = connection.channel() # start a channel
channel.queue_declare(queue=amqp_q, durable=True) # Declare a queue
...
except pika.exceptions.ConnectionClosed:
print('connection closed... and restarted')
I had the same issue with MySQL server which was hosted...
I came to understand that it happened if we open the connection for a long time or unmodified for a long time..
If your program opens the DB or anything until the whole program runs make it in a such a way that it opens the DB writes everything and closes and repeats
I don't know what exactly rabbitmq is but I think the error you wrote as title may be for this reason
I had the same error (using pure PIKA library) and trying to connect to a Rabbitmq broker through Amazon MQ.
The problem resolved when setting up correctly the ssl configuration.
Please check full blog post here: https://docs.aws.amazon.com/amazon-mq/latest/developer-guide/amazon-mq-rabbitmq-pika.html
Core snippets that I used:
Define Pika Client:
import ssl
import pika
class BasicPikaClient:
def __init__(self, rabbitmq_broker_id, rabbitmq_user, rabbitmq_password, region):
# SSL Context for TLS configuration of Amazon MQ for RabbitMQ
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.set_ciphers('ECDHE+AESGCM:!ECDSA')
url = f"amqps://{rabbitmq_user}:{rabbitmq_password}#{rabbitmq_broker_id}.mq.{region}.amazonaws.com:5671"
parameters = pika.URLParameters(url)
parameters.ssl_options = pika.SSLOptions(context=ssl_context)
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
Producer:
from basicClient import BasicPikaClient
class BasicMessageSender(BasicPikaClient):
def declare_queue(self, queue_name, durable):
print(f"Trying to declare queue({queue_name})...")
self.channel.queue_declare(queue=queue_name, durable=durable)
def send_message(self, exchange, routing_key, body):
channel = self.connection.channel()
channel.basic_publish(exchange=exchange,
routing_key=routing_key,
body=body)
print(f"Sent message. Exchange: {exchange}, Routing Key: {routing_key}, Body: {body}")
def close(self):
self.channel.close()
self.connection.close()
Calling Producer:
# Initialize Basic Message Sender which creates a connection
# and channel for sending messages.
basic_message_sender = BasicMessageSender(
credentials["broker_id"],
credentials["username"],
credentials['password'],
credentials['region']
)
# Declare a queue
basic_message_sender.declare_queue("q_name", durable=True)
# Send a message to the queue.
basic_message_sender.send_message(exchange="", routing_key="q_name", body=b'Hello World 2!')
# Close connections.
basic_message_sender.close()
Define Consumer:
class BasicMessageReceiver(BasicPikaClient):
def get_message(self, queue):
method_frame, header_frame, body = self.channel.basic_get(queue)
if method_frame:
print(method_frame, header_frame, body)
self.channel.basic_ack(method_frame.delivery_tag)
return method_frame, header_frame, body
else:
print('No message returned')
def close(self):
self.channel.close()
self.connection.close()
Calling Consumer:
# Create Basic Message Receiver which creates a connection
# and channel for consuming messages.
basic_message_receiver = BasicMessageReceiver(
credentials["broker_id"],
credentials["username"],
credentials['password'],
credentials['region']
)
# Consume the message that was sent.
basic_message_receiver.get_message("q_name")
# Close connections.
basic_message_receiver.close()
I hope the above helps.
Thanks
I have installed the plugin to send a delayed message from here rabbitmq-delayed-message-exchange.
I couldn't find any help for using it in python. I've just started using rabbitmq .
Here is what I've been trying:
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.exchange_declare("test-x", type="x-delayed-message", arguments={"x-delayed-type":"direct"})
channel.queue_declare(queue='task_queue',durable=True)
channel.queue_bind(queue="task_queue", exchange="test-x", routing_key="task_queue")
channel.basic_publish(exchange='test-x',routing_key='task_queue',body='Hello World! Delayed',arguments={"x-delay":100})
print(" [x] Sent 'Hello World! Delayed'")
connection.close()
Here are the exchanges listed:
sudo rabbitmqctl list_exchanges
Listing exchanges ...
amq.direct direct
test-x x-delayed-message
amq.fanout fanout
amq.match headers
amq.headers headers
direct
amq.rabbitmq.trace topic
amq.topic topic
amq.rabbitmq.log topic
I don't have a good idea how I could pass a delay argument to the basic_publish function
Any help is appreciated
You need to add the x-delay header to your message properties and specify the delay value in milliseconds. Try this:
channel.basic_publish(
exchange='test-x',
routing_key='task_queue',
body='Hello World! Delayed',
properties=pika.BasicProperties(headers={"x-delay": 1000})
)
you can actually delay message without using plugin.
Message in Rabbit queue can be delayed in 2 ways
- using QUEUE TTL
- using Message TTL
If all messages in queue are to be delayed for fixed time use queue TTL.
If each message has to be delayed by varied time use Message TTL.
I have explained it using python3 and pika module.
pika BasicProperties argument 'expiration' in milliseconds has to be set to delay message in delay queue.
After setting expiration time, publish message to a delayed_queue ("not actual queue where consumers are waiting to consume") , once message in delayed_queue expires, message will be routed to a actual queue using exchange 'amq.direct'
def delay_publish(self, messages, queue, headers=None, expiration=0):
"""
Connect to RabbitMQ and publish messages to the queue
Args:
queue (string): queue name
messages (list or single item): messages to publish to rabbit queue
expiration(int): TTL in milliseconds for message
"""
delay_queue = "".join([queue, "_delay"])
logging.info('Publishing To Queue: {queue}'.format(queue=delay_queue))
logging.info('Connecting to RabbitMQ: {host}'.format(
host=self.rabbit_host))
credentials = pika.PlainCredentials(
RABBIT_MQ_USER, RABBIT_MQ_PASS)
parameters = pika.ConnectionParameters(
rabbit_host, RABBIT_MQ_PORT,
RABBIT_MQ_VHOST, credentials, heartbeat_interval=0)
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue=queue, durable=True)
channel.queue_bind(exchange='amq.direct',
queue=queue)
delay_channel = connection.channel()
delay_channel.queue_declare(queue=delay_queue, durable=True,
arguments={
'x-dead-letter-exchange': 'amq.direct',
'x-dead-letter-routing-key': queue
})
properties = pika.BasicProperties(
delivery_mode=2, headers=headers, expiration=str(expiration))
if type(messages) not in (list, tuple):
messages = [messages]
try:
for message in messages:
try:
json_data = json.dumps(message)
except Exception as err:
logging.error(
'Error Jsonify Payload: {err}, {payload}'.format(
err=err, payload=repr(message)), exc_info=True
)
if (type(message) is dict) and ('data' in message):
message['data'] = {}
message['error'] = 'Payload Invalid For JSON'
json_data = json.dumps(message)
else:
raise
try:
delay_channel.basic_publish(
exchange='', routing_key=delay_queue,
body=json_data, properties=properties)
except Exception as err:
logging.error(
'Error Publishing Data: {err}, {payload}'.format(
err=err, payload=json_data), exc_info=True
)
raise
except Exception:
raise
finally:
logging.info(
'Done Publishing. Closing Connection to {queue}'.format(
queue=delay_queue
)
)
connection.close()