We Keep Coding

MQTT Subscriber Failed to Subscribe to the Same Topic With MQTT Publisher

I'm trying to make a simple "transit schedule changes" program using python MQTT where the publisher can input flight number (will be used as topic) and new flight time, while the transit location will be picked random from given list.
The subscriber will have to input flight number too which will be used as topic. But in my codes, it looks like the subscriber failed to get the message published to the same topic because it keeps on printing Connected Successfully (I'm using client.loop_forever()). Can someone please help me to figure out what's wrong with my code?
This is my first time asking question, so please ask me if something isn't clear from my explanation. Thank you so much :)
Publisher:
import paho.mqtt.client as mqtt
import time
from datetime import datetime, date
import random
def on_connect(client, userdata, flags, rc):
if (rc==0):
global connected
connected = True
#print("Successfully Connected.")
client.on_publish = on_publish
else:
print("Failed to connect.")
def on_publish(client, userdata, mid):
print("Published successfully. MID: "+str(mid))
listTransit = ["Singapura", "Qatar", "Korea Selatan", "Turki", "Republik Tiongkok",
"Amerika Serikat", "Jepang", "Uni Emirat Arab", "Oman", "Islandia"]
broker_address="broker.emqx.io"
client = mqtt.Client("Publisher")
client.on_connect = on_connect
client.connect(broker_address, port=1883)
client.loop_start()
topic = input("Masukkan nomor penerbangan: ")
negaraTujuan = input("negara tujuan: ")
print("Masukkan waktu penerbangan baru (Format: [jam::menit::detik])")
str_time = input()
Date = date.today()
Time = datetime.strptime(str_time, '%H::%M::%S').time()
Location = random.randrange(0,len(listTransit))
if (listTransit[Location] != negaraTujuan):
message = Date.strftime("%Y/%m/%d")+"\nTujuan: "+negaraTujuan+"\nLokasi Transit : "+listTransit[Location]+"\nJam terbang : "+Time.strftime("%H:%M:%S")
client.publish(topic, message)
print("Topic: ",topic)
print(message)
else:
while listTransit[Location] == negaraTujuan:
Location = random.randrange(0,len(listTransit))
message = Date.strftime("%Y/%m/%d")+"\nTujuan: "+negaraTujuan+"\nLokasi Transit : "+listTransit[Location]+"\nJam terbang : "+Time.strftime("%H:%M:%S")
client.publish(topic, message)
print(message)
client.loop_stop()
Subscriber:
import paho.mqtt.client as mqtt
import time
from datetime import datetime, datetime
import re
def on_connect(client, userdata, flags, rc):
if (rc == 0):
print("Connected successfully.")
#global topic
#topic = input("Masukkan nomor penerbangan anda: ")
#client.subscribe(topic)
else:
print("Connection failed.")
def on_message(client, userdata, msg):
print("on_message callback function activated.")
sched = str(msg.payload.decode("utf-8"))
print(sched)
def on_subscribe(client, userdata, mid, granted_qos):
print("Subscribed to "+topic+" successfully")
broker_address="broker.emqx.io"
topic = input("Masukkan nomor penerbangan anda: ")
negaraTujuan = input("negara tujuan: ")
client = mqtt.Client("Subscriber")
client.subscribe(topic)
client.on_connect = on_connect
client.on_message = on_message
client.on_subscribe = on_subscribe
client.connect(broker_address, port=1883)
client.loop_forever()
what I got from running both codes are
Masukkan nomor penerbangan: YT05TA
negara tujuan: Australia
Masukkan waktu penerbangan baru (Format: [jam::menit::detik])
12::50::00
Topic: YT05TA
2023/01/03
Tujuan: Australia
Lokasi Transit : Amerika Serikat
Jam terbang : 12:50:00
Published successfully. MID: 1
from Publisher
and
Masukkan nomor penerbangan anda: YT05TA
negara tujuan: Australia
Connected successfully.
Connected successfully.
Connected successfully.
Connected successfully.
from subscriber. It doesn't even print the print("on_message callback function activated.")

You are using a public broker, so that means it is likely to have LOTS of over clients.
Every client MUST have a UNIQUE Client ID, so using Publisher and Subscriber is very likely to clash with other clients.
The MQTT specification says the broker must disconnect the currently connected client when a new client connects with the same Client ID. As most client libraries will try and reconnect when they are disconnects this leads to a battle between the two clients to stay connected.
Change them both to random values

How to run multile device under same paho-mqtt script

I am writing a script which will log more than one device with different credentials using paho-mqtt. All the client is running in the same address and with the same port. If I change the username and pass then I get the different feeds depending upon the credentials. It's working fine if I write for different devices different script. But I want to log all the devices in on_connet. I have written the script but it's working only for one device. Here is the script:
import paho.mqtt.client as mqtt
import time, json, threading, logging,ssl
clients=[
{"ursername":"username1","password": 'password1'},
{"ursername":"username2","password":'password2'},
{"ursername":"username3","password":'password3'}
]
nclients=len(clients)
run = True
def Create_connections():
for i in range(nclients):
t=int(time.time())
client_id = "client" + str(t)
client = mqtt.Client(client_id)
username = credentials[i]["ursername"]
password = credentials[i]["password"]
client.on_log=on_log
client.on_connect = on_connect
client.on_subscribe=on_subscribe
client.on_message = on_message
client.on_disconnect = on_disconnect
print("connecting to broker")
client.tls_set("CXXXXX.crt", tls_version=ssl.PROTOCOL_TLSv1_2)
client.tls_insecure_set(True)
client.username_pw_set(username, password)
client.loop_start()
client.connect("XXXXX", XXXX, XX)
print("Loop pass ")
def on_log(client, userdata, level, buf):
print("message:" + str(buf))
print("userdata:" + str(userdata))
def on_message(client, userdata, message):
msg="message received",str(message.payload.decode("utf-8"))
print(msg)
def on_connect(client, userdata, flags, rc):
print("Connected with result code:"+str(rc))
client.subscribe('v3/+/devices/+/up')
def on_disconnect(client, userdata, rc):
pass
def on_publish(client, userdata, mid):
print("mid: " + str(mid) + '\n')
def on_subscribe(mosq, obj, mid, granted_qos):
print("Subscribed: " + str(mid) + " " + str(granted_qos))
mqtt.Client.connected_flag=False
no_threads=threading.active_count()
print("current threads =",no_threads)
print("Creating Connections ",nclients," clients")
Create_connections()
Response
current threads = 1
Creating Connections 2 clients
____________________________________________________________________________
client01597648398
<paho.mqtt.client.Client object at 0x7f73c0a5b1d0>
username1
connecting to broker
message:Sending CONNECT (u1, p1, wr0, wq0, wf0, c1, k60) client_id=b'client01597648398'
userdata:None
message:Received CONNACK (0, 0)
userdata:None
Connected with result code:0
message:Sending SUBSCRIBE (d0, m1) [(b'XX/+/devices/+/XX', 0)]
userdata:None
message:Received SUBACK
userdata:None
Subscribed: 1 (0,)
message:Sending PINGREQ
userdata:None
message:Received PINGRESP
userdata:None
Any help will be highly appreciated. Thanks in advance

Your problem lies here:
while run:
client.loop_forever()
loop_forever() is a blocking call which will only return when the associated client is disconnected:
This is a blocking form of the network loop and will not return until the client calls disconnect(). It automatically handles reconnecting. paho-mqtt
So your other clients never get initialized/connected. You might want to use loop_start instead - this will use a separate thread to handle communication for every client:
def Create_connections():
for i in range(nclients):
# ...
client.loop_start()
client.connect("XXXXXXXXXX", XXXX, XX)
# ...
Create_connections()
while run:
pass

Why does the paho python client publish short messages but not long ones

This works :
while True:
print('')
command_input = input()
if command_input == 'q':
break
mcp = Mqtt_command_publisher
mcp.publish_command(device_ids, command_input)
But this does not:
class Mqtt_command_bl:
def update_minutes_to_run_at(json):
if not json['device_ids']:
return 'Request must contain device ids'
device_ids = json['device_ids']
minutes_to_run_at = json['minutes_to_run_at']
minutes_to_run_at_command_section = ''
for i in minutes_to_run_at:
m = '"{}",'.format(i)
if i == minutes_to_run_at[len(minutes_to_run_at) - 1]:
m = '"{}"'.format(i)
minutes_to_run_at_command_section += m
#command_input = 'jq \'.+{{minutes_to_run_at:[{}]}}\' /home/pi/hallmonitor_lite/config.json > /home/pi/hallmonitor_lite/tmp.json && mv /home/pi/hallmonitor_lite/tmp.json /home/pi/hallmonitor_lite/new_config.json'.format(minutes_to_run_at_command_section)
command_input = 'mkdir /home/pi/hallmonitor_lite/hello_world'
mcp = Mqtt_command_publisher
mcp.publish_command(device_ids, command_input)
return 'Success'
The class they both call:
class Mqtt_command_publisher:
def publish_command(device_ids, command_input):
mqtt_msg = json.dumps({'device_ids':device_ids,'command':command_input})
print('\n{}'.format(mqtt_msg))
client = mqtt.Client()
client.connect('********', ****, 30)
client.publish('topic/commands', mqtt_msg)
client.disconnect()
Looking at the print statements output from the Mqtt_command_publisher, the output can be the exact same, however, only one of them will execute, and I don't see why one works and the other does not.
I tried this command for testing: mkdir /home/pi/hallmonitor_lite/hello_world
This is the receiving part:
device_id = 0
with open('/home/pi/hallmonitor_lite/config.json') as json_data_file:
data = json.load(json_data_file)
device_id = data['device_id']
def on_connect(client, userdata, flags, rc):
print("Connected with result code: " + str(rc))
client.subscribe("topic/commands")
def on_message(client, userdata, msg):
mqtt_message = msg.payload.decode()
print(mqtt_message)
ids_and_command = json.loads(mqtt_message)
if str(device_id) in ids_and_command['device_ids'] or not ids_and_command['device_ids']:
print(('Executing: {}').format(ids_and_command['command']))
os.system(ids_and_command['command'])
client = mqtt.Client()
client.connect("********", ****, 30)
client.on_connect = on_connect
client.on_message = on_message
client.loop_forever()
Any ideas?

The problem is most likely because the second set of code is creating a message bigger than will fit in a single TCP packet.
This is a problem because you are not running the client network loop so the client.publish command can only send a single packet, the rest of the message would have been sent by the network loop, but even if it was running you are calling disconnect immediately after the publish call.
The client is not meant to be spun up for a single message like that, it is meant to be started and then left running with you just calling the publish method when you want to send a message. If you don't want to do that or can't for some reason there is a specific helper class in the paho python package that will do all the heavy lifting of starting the client, sending the message and then tearing everything down nicely. The docs for the single publish are here.
import paho.mqtt.publish as publish
publish.single("paho/test/single", "payload", hostname="mqtt.eclipse.org")

Consuming and replying to separate queues | Pika implementation

I am struggling to solve my issue, hope anyone from the community can help me here.
Our requirement is locked and can't be changed as the producer publishing the queues is controlled by a different team.
Producer which is written in JAVA declares three queues (TASK, RESPONSE, TASK_RESPONSE) and listens on them with the help of spring framework.
A hashmap is sent to the TASK and TASK_RESPONSE queue from the java AMQP client (Producer).
We need to consume these hashmaps and send the responses as follows.
If the queue TASK is processed, the response needs to be sent on RESPONSE queue incrementally.
If the queue TASK_RESPONSE is processed, the response needs to be sent on TASK_RESPONSE queue incrementally (RPC mode).
Now, we need to consume and publish this in python since we need to do some background processing on the tasks.
I tried to work with celery and dramatiq, but was not able to figure out how it can be done with them, so I tried writing myself (with the help of tutorials available online)
Problem is, I am able to consume the messages but not able to reply_to the RESPONSE queue. Here is my code.
from collections import OrderedDict
from concurrent.futures import ThreadPoolExecutor
import pika
import datetime
import logging
import json
from logging import StreamHandler
from time import sleep
from random import randint
from pika import SelectConnection
from settings import *
logging.basicConfig(handlers=[StreamHandler()], level=logging.INFO, format=logging.BASIC_FORMAT)
_logger = logging.getLogger(__name__)
class QueueConsumer(object):
"""The consumer class to manage connections to the AMQP server/queue"""
def __init__(self, queue, logger, parameters, thread_id=0):
self.channel = None
self.connection = None
self.queue_name_task = queue['task']
self.queue_name_response = queue['response']
self.logger = logger
self.consumer_id = 'Consumer Thread: %d' % (thread_id,)
self.parameters = pika.ConnectionParameters(**parameters)
def consume(self):
try:
self.connection = SelectConnection(parameters=self.parameters, on_open_callback=self._on_connected)
self.connection.ioloop.start()
except Exception as e:
self.logger.error('{} {}'.format(self.consumer_id, str(e)))
self.connection.close()
self.connection.ioloop.start()
def _on_connected(self, connection):
connection.channel(on_open_callback=self._on_channel_open)
def _on_channel_open(self, channel):
self.channel = channel
try:
# Declare Task Queue
self.channel.queue_declare(queue=self.queue_name_task,
exclusive=False,
durable=True,
auto_delete=False,
callback=self._on_queue_declared)
self.logger.info("{} Opened Channel....".format(self.consumer_id))
# Declare Task Response Queue
self.channel.queue_declare(queue=self.queue_name_response,
exclusive=False,
durable=True,
auto_delete=False)
self.logger.info("{} Opened Channel....".format(self.consumer_id))
except Exception as e:
self.logger.error('{} {}'.format(self.consumer_id, str(e)))
def _on_queue_declared(self, frame):
self.logger.debug('{} ... declaring queue'.format(self.consumer_id))
self.channel.basic_qos(prefetch_count=1)
try:
self.channel.basic_consume(queue=self.queue_name_task,
on_message_callback=self.handle_delivery,
auto_ack=True)
self.logger.info("{} Declared queue...".format(self.consumer_id))
except Exception as e:
self.logger.error('{} crashing:--> {}'.format(self.consumer_id, str(e)))
def handle_delivery(self, channel, method, header, body):
try:
start_time = datetime.datetime.now()
_logger.info("Received...")
_logger.info("Content: %s" % body)
req = json.loads(self.decode(body))
# Do something
sleep(randint(10, 20))
time_taken = datetime.datetime.now() - start_time
log_msg = "[{}] Time Taken: {}.{}".format(req['bar']['baz'], time_taken.seconds, time_taken.microseconds)
_logger.info(log_msg)
# Publish the result to another queue.
try:
self.channel.basic_publish(exchange='',
routing_key=self.queue_name_response,
properties=pika.BasicProperties(),
body=log_msg)
_logger.info("Message Published...\t(%s)" % self.queue_name_response)
except Exception as e:
self.logger.error('{} Message publishing failed:--> {}'.format(self.consumer_id, str(e)))
except Exception as err:
_logger.exception(err)
def decode(self, body):
try:
_body = body.decode('utf-8')
except AttributeError:
_body = body
return _body
if __name__ == "__main__":
pika_parameters = OrderedDict([
('host', TF_BROKER_HOST),
('port', TF_BROKER_PORT),
('virtual_host', TF_BROKER_VHOST)
])
queue = {'task': TF_IAAS_TASK_QUEUE, 'response': TF_IAAS_REPLY_QUEUE}
try:
with ThreadPoolExecutor(max_workers=TF_IAAS_THREAD_SIZE, thread_name_prefix=TF_IAAS_THREAD_PREFIX) as executor:
start = 1
for thread_id in range(start, (TF_IAAS_THREAD_SIZE + start)):
executor.submit(QueueConsumer(queue, _logger, pika_parameters, thread_id).consume)
except Exception as err:
_logger.exception(err)
Publish Messages On RabbitMQ
import pika
import json
import random
import datetime
from faker import Faker
from random import randint
fake = Faker('en_US')
if __name__ == '__main__':
try:
connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='tf_task', durable=True)
started_at = datetime.datetime.now()
properties = pika.BasicProperties(delivery_mode=2)
for i in range(0, 10000):
body = {
'foo': randint(i, i+100),
'bar': {
'baz': fake.name(),
'poo': float(random.randrange(155+i, 389+i))/100
}
}
channel.basic_publish(exchange='',
routing_key='tf_task',
body=json.dumps(body),
properties=properties)
if i%10000 == 0:
duration = datetime.datetime.now() - started_at
print(i, duration.total_seconds())
print(" [x] Sent 'Hello World!'")
connection.close()
now = datetime.datetime.now()
duration = now - started_at
print(duration.total_seconds())
except Exception as e:
print(e)

Python, websocket auto-close on some machines

I've written some api to communicate with a website using websocketapp. It works fine only on 2 pc. If i put my code on every other pc the websocket doesnt receive any message and closes. I've tried a lot of different machines and operating systems, many version of python (included the same that works), wireless and wired connection but nothing changed. There's no error or exception. What can it be?
EDIT: i don't own the website or the server. All other methods send messages and parse the response in on_socket_message
import requests
import websocket
import time
from threading import Thread
from datetime import datetime
import json
from position import Position
from constants import ACTIVES
class IQOption():
practice_balance = 0
real_balance = 0
server_time = 0
positions = {}
instruments_categories = ["cfd","forex","crypto"]
top_assets_categories = ["forex","crypto","fx-option"]
instruments_to_id = ACTIVES
id_to_instruments = {y:x for x,y in ACTIVES.items()}
market_data = {}
binary_expiration_list = {}
open_markets = {}
digital_strike_list = {}
candle_data = []
latest_candle = 0
position_id = 0
quotes =[]
position_id_list=[]
def __init__(self,username,password,host="iqoption.com"):
self.username = username
self.password = password
self.host = host
self.session = requests.Session()
self.generate_urls()
self.socket = websocket.WebSocketApp(self.socket_url,on_open=self.on_socket_connect,on_message=self.on_socket_message,on_close=self.on_socket_close,on_error=self.on_socket_error)
def generate_urls(self):
"""Generates Required Urls to operate the API"""
#https://auth.iqoption.com/api/v1.0/login
self.api_url = "https://{}/api/".format(self.host)
self.socket_url = "wss://{}/echo/websocket".format(self.host)
self.login_url = self.api_url+"v1.0/login"
self.profile_url = self.api_url+"profile"
self.change_account_url = self.profile_url+"/"+"changebalance"
self.getprofile_url = self.api_url+"getprofile"
def login(self):
"""Login and set Session Cookies"""
print("LOGIN")
data = {"email":self.username,"password":self.password}
self.log_resp = self.session.request(url="https://auth.iqoption.com/api/v1.0/login",data=data,method="POST")
requests.utils.add_dict_to_cookiejar(self.session.cookies, dict(platform="9"))
self.__ssid = self.log_resp.cookies.get("ssid")
print(self.__ssid)
self.start_socket_connection()
time.sleep(1) ## artificial delay to complete socket connection
self.log_resp2 = self.session.request(url="https://eu.iqoption.com/api/getprofile",method="GET")
ss = self.log_resp2._content.decode('utf-8')
js_ss=json.loads(ss)
self.parse_account_info(js_ss)
self.balance_id = js_ss["result"]["balance_id"]
self.get_instruments()
self.get_top_assets()
self.setOptions()
#self.getFeatures()
time.sleep(1)
print(js_ss["isSuccessful"])
return js_ss["isSuccessful"]
def on_socket_message(self,socket,message):
#do things
def on_socket_connect(self,socket):
"""Called on Socket Connection"""
self.initial_subscriptions()
print("On connect")
def initial_subscriptions(self):
self.send_socket_message("ssid",self.__ssid)
self.send_socket_message("subscribe","tradersPulse")
def on_socket_error(self,socket,error):
"""Called on Socket Error"""
print(message)
def on_socket_close(self,socket):
"""Called on Socket Close, does nothing"""
def start_socket_connection(self):
"""Start Socket Connection"""
self.socket_thread = Thread(target=self.socket.run_forever)
self.socket_thread.start()
def send_socket_message(self,name,msg):
#print(msg)
data = {"name":name,"msg":msg}
self.socket.send(json.dumps(data))

Here is an example running under Gevent Websockets. This makes it ASYNC (which I suspect is part of your problem) and allows for bidirectional communication.
import gevent
from gevent import monkey, signal, Timeout, sleep, spawn as gspawn
monkey.patch_all()
from gevent.pywsgi import WSGIServer
from geventwebsocket.handler import WebSocketHandler
from geventwebsocket import WebSocketError
import bottle
from bottle import get, route, template, request, response, abort, static_file
import ujson as json
#route('/static/<filepath:path>')
def server_static(filepath):
return static_file(filepath, root='static')
#route('/ws/remote')
def handle_websocket():
wsock = request.environ.get('wsgi.websocket')
if not wsock:
abort(400, 'Expected WebSocket request.')
while 1:
try:
message = ''
with Timeout(2, False) as timeout:
message = wsock.receive()
if message:
message = json.loads(message)
if 'command' in message:
r.command(message['command'])
except WebSocketError:
break
except Exception as exc:
print(str(exc))
#get('/')
def remote():
return template('templates/remote.tpl', title='WebsocketTest', websocket=WEBSOCKET, command='command', status=status)
if __name__ == '__main__':
r=None
status="Connecting..."
gspawn(initialize)
print 'Started...'
HOST = socket.gethostbyname(socket.gethostname())
HOST = 'localhost'
WEBSOCKET = 'ws://{}/ws/remote'.format(HOST)
botapp = bottle.app()
server = WSGIServer(("0.0.0.0", 80), botapp, handler_class=WebSocketHandler)
def shutdown():
print('Shutting down ...')
server.stop(timeout=60)
exit(signal.SIGTERM)
gevent.signal(signal.SIGTERM, shutdown)
gevent.signal(signal.SIGINT, shutdown) #CTRL C
server.serve_forever()
Then in your HTML you really should use reconnecting websocket library
https://github.com/joewalnes/reconnecting-websocket
<button id="TRIGGERED" type="button" class="btn btn-outline-primary">TRIGGER</button>
<script type="text/javascript" src="/static/reconnecting-websocket.min.js"></script>
<script>
var ws = new ReconnectingWebSocket('{{websocket}}');
ws.reconnectInterval = 3000;
ws.maxReconnectAttempts = 10;
ws.onmessage = function (evt) {
var wsmsg = JSON.parse(evt.data);
console.log(evt.data)
};
$("button").click(function() {
<!--console.log(this.id);-->
ws.send(JSON.stringify({'{{command}}': this.id}));
});
</script>