It seems there is not very much support for what I am trying to do, but it is supposed to be possible since it is demonstrated in temperature sensor and sensor filter tutorial. However, there are no examples for the actual message creation from an edge module in python. That tutorial only shows forwarding messages. There are examples of sending from a device, but devices use a different class than edge modules. From the filter example and from a couple of device examples I have pieced together the following:
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for
# full license information.
import random
import time
import sys
import iothub_client
from iothub_client import IoTHubModuleClient, IoTHubClientError, IoTHubTransportProvider
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError
# messageTimeout - the maximum time in milliseconds until a message times out.
# The timeout period starts at IoTHubModuleClient.send_event_async.
# By default, messages do not expire.
MESSAGE_TIMEOUT = 10000
# global counters
RECEIVE_CALLBACKS = 0
SEND_CALLBACKS = 0
# Choose HTTP, AMQP or MQTT as transport protocol. Currently only MQTT is supported.
PROTOCOL = IoTHubTransportProvider.MQTT
# Callback received when the message that we're forwarding is processed.
def send_confirmation_callback(message, result, user_context):
global SEND_CALLBACKS
print ( "Confirmation[%d] received for message with result = %s" % (user_context, result) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
SEND_CALLBACKS += 1
print ( " Total calls confirmed: %d" % SEND_CALLBACKS )
# receive_message_callback is invoked when an incoming message arrives on the specified
# input queue (in the case of this sample, "input1"). Because this is a filter module,
# we will forward this message onto the "output1" queue.
def receive_message_callback(message, hubManager):
global RECEIVE_CALLBACKS
message_buffer = message.get_bytearray()
size = len(message_buffer)
print ( " Data: <<<%s>>> & Size=%d" % (message_buffer[:size].decode('utf-8'), size) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
RECEIVE_CALLBACKS += 1
print ( " Total calls received: %d" % RECEIVE_CALLBACKS )
hubManager.forward_event_to_output("output1", message, 0)
return IoTHubMessageDispositionResult.ACCEPTED
def construct_message(message_body, topic):
try:
msg_txt_formatted = message_body
message = IoTHubMessage(msg_txt_formatted)
# Add a custom application property to the message.
# An IoT hub can filter on these properties without access to the message body.
prop_map = message.properties()
prop_map.add("topic", topic)
# TODO Use logging
# Send the message.
print( "Sending message: %s" % message.get_string() )
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
return message
class HubManager(object):
def __init__(
self,
protocol=IoTHubTransportProvider.MQTT):
self.client_protocol = protocol
self.client = IoTHubModuleClient()
self.client.create_from_environment(protocol)
# set the time until a message times out
self.client.set_option("messageTimeout", MESSAGE_TIMEOUT)
# sets the callback when a message arrives on "input1" queue. Messages sent to
# other inputs or to the default will be silently discarded.
self.client.set_message_callback("input1", receive_message_callback, self)
# Forwards the message received onto the next stage in the process.
def forward_event_to_output(self, outputQueueName, event, send_context):
self.client.send_event_async(
outputQueueName, event, send_confirmation_callback, send_context)
def send_message(self, message):
# No callback
# TODO what is the third arg?
self.client.send_event_async(
"output1", message, send_confirmation_callback, 0)
self.client.send_message()
def mypublish(self, topic, msg):
message = construct_message(msg, topic)
self.send_message(message)
print('publishing %s', msg)
def main(protocol):
try:
print ( "\nPython %s\n" % sys.version )
print ( "IoT Hub Client for Python" )
hub_manager = HubManager(protocol)
print ( "Starting the IoT Hub Python sample using protocol %s..." % hub_manager.client_protocol )
print ( "The sample is now waiting for messages and will indefinitely. Press Ctrl-C to exit. ")
while True:
hub_manager.mypublish('testtopic', 'hello world this is a module')
time.sleep(1)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubModuleClient sample stopped" )
if __name__ == '__main__':
main(PROTOCOL)
When I build and deploy this it executes on the edge device without errors and in the log, the callback reports that the messages are sent ok. However, no messages come through when I attempt to monitor D2C messages.
I used this to create and send a message from a JSON dict.
new_message = json.dumps(json_obj)
new_message = IoTHubMessage(new_message)
hubManager.forward_event_to_output("output1", new_message, 0)
You can send anything you need, even strings or whatever.
To narrow down the issue, you can install the azureiotedge-simulated-temperature-sensor module published by Microsoft to see whether the issue relative to the Edge environment issue or coding.
I also wrote a sample Python code module based on the Python Module templates which works well for me, you can refer the code below:
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for
# full license information.
import random
import time
import sys
import iothub_client
from iothub_client import IoTHubModuleClient, IoTHubClientError, IoTHubTransportProvider
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError
# messageTimeout - the maximum time in milliseconds until a message times out.
# The timeout period starts at IoTHubModuleClient.send_event_async.
# By default, messages do not expire.
MESSAGE_TIMEOUT = 10000
# global counters
RECEIVE_CALLBACKS = 0
SEND_CALLBACKS = 0
# Choose HTTP, AMQP or MQTT as transport protocol. Currently only MQTT is supported.
PROTOCOL = IoTHubTransportProvider.MQTT
# Callback received when the message that we're forwarding is processed.
def send_confirmation_callback(message, result, user_context):
global SEND_CALLBACKS
print ( "Confirmation[%d] received for message with result = %s" % (user_context, result) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
SEND_CALLBACKS += 1
print ( " Total calls confirmed: %d" % SEND_CALLBACKS )
# receive_message_callback is invoked when an incoming message arrives on the specified
# input queue (in the case of this sample, "input1"). Because this is a filter module,
# we will forward this message onto the "output1" queue.
def receive_message_callback(message, hubManager):
global RECEIVE_CALLBACKS
message_buffer = message.get_bytearray()
size = len(message_buffer)
print ( " Data: <<<%s>>> & Size=%d" % (message_buffer[:size].decode('utf-8'), size) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
RECEIVE_CALLBACKS += 1
print ( " Total calls received: %d" % RECEIVE_CALLBACKS )
hubManager.forward_event_to_output("output1", message, 0)
return IoTHubMessageDispositionResult.ACCEPTED
class HubManager(object):
def __init__(
self,
protocol=IoTHubTransportProvider.MQTT):
self.client_protocol = protocol
self.client = IoTHubModuleClient()
self.client.create_from_environment(protocol)
# set the time until a message times out
self.client.set_option("messageTimeout", MESSAGE_TIMEOUT)
# sets the callback when a message arrives on "input1" queue. Messages sent to
# other inputs or to the default will be silently discarded.
self.client.set_message_callback("input1", receive_message_callback, self)
# Forwards the message received onto the next stage in the process.
def forward_event_to_output(self, outputQueueName, event, send_context):
self.client.send_event_async(
outputQueueName, event, send_confirmation_callback, send_context)
def SendSimulationData(self, msg):
print"sending message..."
message=IoTHubMessage(msg)
self.client.send_event_async(
"output1", message, send_confirmation_callback, 0)
print"finished sending message..."
def main(protocol):
try:
print ( "\nPython %s\n" % sys.version )
print ( "IoT Hub Client for Python" )
hub_manager = HubManager(protocol)
print ( "Starting the IoT Hub Python sample using protocol %s..." % hub_manager.client_protocol )
print ( "The sample is now waiting for messages and will indefinitely. Press Ctrl-C to exit. ")
while True:
hub_manager.SendSimulationData("test msg")
time.sleep(1)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubModuleClient sample stopped" )
if __name__ == '__main__':
main(PROTOCOL)
If it can help someone, I think you miss await send_message.
Seems the same problem I answered here
Related
I'm doing a telemetry application using Azure IoT Hub, Azure IoT SDK in Python and a raspberry pi with temperature and humidity sensors.
Humidity + Temperature sensors => Rasperry Pi => Azure IoT Hub
In my first implementation thanks azure examples, I used one loop that collect data from the temperature sensor and the humidity sensor, and send them to Azure IoT Hub in the same time every 60 second.
>>> 1 Loop every 60s = Collect data & send data of temperature and humidity
Now I would like to send them with different frequencies, I mean :
One loop will collect the data of the temperature sensor and send it to Azure IoT Hub every 60 seconds;
Whereas a second loop will collect the data of the humidity sensor and send it to Azure IoT Hub every 600 seconds.
>>> 1 Loop every 60s= Collect data & send data of temperature
>>> 2 Loop every 600s= Collect data & send data of humidity
I think the tool I need is multi-threading, but I don't understand which library or structure I have to implement in my case.
Here is the code provided by Azure, including one loop that handles temperature and humidity at the same time. Reading the data and sending to Azure every 60 seconds.
import random
import time
import sys
# Using the Python Device SDK for IoT Hub:
from iothub_client import IoTHubClient, IoTHubClientError,
IoTHubTransportProvider, IoTHubClientResult
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult,
IoTHubError, DeviceMethodReturnValue
# The device connection string to authenticate the device with your IoT hub.
CONNECTION_STRING = "{Your IoT hub device connection string}"
# Using the MQTT protocol.
PROTOCOL = IoTHubTransportProvider.MQTT
MESSAGE_TIMEOUT = 10000
# Define the JSON message to send to IoT Hub.
TEMPERATURE = 20.0
HUMIDITY = 60
MSG_TXT = "{\"temperature\": %.2f,\"humidity\": %.2f}"
def send_confirmation_callback(message, result, user_context):
print ( "IoT Hub responded to message with status: %s" % (result) )
def iothub_client_init():
# Create an IoT Hub client
client = IoTHubClient(CONNECTION_STRING, PROTOCOL)
return client
def iothub_client_telemetry_sample_run():
try:
client = iothub_client_init()
print ( "IoT Hub device sending periodic messages, press Ctrl-C to exit" )
#******************LOOP*******************************
while True:
# Build the message with simulated telemetry values.
temperature = TEMPERATURE + (random.random() * 15)
humidity = HUMIDITY + (random.random() * 20)
msg_txt_formatted = MSG_TXT % (temperature, humidity)
message = IoTHubMessage(msg_txt_formatted)
# Send the message.
print( "Sending message: %s" % message.get_string() )
client.send_event_async(message, send_confirmation_callback, None)
time.sleep(60)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubClient sample stopped" )
if __name__ == '__main__':
print ( "IoT Hub Quickstart #1 - Simulated device" )
print ( "Press Ctrl-C to exit" )
iothub_client_telemetry_sample_run()
I would like to use the same structure of functions, including two loops that handles temperature and humidity, one every 60s and one every 600s.
while True:
# Build the message with simulated telemetry values.
temperature = TEMPERATURE + (random.random() * 15)
msg_txt_formatted1 = MSG_TXT1 % (temperature)
message1 = IoTHubMessage(msg_txt_formatted1)
# Send the message.
print( "Sending message: %s" % message1.get_string() )
client.send_event_async(message1, send_confirmation_callback, None)
time.sleep(60)
while True:
# Build the message with simulated telemetry values.
humidity = HUMIDITY + (random.random() * 20)
msg_txt_formatted2 = MSG_TXT2 % (humidity)
message2 = IoTHubMessage(msg_txt_formatted2)
# Send the message.
print( "Sending message: %s" % message2.get_string() )
client.send_event_async(message2, send_confirmation_callback, None)
time.sleep(600)
How can I do that? How to call those loops with multi-threading or another method?
It may be simpler to do something like
while True:
loop_b()
for _ in range(10):
loop_a()
time.sleep(60)
or even
while True:
time.sleep(1)
now = time.time()
if now % 60 == 0:
loop_a()
if now % 600 == 0:
loop_b()
But if you really want to use threads, then:
import threading
class LoopAThread(threading.Thread):
def run(self):
loop_a()
class LoopBThread(threading.Thread):
def run(self):
loop_b()
...
thread_a = LoopAThread()
thread_b = LoopBThread()
thread_a.start()
thread_b.start()
thread_a.join()
thread_b.join()
Here are two competing approaches to consider
Don't bother with threads at all. Just have one loop that sleeps every 60 seconds like you have now. Keep track of the last time you sent humidity data. If 600 seconds has passed, then send it. Otherwise, skip it and go to sleep for 60 seconds. Something like this:
from datetime import datetime, timedelta
def iothub_client_telemetry_sample_run():
last_humidity_run = None
humidity_period = timedelta(seconds=600)
client = iothub_client_init()
while True:
now = datetime.now()
send_temperature_data(client)
if not last_humidity_run or now - last_humidity_run >= humidity_period:
send_humidity_data(client)
last_humidity_run = now
time.sleep(60)
Rename iothub_client_telemetry_sample_run to temperature_thread_func or something like it. Create a separate function that looks just like it for humidity. Spawn two threads from the main function of your program. Set them to daemon mode so they shutdown when the user exits
from threading import Thread
def temperature_thread_func():
client = iothub_client_init()
while True:
send_temperature_data(client)
time.sleep(60)
def humidity_thread_func():
client = iothub_client_init()
while True:
send_humidity_data(client)
time.sleep(600)
if __name__ == '__main__':
temp_thread = Thread(target=temperature_thread_func)
temp_thread.daemon = True
humidity_thread = Thread(target=humidity_thread_func)
humidity_thread.daemon = True
input('Polling for data. Press a key to exit')
Notes:
If you decide to use threads, consider using an
event
to terminate them cleanly.
time.sleep is not a precise way to keep
time. You might need a different timing mechanism if the samples need
to be taken at precise moments.
I had divided my software update process in various stages like (download, unzip, pre-install, install, post-install). I am setting reported properties
at every stage accordingly. But these properties are not updating during installation process (i.e. unable to see changes in reported property in device twin on azure portal) but at the end of installation I am getting callback responses for all the set "reported" properties.
I am working on software update with Azure IOT using python device sdk.
For this I have modified the sample given in SDK (i.e. iothub_client_sample_class.py file). I am using device twin for updating the software. I had created "desired" property for "software_version" in device twin. Once "desired" property for "software_version" is changed, software update process is started. Software update process perform various operation so I have divided this process in various stages. I am sending "reported" properties for ever stage to IotHub. But these "reported" are not updating in seqence in device twin reported property on azure portal.
import random
import time
import sys
import iothub_client
import json
from iothub_client import IoTHubClient, IoTHubClientError, IoTHubTransportProvider
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError, DeviceMethodReturnValue
from iothub_client_args import get_iothub_opt, OptionError
# HTTP options
# Because it can poll "after 9 seconds" polls will happen effectively
# at ~10 seconds.
# Note that for scalabilty, the default value of minimumPollingTime
# is 25 minutes. For more information, see:
# https://azure.microsoft.com/documentation/articles/iot-hub-devguide/#messaging
TIMEOUT = 241000
MINIMUM_POLLING_TIME = 9
# messageTimeout - the maximum time in milliseconds until a message times out.
# The timeout period starts at IoTHubClient.send_event_async.
# By default, messages do not expire.
MESSAGE_TIMEOUT = 1000
RECEIVE_CONTEXT = 0
AVG_WIND_SPEED = 10.0
MIN_TEMPERATURE = 20.0
MIN_HUMIDITY = 60.0
MESSAGE_COUNT = 5
RECEIVED_COUNT = 0
TWIN_CONTEXT = 0
METHOD_CONTEXT = 0
# global counters
RECEIVE_CALLBACKS = 0
SEND_CALLBACKS = 0
BLOB_CALLBACKS = 0
TWIN_CALLBACKS = 0
SEND_REPORTED_STATE_CALLBACKS = 0
METHOD_CALLBACKS = 0
firstTime = True
hub_manager = None
PROTOCOL = IoTHubTransportProvider.MQTT
CONNECTION_STRING = "XXXXXX"
base_version = '1.0.0.000'
SUCCESS = 0
firstTime = True
def downloadImage(url):
# Code for downloading the package from url
return 0
def unzipPackage():
#code for unzipping the package
return 0
def readPackageData():
# code reading package data
return 0
def pre_install():
#code for installing dependencies
return 0
def install():
#code for installing main package
return 0
def post_install():
#code for verifying installation
return 0
def start_software_update(url,message):
global hub_manager
print "Starting software update process!!!!"
reported_state = "{\"updateStatus\":\"softwareUpdateinprogress\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1003)
time.sleep(1)
status = downloadImage(url)
if status == SUCCESS:
reported_state = "{\"updateStatus\":\"downloadComplete\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1004)
print "Downlaod Phase Done!!!"
time.sleep(1)
else:
print "Download Phase failed!!!!"
return False
status = unzipPackage()
if status == SUCCESS:
reported_state = "{\"updateStatus\":\"UnzipComplete\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1005)
print "Unzip Package Done!!!"
time.sleep(1)
else:
print "Unzip package failed!!!"
return False
status = readPackageData()
if status == SUCCESS:
reported_state = "{\"updateStatus\":\"ReadPackageData\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1006)
print "Reading package json data"
time.sleep(1)
else:
print "Failed Reading package!!!"
return False
status = pre_install()
if status == SUCCESS:
reported_state = "{\"updateStatus\":\"PreInstallComplete\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1007)
time.sleep(1)
print "pre_install state successful!!!"
else:
print "pre_install failed!!!!"
return False
status = install()
if status == SUCCESS:
reported_state = "{\"updateStatus\":\"InstallComplete\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1008)
time.sleep(1)
print "install sucessful!!!"
else:
print "install failed!!!"
return False
status = post_install()
if status == SUCCESS:
reported_state = "{\"updateStatus\":\"SoftwareUpdateComplete\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1009)
time.sleep(1)
print "post install sucessful!!!"
else:
print "post install failed!!!"
return False
return True
def device_twin_callback(update_state, payload, user_context):
global TWIN_CALLBACKS
global firstTime
global base_version
print ( "\nTwin callback called with:\nupdateStatus = %s\npayload = %s\ncontext = %s" % (update_state, payload, user_context) )
TWIN_CALLBACKS += 1
print ( "Total calls confirmed: %d\n" % TWIN_CALLBACKS )
message = json.loads(payload)
if not firstTime:
if message["software_version"] != base_version:
url = message["url"]
status = start_software_update(url,message)
if status:
print "software Update Successful!!!"
else:
print "software Update Unsuccessful!!!"
else:
base_version = message["desired"]["software_version"]
print "Set firstTime to false", base_version
firstTime = False
def send_reported_state_callback(status_code, user_context):
global SEND_REPORTED_STATE_CALLBACKS
print ( "Confirmation for reported state received with:\nstatus_code = [%d]\ncontext = %s" % (status_code, user_context) )
SEND_REPORTED_STATE_CALLBACKS += 1
print ( " Total calls confirmed: %d" % SEND_REPORTED_STATE_CALLBACKS )
class HubManager(object):
def __init__(
self,
connection_string,
protocol=IoTHubTransportProvider.MQTT):
self.client_protocol = protocol
self.client = IoTHubClient(connection_string, protocol)
if protocol == IoTHubTransportProvider.HTTP:
self.client.set_option("timeout", TIMEOUT)
self.client.set_option("MinimumPollingTime", MINIMUM_POLLING_TIME)
# set the time until a message times out
self.client.set_option("messageTimeout", MESSAGE_TIMEOUT)
# some embedded platforms need certificate information
# self.set_certificates()
self.client.set_device_twin_callback(device_twin_callback, TWIN_CONTEXT)
def send_reported_state(self, reported_state, size, user_context):
self.client.send_reported_state(
reported_state, size,
send_reported_state_callback, user_context)
def main(connection_string, protocol):
global hub_manager
try:
print ( "\nPython %s\n" % sys.version )
print ( "IoT Hub Client for Python" )
hub_manager = HubManager(connection_string, protocol)
print ( "Starting the IoT Hub Python sample using protocol %s..." % hub_manager.client_protocol )
reported_state = "{\"updateStatus\":\"waitingforupdate\"}"
hub_manager.send_reported_state(reported_state, len(reported_state), 1002)
while True:
time.sleep(1)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubClient sample stopped" )
if __name__ == '__main__':
try:
(CONNECTION_STRING, PROTOCOL) = get_iothub_opt(sys.argv[1:], CONNECTION_STRING)
except OptionError as option_error:
print ( option_error )
usage()
sys.exit(1)
main(CONNECTION_STRING, PROTOCOL)
Expected Result: The "reported" property for every stage in software update should update properly in device twin in azure portal.
Actual Result: The "reported" property for each stage in software update process is not updating properly.
I was setting up an Websocket Server which is loggin into another server and pushing data trough the socket to the webpage (trough a subscribe function). As long as i keep calling the broadcast function from the file where the websocket runs, everything is fine. But calling the broadcast method from another python-file where my push-function is printing to command line, no client is recieving a message.
I assume, that calling the broadcast from another file creates another instance and with that the self.clients is empty.
So to sum up, clients connected get the broadcast from loginGESI() but not in my second file from scrptCallbackHandlerExample(subType).
Would be happy about any help!
here is my Websocket file:
class BroadcastServerProtocol(WebSocketServerProtocol):
def onOpen(self):
self.factory.register(self)
def connectionLost(self, reason):
WebSocketServerProtocol.connectionLost(self, reason)
self.factory.unregister(self)
class BroadcastServerFactory(WebSocketServerFactory):
clients = []
def __init__(self, url):
WebSocketServerFactory.__init__(self, url)
def register(self, client):
if client not in self.clients:
print("registered client {}".format(client.peer))
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print("unregistered client {}".format(client.peer))
self.clients.remove(client)
#classmethod
def broadcast(self, msg):
print("broadcasting message '{}' ..".format(msg))
print(self.clients)
for c in self.clients:
c.sendMessage(msg.encode('utf8'))
print("message sent to {}".format(c.peer))
def login():
codesys = Test_Client("FTS_test")
result = codesys.login()
# FTS = codesys.searchForPackage("F000012")
FTS = ["15900"];
scrptContextId = [None] * len(FTS)
itemContextIds_array = [None] * len(FTS)
for i in range(0,len(FTS)):
result, scrptContextId[i] = codesys.createSubscription(c_ScrptCallbackHandlerExample, 100, int(FTS[i]))
print("SubscriptionRoomId: "+str(scrptContextId[i]))
result, itemContextIds_array[i], diagInfo = codesys.attachToSubscription(1, [FTS[i]+'.speed'], [100])
print("Subscription done for: "+str(itemContextIds_array[i]))
print("Subscription for: Speed")
BroadcastServerFactory.broadcast(str(FTS[0]))
if __name__ == '__main__':
# Logger Websocket
log.startLogging(sys.stdout)
# factory initialisieren
ServerFactory = BroadcastServerFactory
factory = ServerFactory("ws://127.0.0.1:9000")
factory.protocol = BroadcastServerProtocol
listenWS(factory)
# reactor initialisieren
webdir = File(".")
web = Site(webdir)
reactor.listenTCP(8080, web)
reactor.callLater(5, login)
reactor.run()
and here my subscription file:
# Launch of the CallbackHandler named in the createSubscription function
# CallbackHandler describes what happens to a variable which changes its value
def scrptCallbackHandlerExample(subType):
BroadcastServerFactory.broadcast('test')
# Saves the value of the variables(s) in an array
dataValue = []
for i in range(0,subType.size):
dataValue.append(subType.dataItems[i].node.dataValue)
# Print variabel informations on the screen
print "*****Callback - Data Change in a Variable*****"
print( 'Subscription ID: %d' % subType.subscrId )
for idx in range(0,subType.size):
print( '** Item %d **' % idx )
print( 'Item Id: %d' % subType.dataItems[idx].dataItemId )
print( 'Item Node ID: %s' % subType.dataItems[idx].node.nodeId )
print( 'Item data value: %s' % subType.dataItems[idx].node.dataValue )
print( 'Item data type: %s' % subType.dataItems[idx].node.dataType )
print( '******************************' )
# Define the type of the function as an eSubscriptionType
CB_FUNC_TYPE = CFUNCTYPE( None, eSubscriptionType)
c_ScrptCallbackHandlerExample = CB_FUNC_TYPE( scrptCallbackHandlerExample )
Regards
I found, in my oppinion, a pretty neat workaround.
Whenever my subscribe function is called, I connect with a local client to my websocket server and send him a message with the new values, the websocket server then pushes this to the other clients. Since I am still working on it I can't post any code but the functionality is given. So if someone is interested in a solution let me know and I can post mine.
I'm using this wonderful resource to grab tweets from twitter.
Twitter streams the tweets, and this script grabs them and displays them in the terminal.
I apologize for the beginners question, but how would I limit the number of tweets or the total size of all tweets that are received using this script?
I understand that I can write these to a file and watch the size of the file, but is there a way to do count the bytes received or number of tweets received?
The script is below as well as in the link above.
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright (C) 2012 Gustav Arngården
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import time
import pycurl
import urllib
import json
import oauth2 as oauth
API_ENDPOINT_URL = 'https://stream.twitter.com/1.1/statuses/filter.json'
USER_AGENT = 'TwitterStream 1.0' # This can be anything really
# You need to replace these with your own values
OAUTH_KEYS = {'consumer_key': <Consumer key>,
'consumer_secret': <Consumer secret>,
'access_token_key': <Token key>,
'access_token_secret': <Token secret>}
# These values are posted when setting up the connection
POST_PARAMS = {'include_entities': 0,
'stall_warning': 'true',
'track': 'iphone,ipad,ipod'}
class TwitterStream:
def __init__(self, timeout=False):
self.oauth_token = oauth.Token(key=OAUTH_KEYS['access_token_key'], secret=OAUTH_KEYS['access_token_secret'])
self.oauth_consumer = oauth.Consumer(key=OAUTH_KEYS['consumer_key'], secret=OAUTH_KEYS['consumer_secret'])
self.conn = None
self.buffer = ''
self.timeout = timeout
self.setup_connection()
def setup_connection(self):
""" Create persistant HTTP connection to Streaming API endpoint using cURL.
"""
if self.conn:
self.conn.close()
self.buffer = ''
self.conn = pycurl.Curl()
# Restart connection if less than 1 byte/s is received during "timeout" seconds
if isinstance(self.timeout, int):
self.conn.setopt(pycurl.LOW_SPEED_LIMIT, 1)
self.conn.setopt(pycurl.LOW_SPEED_TIME, self.timeout)
self.conn.setopt(pycurl.URL, API_ENDPOINT_URL)
self.conn.setopt(pycurl.USERAGENT, USER_AGENT)
# Using gzip is optional but saves us bandwidth.
self.conn.setopt(pycurl.ENCODING, 'deflate, gzip')
self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.POSTFIELDS, urllib.urlencode(POST_PARAMS))
self.conn.setopt(pycurl.HTTPHEADER, ['Host: stream.twitter.com',
'Authorization: %s' % self.get_oauth_header()])
# self.handle_tweet is the method that are called when new tweets arrive
self.conn.setopt(pycurl.WRITEFUNCTION, self.handle_tweet)
def get_oauth_header(self):
""" Create and return OAuth header.
"""
params = {'oauth_version': '1.0',
'oauth_nonce': oauth.generate_nonce(),
'oauth_timestamp': int(time.time())}
req = oauth.Request(method='POST', parameters=params, url='%s?%s' % (API_ENDPOINT_URL,
urllib.urlencode(POST_PARAMS)))
req.sign_request(oauth.SignatureMethod_HMAC_SHA1(), self.oauth_consumer, self.oauth_token)
return req.to_header()['Authorization'].encode('utf-8')
def start(self):
""" Start listening to Streaming endpoint.
Handle exceptions according to Twitter's recommendations.
"""
backoff_network_error = 0.25
backoff_http_error = 5
backoff_rate_limit = 60
while True:
self.setup_connection()
try:
self.conn.perform()
except:
# Network error, use linear back off up to 16 seconds
print 'Network error: %s' % self.conn.errstr()
print 'Waiting %s seconds before trying again' % backoff_network_error
time.sleep(backoff_network_error)
backoff_network_error = min(backoff_network_error + 1, 16)
continue
# HTTP Error
sc = self.conn.getinfo(pycurl.HTTP_CODE)
if sc == 420:
# Rate limit, use exponential back off starting with 1 minute and double each attempt
print 'Rate limit, waiting %s seconds' % backoff_rate_limit
time.sleep(backoff_rate_limit)
backoff_rate_limit *= 2
else:
# HTTP error, use exponential back off up to 320 seconds
print 'HTTP error %s, %s' % (sc, self.conn.errstr())
print 'Waiting %s seconds' % backoff_http_error
time.sleep(backoff_http_error)
backoff_http_error = min(backoff_http_error * 2, 320)
def handle_tweet(self, data):
""" This method is called when data is received through Streaming endpoint.
"""
self.buffer += data
if data.endswith('\r\n') and self.buffer.strip():
# complete message received
message = json.loads(self.buffer)
self.buffer = ''
msg = ''
if message.get('limit'):
print 'Rate limiting caused us to miss %s tweets' % (message['limit'].get('track'))
elif message.get('disconnect'):
raise Exception('Got disconnect: %s' % message['disconnect'].get('reason'))
elif message.get('warning'):
print 'Got warning: %s' % message['warning'].get('message')
else:
print 'Got tweet with text: %s' % message.get('text')
if __name__ == '__main__':
ts = TwitterStream()
ts.setup_connection()
ts.start()
Why not a counter in your __init__:
def __init__(self, timeout=False):
...
self.tweetsSoFar = 0
def handleTweet(self, data):
...
else:
self.tweetsSoFar += 1 # we've seen another tweet!
print 'Got tweet with text: %s' % message.get('text')
def start(self):
...
while self.tweetsSoFar < 50: # stop at 50 tweets
...
Hope this helps
I'm not too familiar with the API, but assuming each complete message represents a tweet, then what you can do is to first add a counter in the class:
self.tweets = 0
Then increment it in handle_tweet after a full message is received:
if data.endswith('\r\n') and self.buffer.strip():
# complete message received
message = json.loads(self.buffer)
...
...
else:
print 'Got tweet with text: %s' % message.get('text')
self.tweets += 1
if self.tweets == SOME_LIMIT:
self.conn.close() # or exit()
I'm trying to transmit TCP/IP over a radio that is connected to my computer (specifically, the USRP). Right now, it's done very simply using Tun/Tap to set up a new network interface. Here's the code:
from gnuradio import gr, gru, modulation_utils
from gnuradio import usrp
from gnuradio import eng_notation
from gnuradio.eng_option import eng_option
from optparse import OptionParser
import random
import time
import struct
import sys
import os
# from current dir
from transmit_path import transmit_path
from receive_path import receive_path
import fusb_options
#print os.getpid()
#raw_input('Attach and press enter')
# Linux specific...
# TUNSETIFF ifr flags from <linux/tun_if.h>
IFF_TUN = 0x0001 # tunnel IP packets
IFF_TAP = 0x0002 # tunnel ethernet frames
IFF_NO_PI = 0x1000 # don't pass extra packet info
IFF_ONE_QUEUE = 0x2000 # beats me ;)
def open_tun_interface(tun_device_filename):
from fcntl import ioctl
mode = IFF_TAP | IFF_NO_PI
TUNSETIFF = 0x400454ca
tun = os.open(tun_device_filename, os.O_RDWR)
ifs = ioctl(tun, TUNSETIFF, struct.pack("16sH", "gr%d", mode))
ifname = ifs[:16].strip("\x00")
return (tun, ifname)
# /////////////////////////////////////////////////////////////////////////////
# the flow graph
# /////////////////////////////////////////////////////////////////////////////
class my_top_block(gr.top_block):
def __init__(self, mod_class, demod_class,
rx_callback, options):
gr.top_block.__init__(self)
self.txpath = transmit_path(mod_class, options)
self.rxpath = receive_path(demod_class, rx_callback, options)
self.connect(self.txpath);
self.connect(self.rxpath);
def send_pkt(self, payload='', eof=False):
return self.txpath.send_pkt(payload, eof)
def carrier_sensed(self):
"""
Return True if the receive path thinks there's carrier
"""
return self.rxpath.carrier_sensed()
# /////////////////////////////////////////////////////////////////////////////
# Carrier Sense MAC
# /////////////////////////////////////////////////////////////////////////////
class cs_mac(object):
"""
Prototype carrier sense MAC
Reads packets from the TUN/TAP interface, and sends them to the PHY.
Receives packets from the PHY via phy_rx_callback, and sends them
into the TUN/TAP interface.
Of course, we're not restricted to getting packets via TUN/TAP, this
is just an example.
"""
def __init__(self, tun_fd, verbose=False):
self.tun_fd = tun_fd # file descriptor for TUN/TAP interface
self.verbose = verbose
self.tb = None # top block (access to PHY)
def set_top_block(self, tb):
self.tb = tb
def phy_rx_callback(self, ok, payload):
"""
Invoked by thread associated with PHY to pass received packet up.
#param ok: bool indicating whether payload CRC was OK
#param payload: contents of the packet (string)
"""
if self.verbose:
print "Rx: ok = %r len(payload) = %4d" % (ok, len(payload))
if ok:
os.write(self.tun_fd, payload)
def main_loop(self):
"""
Main loop for MAC.
Only returns if we get an error reading from TUN.
FIXME: may want to check for EINTR and EAGAIN and reissue read
"""
min_delay = 0.001 # seconds
while 1:
payload = os.read(self.tun_fd, 10*1024)
if not payload:
self.tb.send_pkt(eof=True)
break
if self.verbose:
print "Tx: len(payload) = %4d" % (len(payload),)
delay = min_delay
while self.tb.carrier_sensed():
sys.stderr.write('B')
time.sleep(delay)
if delay < 0.050:
delay = delay * 2 # exponential back-off
self.tb.send_pkt(payload)
# /////////////////////////////////////////////////////////////////////////////
# main
# /////////////////////////////////////////////////////////////////////////////
def main():
mods = modulation_utils.type_1_mods()
demods = modulation_utils.type_1_demods()
parser = OptionParser (option_class=eng_option, conflict_handler="resolve")
expert_grp = parser.add_option_group("Expert")
parser.add_option("-m", "--modulation", type="choice", choices=mods.keys(),
default='gmsk',
help="Select modulation from: %s [default=%%default]"
% (', '.join(mods.keys()),))
parser.add_option("-v","--verbose", action="store_true", default=False)
expert_grp.add_option("-c", "--carrier-threshold", type="eng_float", default=30,
help="set carrier detect threshold (dB) [default=%default]")
expert_grp.add_option("","--tun-device-filename", default="/dev/net/tun",
help="path to tun device file [default=%default]")
transmit_path.add_options(parser, expert_grp)
receive_path.add_options(parser, expert_grp)
for mod in mods.values():
mod.add_options(expert_grp)
for demod in demods.values():
demod.add_options(expert_grp)
fusb_options.add_options(expert_grp)
(options, args) = parser.parse_args ()
if len(args) != 0:
parser.print_help(sys.stderr)
sys.exit(1)
if options.rx_freq is None or options.tx_freq is None:
sys.stderr.write("You must specify -f FREQ or --freq FREQ\n")
parser.print_help(sys.stderr)
sys.exit(1)
# open the TUN/TAP interface
(tun_fd, tun_ifname) = open_tun_interface(options.tun_device_filename)
# Attempt to enable realtime scheduling
r = gr.enable_realtime_scheduling()
if r == gr.RT_OK:
realtime = True
else:
realtime = False
print "Note: failed to enable realtime scheduling"
# If the user hasn't set the fusb_* parameters on the command line,
# pick some values that will reduce latency.
if options.fusb_block_size == 0 and options.fusb_nblocks == 0:
if realtime: # be more aggressive
options.fusb_block_size = gr.prefs().get_long('fusb', 'rt_block_size', 1024)
options.fusb_nblocks = gr.prefs().get_long('fusb', 'rt_nblocks', 16)
else:
options.fusb_block_size = gr.prefs().get_long('fusb', 'block_size', 4096)
options.fusb_nblocks = gr.prefs().get_long('fusb', 'nblocks', 16)
#print "fusb_block_size =", options.fusb_block_size
#print "fusb_nblocks =", options.fusb_nblocks
# instantiate the MAC
mac = cs_mac(tun_fd, verbose=True)
# build the graph (PHY)
tb = my_top_block(mods[options.modulation],
demods[options.modulation],
mac.phy_rx_callback,
options)
mac.set_top_block(tb) # give the MAC a handle for the PHY
if tb.txpath.bitrate() != tb.rxpath.bitrate():
print "WARNING: Transmit bitrate = %sb/sec, Receive bitrate = %sb/sec" % (
eng_notation.num_to_str(tb.txpath.bitrate()),
eng_notation.num_to_str(tb.rxpath.bitrate()))
print "modulation: %s" % (options.modulation,)
print "freq: %s" % (eng_notation.num_to_str(options.tx_freq))
print "bitrate: %sb/sec" % (eng_notation.num_to_str(tb.txpath.bitrate()),)
print "samples/symbol: %3d" % (tb.txpath.samples_per_symbol(),)
#print "interp: %3d" % (tb.txpath.interp(),)
#print "decim: %3d" % (tb.rxpath.decim(),)
tb.rxpath.set_carrier_threshold(options.carrier_threshold)
print "Carrier sense threshold:", options.carrier_threshold, "dB"
print
print "Allocated virtual ethernet interface: %s" % (tun_ifname,)
print "You must now use ifconfig to set its IP address. E.g.,"
print
print " $ sudo ifconfig %s 192.168.200.1" % (tun_ifname,)
print
print "Be sure to use a different address in the same subnet for each machine."
print
tb.start() # Start executing the flow graph (runs in separate threads)
mac.main_loop() # don't expect this to return...
tb.stop() # but if it does, tell flow graph to stop.
tb.wait() # wait for it to finish
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
pass
(Anyone familiar with GNU Radio will recognize this as tunnel.py)
My question is, is there a better way to move packets to and from the kernel than tun/tap? I've been looking at ipip or maybe using sockets, but I'm pretty sure those won't be very fast. Speed is what I'm most concerned with.
Remember that tunnel.py is a really, really rough example, and hasn't been updated in a while. It's not really meant to be a basis for other code, so be careful of how much you rely on the code.
Also, remember that TCP over unreliable radio links has significant issues:
http://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_over_wireless_networks