We Keep Coding

kafka consumer to ruby on rails

I am currently working with a rails server which is supposed to run python script, which are kafka consumer/producer.
The server must to run the script then receive the processed data from consumer, and render them to the site.
I am able to run a script but can not fund a solution for the consumer to be connected. As the consumer is either running non stop accepting the messages, or running in while loop. I tried to run the consumer first from ruby, which starts the consumer, but never gets the consumer, as it is listening, but the other script could not be run.
So the flow of the message ideally should be something like this -> email from logged user to kafka producer -> MQ -> kafka consumer generates data writes to db -> producer query data from database -> MQ -> consumer accepts the data and renders them to the site.
The ideal scenario would be a one script lets call it manager that does all the work only accepts data and returns it. It also was not able to do that because, the one script also runs consumer and listens for producer, but it is never ran.
so here is my code:
from kafka import KafkaProducer
from faker import Faker
import json
import time
class producer1():
'''
fr_instance= Faker()
def get_new_user():
return {"email_address":fr_instance.email(),"first_name": fr_instance.first_name(),
"lastname": fr_instance.last_name(), "occupation":fr_instance.job()}
'''
def __init__(self):
self
def json_serializer(self, data):
return json.dumps(data).encode("utf-8")
def send(self,email):
print(email)
producer = KafkaProducer(bootstrap_servers='localhost:9092',
value_serializer = self.json_serializer)
registred_user = {"email": email}
future = producer.send("NewUserTopic", registred_user)
print (registred_user)
result = future.get(timeout=10)
p = producer1()
if __name__ == '__main__':
email = "testmail#aaaaaaaa.com"
p.send(email)
then 1st consumer:
from kafka import KafkaConsumer
import json
import random
from sqlalchemy.orm import sessionmaker
import dbservice
import time
class consumer1():
def __init__(self) -> None:
self
def email(self):
consumer = KafkaConsumer('NewUserTopic',
bootstrap_servers='localhost:9092',
auto_offset_reset = 'latest', enable_auto_commit= False)
for msg in consumer:
msg_out = json.loads(msg.value)
for value in msg_out.values():
#return print(msg_out)
return (value)
#generate dummy address , eth
def gen_ETHw (self):
numbers = str(random.randint(11111,99999))
wallet_num = str("Ox"+numbers)
return (wallet_num)
#generate dummy address , btc
def gen_BTCw (self):
numbers = str(random.randint(11111,99999))
wallet_num = str("Ox"+numbers)
return (wallet_num)
def commit_db (self, email, ETHw, BTCw):
Session = sessionmaker(bind=dbservice.engine)
s = Session()
input = dbservice.walletdb( email,ETHw, BTCw)
time.sleep(2)
s.add(input)
s.commit()
if __name__ =='__main__':
while True:
c = consumer1()
c.commit_db(c.email(),c.gen_ETHw(),c.gen_BTCw())
query producer:
import dbservice
import dbservice
from sqlalchemy.orm import sessionmaker
from kafka import KafkaProducer
import json
class query_prod ():
def __init__(self, email) -> None:
self = self
self.email = email
def json_serializer(data):
return json.dumps(data).encode("utf-8")
producer = KafkaProducer(bootstrap_servers='localhost:9092',
value_serializer = json_serializer)
Session = sessionmaker(bind=dbservice.engine)
s = Session()
def query_address(self,email):
Session = sessionmaker(bind=dbservice.engine)
s = Session()
for s in s.query(dbservice.walletdb).filter_by(email=email):
return {"email":s.email,"ETH_w":s.ETH_w,"BTC_w":s.BTC_w}
def send(self, email):
data_to_send = self.query_address(email)
future = self.producer.send("QueryAdressToServer", data_to_send)
print (data_to_send)
result = future.get(timeout=10)
if __name__ == '__main__':
email = "testmail#aaaaaaaa.com"
query_prod=query_prod(email)
query_prod.send(email)
and consume data which should be returned to the site:
from kafka import KafkaConsumer
import json
import time
class consume_for_web():
string=""
def __init__(self) -> None:
self = self
string = self.string
def consumer(self):
consumer = KafkaConsumer('QueryAdressToServer',
bootstrap_servers='localhost:9092',
auto_offset_reset = 'latest', enable_auto_commit= False)
print('starting consumer')
for msg in consumer:
data = (('{}'.format(json.loads(msg.value))))
self.string = self.string + data
return print(data)
def read_str(self):
return print(self.string)
if __name__ =='__main__':
while True:
c = consume_for_web()
c.consumer()
##print("reading")
#c.read_str()
and finally my rails pages controller:
class PagesController < ApplicationController
def home
end
def about
end
before_action :require_login
def genw
our_input = current_user.email
puts our_input
#consumer_result = `python3 /Users/samuelrybar/python_projects/Kafka_demo1/kafka-prod-coms/consumer2.py`
end
def mywa
end
def save
end
end
Thanks for your time and help, I really appreciate it. :))

Not sure why you are trying to run python scripts from a running Rails server. It sounds like a very bad idea to me. You can run Kafka consumers/producers from Ruby and Ruby on Rails directly. I suggest you investigate Karafka. We've been using it successfully at work.

Export data with OPCUa subscription in IoT Edge

I am trying to receive data as an OPCUA client, therefore I created a subscription for some data nodes. Receiving data is no problem, but exporting it as an IoT Client is the problem. I think I have to run 2 threads paralel. The following code is used:
class Publisher():
def __init__(self):
self.module_client = IoTHubModuleClient.create_from_edge_environment()
async def export_data(self, message, output):
"""Serializes databuffer to json string"""
print('Export')
self.module_client.send_message_to_output(message, output)
print('Exported completed')
class Handler(object):
"""
Client to subscription. It will receive events from server
"""
def __init__(self):
""""""
self.publisher = Publisher()
def datachange_notification(self, node, val, data):
self.publisher.export_data('test', 'output1')
url = "opc.tcp://192.168.3.5:4840"
opcua_client = Client(url)
opcua_client.connect()
opcua_broker = Handler()
sub = opcua_client.create_subscription(20, opcua_broker)
node_list = [opcua_client.get_node('ns=3;s="EM126_Log"."DataLogL"[1]')]
handler = sub.subscribe_data_change(node_list)
time.sleep(100)
The error is as follows:
"RuntimeWarning: coroutine 'Publisher.export_data' was never awaited"
Does anyone know how to export the incoming data in de Handler class (datachange_notfication)

How to mock a dependency's response when unit-testing in Python

I am trying to write unit test for some of Python classes I have created. I have created a class for wrapping s3 functionalities, and in that class I'm initializing boto3 s3 client.
class S3_Client:
def __init__(self, bucket_name):
self.s3 = boto3.client("s3", aws_access_key_id=e_config["aws_access_key"], aws_secret_access_key=e_config["aws_secret_key"])
self.bucket_name = bucket_name
def fetch(self, key):
response = self.s3.get_object(Bucket=self.bucket_name, Key=key)
return self.__prepare_file_info(response, key) # return formatted response
I would like to test method fetch with mocked response from self.s3.get_object. This is my test class:
import unittest
from .aws_s3_service import S3_Client # class I want to test
import boto3
from botocore.stub import Stubber
class TestS3_Client(unittest.TestCase):
def setUp(self):
self.client = boto3.client('s3')
self.stubber = Stubber(self.client)
def test_fetch(self):
get_object_response = {...} # hardcoded response
self.stubber.add_response('get_object', get_object_response, {"Bucket": "test_bucket", "Key": "path/to/file/test_file.txt"})
with self.stubber:
client = S3_Client("test_bucket")
result = client.fetch("path/to/file/test_file.txt")
The stubber is not actually injected into S3_Client, a real call to S3 is made. How do I inject the stubber? Any help is appreciated, thanks.

You need to make S3_Client accept a client object in a constructor argument.
In this way in your tests you can create a client, stub it, then inject it to S3_Client as a parameter.
If you don't like having to always create that client outside of the class, you can make it an optional argument, and create an instance in __init__ if none was passed:
class S3_Client:
def __init__(self, bucket_name, s3=None):
if s3 is None:
self.s3 = boto3.client("s3", aws_access_key_id=e_config["aws_access_key"], aws_secret_access_key=e_config["aws_secret_key"])
else:
self.s3 = s3
self.bucket_name = bucket_name
...
In the code of your test you would then say: client = S3_Client("test_bucket", self.client).

Why can't I send image to Google Cloud IoT Core with MQTT using Python (with this code)?

Below is my attempt to send an image to Google Cloud IoT Core with MQTT. I have read the following posts which have helped me somewhat, but my code still isn't working: How Can I Publish File to AWS- IoT using Mosquitto in Python and How can I publish a file using Mosquitto in python?.
I would guess my error has to do either with the qos in the client.publish or how I have used the looping, but I'm afraid my experimenting with these factors hasn't helped me so far (trying qos = 0/1/2 and e.g. client.loop_forever()). My image is 1.2 Mb, so this shouldn't be a problem as far as I understand.
#!/usr/bin/python
from picamera import PiCamera
import datetime
import time
import jwt
import paho.mqtt.client as mqtt
from time import sleep
# Define some project-based variables to be used below. This should be the only
# block of variables that you need to edit in order to run this script
ssl_private_key_filepath = 'FILE1.pem'
ssl_algorithm = 'RS256' # Either RS256 or ES256
root_cert_filepath = 'FILE2.PEM'
project_id = 'PROJECT_ID'
gcp_location = 'LOCATION'
registry_id = 'REG_ID'
device_id = 'DEVICE_ID'
# end of user-variables
run = True
cur_time = datetime.datetime.utcnow()
def create_jwt():
token = {
'iat': cur_time,
'exp': cur_time + datetime.timedelta(minutes=60),
'aud': project_id
}
with open(ssl_private_key_filepath, 'r') as f:
private_key = f.read()
return jwt.encode(token, private_key, ssl_algorithm)
_CLIENT_ID = 'projects/{}/locations/{}/registries/{}/devices/{}'.format(project_id, gcp_location, registry_id, device_id)
_MQTT_TOPIC = '/devices/{}/events'.format(device_id)
client = mqtt.Client(client_id=_CLIENT_ID)
# authorization is handled purely with JWT, no user/pass, so username can be whatever
client.username_pw_set(
username='unused',
password=create_jwt())
def error_str(rc):
return '{}: {}'.format(rc, mqtt.error_string(rc))
def on_connect(unusued_client, unused_userdata, unused_flags, rc):
print('on_connect', error_str(rc))
def on_publish(unused_client, unused_userdata, unused_mid):
print('on_publish')
run = False
client.on_connect = on_connect
client.on_publish = on_publish
client.tls_set(ca_certs=root_cert_filepath) # Replace this with 3rd party cert if that was used when creating registry
client.connect('mqtt.googleapis.com', 8883)
camera = PiCamera()
camera.start_preview()
sleep(5)
camera.capture('/tmp/picture.jpg')
camera.stop_preview()
with open("/tmp/picture.jpg", 'rb') as f:
imagestring = f.read()
byteArray = bytes(imagestring)
try:
client.publish(_MQTT_TOPIC, byteArray, qos=2)
except:
print('Error')
while run:
client.loop()
client.disconnect()

The answer was provided by #Aaron.
"1.2Mb is way too big..."
Telemetry event payload limit is 256 KB and cannot be increased

google app engine, python , unicode, email, mail api

I'm using an open source web service python application to send email through GAE but if the name or email body contains Arabic or Hebrew characters the application throws some errors (e.g "The indicated parameters are not valid"). Therefore I need to know how to fix this issue. I have to note that I'm a Python beginner (one week since I started playing with Python).
#
import cgi
import os
import logging
import contextlib
from xml.dom import minidom
from xml.dom.minidom import Document
import exceptions
import warnings
import imghdr
from google.appengine.api import images
from google.appengine.api import users
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext.webapp import template
from google.appengine.api import mail
import wsgiref.handlers
# START Constants
CONTENT_TYPE_HEADER = "Content-Type"
CONTENT_TYPE_TEXT = "text/plain"
XML_CONTENT_TYPE = "application/xml"
XML_ENCODING = "utf-8"
"""
Allows you to specify IP addresses and associated "api_key"s to prevent others from using your app.
Storage and Manipulation methods will check for this "api_key" in the POST/GET params.
Retrieval methods don't use it (however you could enable them to use it, but maybe rewrite so you have a "read" key and a "write" key to prevent others from manipulating your data).
Set "AUTH = False" to disable (allowing anyone use your app and CRUD your data).
To generate a hash/api_key visit https://www.grc.com/passwords.htm
To find your ip visit http://www.whatsmyip.org/
"""
AUTH = False
# END Constants
# START Exception Handling
class Error(StandardError):
pass
class Forbidden(Error):
pass
logging.getLogger().setLevel(logging.DEBUG)
#contextlib.contextmanager
def mailExcpHandler(ctx):
try:
yield {}
except (ValueError), exc:
xml_error_response(ctx, 400 ,'app.invalid_parameters', 'The indicated parameters are not valid: ' + exc.message)
except (Forbidden), exc:
xml_error_response(ctx, 403 ,'app.forbidden', 'You don\'t have permission to perform this action: ' + exc.message)
except (Exception), exc:
xml_error_response(ctx, 500 ,'system.other', 'An unexpected error in the web service has happened: ' + exc.message)
def xml_error_response(ctx, status, error_id, error_msg):
ctx.error(status)
doc = Document()
errorcard = doc.createElement("error")
errorcard.setAttribute("id", error_id)
doc.appendChild(errorcard)
ptext = doc.createTextNode(error_msg)
errorcard.appendChild(ptext)
ctx.response.headers[CONTENT_TYPE_HEADER] = XML_CONTENT_TYPE
ctx.response.out.write(doc.toxml(XML_ENCODING))
# END Exception Handling
# START Helper Methods
def isAuth(ip = None, key = None):
if AUTH == False:
return True
elif AUTH.has_key(ip) and key == AUTH[ip]:
return True
else:
return False
# END Helper Methods
# START Request Handlers
class Send(webapp.RequestHandler):
def post(self):
"""
Sends an email based on POST params. It will queue if resources are unavailable at the time.
Returns "Success"
POST Args:
to: the receipent address
from: the sender address (must be a registered GAE email)
subject: email subject
body: email body content
"""
with mailExcpHandler(self):
# check authorised
if isAuth(self.request.remote_addr,self.request.POST.get('api_key')) == False:
raise Forbidden("Invalid Credentials")
# read data from request
mail_to = str(self.request.POST.get('to'))
mail_from = str(self.request.POST.get('from'))
mail_subject = str(self.request.POST.get('subject'))
mail_plain = str(self.request.POST.get('plain'))
mail_html = str(self.request.POST.get('html'))
message = mail.EmailMessage()
message.sender = mail_from
message.to = mail_to
message.subject = mail_subject
message.body = mail_plain
if mail_html != None and mail_html != "":
message.html = mail_html
message.send()
self.response.headers[CONTENT_TYPE_HEADER] = CONTENT_TYPE_TEXT
self.response.out.write("Success")
# END Request Handlers
# START Application
application = webapp.WSGIApplication([
('/send', Send)
],debug=True)
def main():
run_wsgi_app(application)
if __name__ == '__main__':
main()
# END Application

mail_to = str(self.request.POST.get('to'))
mail_from = str(self.request.POST.get('from'))
mail_subject = str(self.request.POST.get('subject'))
mail_plain = str(self.request.POST.get('plain'))
mail_html = str(self.request.POST.get('html'))
I doubt you need to convert them to strings. Try without str(), it could work.