We Keep Coding

kafka consumer to ruby on rails

I am currently working with a rails server which is supposed to run python script, which are kafka consumer/producer.
The server must to run the script then receive the processed data from consumer, and render them to the site.
I am able to run a script but can not fund a solution for the consumer to be connected. As the consumer is either running non stop accepting the messages, or running in while loop. I tried to run the consumer first from ruby, which starts the consumer, but never gets the consumer, as it is listening, but the other script could not be run.
So the flow of the message ideally should be something like this -> email from logged user to kafka producer -> MQ -> kafka consumer generates data writes to db -> producer query data from database -> MQ -> consumer accepts the data and renders them to the site.
The ideal scenario would be a one script lets call it manager that does all the work only accepts data and returns it. It also was not able to do that because, the one script also runs consumer and listens for producer, but it is never ran.
so here is my code:
from kafka import KafkaProducer
from faker import Faker
import json
import time
class producer1():
'''
fr_instance= Faker()
def get_new_user():
return {"email_address":fr_instance.email(),"first_name": fr_instance.first_name(),
"lastname": fr_instance.last_name(), "occupation":fr_instance.job()}
'''
def __init__(self):
self
def json_serializer(self, data):
return json.dumps(data).encode("utf-8")
def send(self,email):
print(email)
producer = KafkaProducer(bootstrap_servers='localhost:9092',
value_serializer = self.json_serializer)
registred_user = {"email": email}
future = producer.send("NewUserTopic", registred_user)
print (registred_user)
result = future.get(timeout=10)
p = producer1()
if __name__ == '__main__':
email = "testmail#aaaaaaaa.com"
p.send(email)
then 1st consumer:
from kafka import KafkaConsumer
import json
import random
from sqlalchemy.orm import sessionmaker
import dbservice
import time
class consumer1():
def __init__(self) -> None:
self
def email(self):
consumer = KafkaConsumer('NewUserTopic',
bootstrap_servers='localhost:9092',
auto_offset_reset = 'latest', enable_auto_commit= False)
for msg in consumer:
msg_out = json.loads(msg.value)
for value in msg_out.values():
#return print(msg_out)
return (value)
#generate dummy address , eth
def gen_ETHw (self):
numbers = str(random.randint(11111,99999))
wallet_num = str("Ox"+numbers)
return (wallet_num)
#generate dummy address , btc
def gen_BTCw (self):
numbers = str(random.randint(11111,99999))
wallet_num = str("Ox"+numbers)
return (wallet_num)
def commit_db (self, email, ETHw, BTCw):
Session = sessionmaker(bind=dbservice.engine)
s = Session()
input = dbservice.walletdb( email,ETHw, BTCw)
time.sleep(2)
s.add(input)
s.commit()
if __name__ =='__main__':
while True:
c = consumer1()
c.commit_db(c.email(),c.gen_ETHw(),c.gen_BTCw())
query producer:
import dbservice
import dbservice
from sqlalchemy.orm import sessionmaker
from kafka import KafkaProducer
import json
class query_prod ():
def __init__(self, email) -> None:
self = self
self.email = email
def json_serializer(data):
return json.dumps(data).encode("utf-8")
producer = KafkaProducer(bootstrap_servers='localhost:9092',
value_serializer = json_serializer)
Session = sessionmaker(bind=dbservice.engine)
s = Session()
def query_address(self,email):
Session = sessionmaker(bind=dbservice.engine)
s = Session()
for s in s.query(dbservice.walletdb).filter_by(email=email):
return {"email":s.email,"ETH_w":s.ETH_w,"BTC_w":s.BTC_w}
def send(self, email):
data_to_send = self.query_address(email)
future = self.producer.send("QueryAdressToServer", data_to_send)
print (data_to_send)
result = future.get(timeout=10)
if __name__ == '__main__':
email = "testmail#aaaaaaaa.com"
query_prod=query_prod(email)
query_prod.send(email)
and consume data which should be returned to the site:
from kafka import KafkaConsumer
import json
import time
class consume_for_web():
string=""
def __init__(self) -> None:
self = self
string = self.string
def consumer(self):
consumer = KafkaConsumer('QueryAdressToServer',
bootstrap_servers='localhost:9092',
auto_offset_reset = 'latest', enable_auto_commit= False)
print('starting consumer')
for msg in consumer:
data = (('{}'.format(json.loads(msg.value))))
self.string = self.string + data
return print(data)
def read_str(self):
return print(self.string)
if __name__ =='__main__':
while True:
c = consume_for_web()
c.consumer()
##print("reading")
#c.read_str()
and finally my rails pages controller:
class PagesController < ApplicationController
def home
end
def about
end
before_action :require_login
def genw
our_input = current_user.email
puts our_input
#consumer_result = `python3 /Users/samuelrybar/python_projects/Kafka_demo1/kafka-prod-coms/consumer2.py`
end
def mywa
end
def save
end
end
Thanks for your time and help, I really appreciate it. :))

Not sure why you are trying to run python scripts from a running Rails server. It sounds like a very bad idea to me. You can run Kafka consumers/producers from Ruby and Ruby on Rails directly. I suggest you investigate Karafka. We've been using it successfully at work.

Why does my Twitter stream give me the HTTP Error: 406?

That's my code. Every thing works well, but when the Twitter Stream should start there comes the following error: Stream encountered HTTP Error: 406
With the normal stream it works well, but with the asynchronous stream it doesn't. Where is my mistake?
import discord
import tweepy
import tweepy.asynchronous
class Stream(tweepy.asynchronous.AsyncStream):
async def on_connect(self):
print("connected")
async def on_status(self, status):
print(status.text)
async def on_exception(self, exception):
print(exception)
print("!exception")
"""async def on_request_error(self, status_code):
print(status_code)"""
#If An error occurs then the programm should be restartet
"""await restart(discord_client=client, twitter_stream = self)"""
async def on_closed(self, resp):
print(resp)
class Client(discord.Client):
async def on_ready(self):
#creating stream
stream = Stream(keys + tokens)
#creating api
auth = tweepy.OAuthHandler(keys)
auth.set_access_token(tokens)
api = tweepy.API(auth)
#getting specific users which the stream should follow
follow_list = [screen_name]
follower_ids = []
for _ in follow_list:
follow = int(api.get_user(screen_name = _).id)
print(follow)
follower_ids.append(follow)
print(str(follower_ids))
#starting Stream
stream.filter(follow = follow_list)
print("start")
client = Client()
client.run(token)

This is likely because you're passing invalid IDs for AsyncStream.filter's follow parameter.
follow_list seems to be a list with a single screen name or a list of screen names, not a list of user IDs. It seems like you might have intended to use follower_ids instead.
A 406 Not Acceptable HTTP Error means:
At least one request parameter is invalid. For example, the filter endpoint returns this status if:
The track keyword is too long or too short.
An invalid bounding box is specified.
Neither the track nor follow parameter are specified.
The follow user ID is not valid.
https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/guides/connecting

Send data to a Django channels consumer

I have the following basic Django Channels consumer:
class EchoConsumer(AsyncJsonWebsocketConsumer):
async def connect(self):
await self.accept()
await self.send_json('Connected!')
And, in parallel, i have a normal Python script which connects to a websocket and receives some data in real time:
from binance.client import Client
import json
from binance.websockets import BinanceSocketManager
client = Client('', '')
# get all symbol prices
prices = client.get_all_tickers()
trades = client.get_recent_trades(symbol='BNBBTC')
# start aggregated trade websocket for BNBBTC
def process_message(message):
JSON1 = json.dumps(message)
JSON2 = json.loads(JSON1)
#define variables
Rate = JSON2['p']
Quantity = JSON2['q']
Symbol = JSON2['s']
Order = JSON2['m']
print(Rate, Quantity, Order)
bm = BinanceSocketManager(client)
bm.start_trade_socket('BNBBTC', process_message)
bm.start()
I would like to do the following: instead of only printing the data received, the second script should send somehow that data to the Django Channels consumer. Whenever a user opens the page, that page should receive that data. If a second user opens the page at the same time, that second user should receive the data too. Is it possible to do this? Am i supposed to use another service?

so if you want to send this data to all currently open websocket connections you can do the following.
class EchoConsumer(AsyncJsonWebsocketConsumer):
groups = ["echo_group"]
async def on_message(self, message):
await self.send_json(... something here based on the message ... )
Then in your script you need to import channels (and have django configured so its best for this to be a django command see: https://docs.djangoproject.com/en/3.0/howto/custom-management-commands/
from channels.layers import get_channel_layer
channel_layer = get_channel_layer()
... your other stuff to connect to
def process_message(message):
JSON1 = json.dumps(message)
JSON2 = json.loads(JSON1)
#define variables
Rate = JSON2['p']
Quantity = JSON2['q']
Symbol = JSON2['s']
Order = JSON2['m']
async_to_sync(channel_layer.group_send)(
"echo_group",
{"type": "on.message", "rate":Rate, "quantity": Quantity, "symbol": Symbol, "order": Order},
)

Django channels: pass form data to consumer

I'm learning Django and I'm working in a webpage where I need to offer the user the possibility to log in to an external service. I can't simply use the traditional Django views system because otherwise, I would lose the connection with a simple refresh. For that reason, I thought about using Django Channels.
My problem now is how do I send the data to the consumer class? Using the consumers.py given in the tutorial, I would like to send data from a form submission to the connect function and then make the connection if the login to the external service is ok. Then, in that scenario, I could use the clientinstance and the methods from these external services.
So, in short, is it possible to send form data the consumer? And would this be ok, with respect to security in case of sensitive data?
from channels.generic.websocket import AsyncWebsocketConsumer
import json
class ChatConsumer(AsyncWebsocketConsumer):
async def connect(self):
######
## login to external service
######
#get login data from form submited when the websockted is initiated
username = ...
pass = ...
self.client = Client(username, password)
if client:
await self.accept()
# Receive message from room group
async def chat_message(self, event):
message = event['message']
self.client.send(event['message'])
Update:
To clear the explanation: I can't save the user username and pass of the external service, and that I want to offer the user the possibility to use this [sms service](https://clxcommunications.github.io/sdk-xms-python/tutorial.html) with a text field and phone number.
So the problem is that even if I create a form and the username and password to log in (in the view) with
client = clx.xms.Client('myserviceplan', 'mytoken')
then in next request, I would lose the client instance. That's why I thought about Django Channels. But I'm not really sure if it is the best solution...

This will help you.
consumers.py
from asgiref.sync import async_to_sync
from channels.generic.websocket import WebsocketConsumer
import json
class EventConsumer(WebsocketConsumer):
def connect(self):
# self.room_name = self.scope['url_route']['kwargs']['room_name']
# self.room_group_name = 'chat_%s' % self.room_name
self.room_name = 'event'
self.room_group_name = self.room_name+"_sharif"
async_to_sync(self.channel_layer.group_add)(
self.room_group_name,
self.channel_name
)
print(self.room_group_name)
self.accept()
print("#######CONNECTED############")
def disconnect(self, code):
async_to_sync(self.channel_layer.group_discard)(
self.room_group_name,
self.channel_name
)
print("DISCONNECED CODE: ",code)
def receive(self, text_data=None, bytes_data=None):
print(" MESSAGE RECEIVED")
data = json.loads(text_data)
message = data['message']
async_to_sync(self.channel_layer.group_send)(
self.room_group_name,{
"type": 'send_message_to_frontend',
"message": message
}
)
def send_message_to_frontend(self,event):
print("EVENT TRIGERED")
# Receive message from room group
message = event['message']
# Send message to WebSocket
self.send(text_data=json.dumps({
'message': message
}))
then call the function outside/anywhere from your app like
def event_triger():
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
'event_sharif',
{
'type': 'send_message_to_frontend',
'message': "event_trigered_from_views"
}
)
# here 'event_sharif' is your room_group_name as i defined before in consumer
# 'type' is like a command, for which method you wants to trigger in your consumer
for more you may take a look Send message using Django Channels from outside Consumer class

Generally, you can call a method in the consumer from an external code in the following way:
from channels.layers import get_channel_layer
channel_layer = get_channel_layer()
await self.channel_layer.send(
'<channel_name>',
{
'type': '<method_name>',
}
)
But as you can see, this requires that you specify the channel name which you can only get after the client has connected. In other words, you should not try to call connect but some other method in the consumer. Also, your client should first connect to the websocket before you eventually access it. I don't understand your use case completey but I hope this gives you an idea

Tornado: How to get and return large data with less memory usage?

I have web-crawler and http interface for it.
Crawler gets grouped urls as dictionary. I need to return a result in the same format in JSON. But I was faced with a large memory usage, which is not returned to the operating system. How can I implement this solution without large memory usage?
Code:
#!/usr/bin/env python
# coding=utf-8
import collections
import tornado.web
import tornado.ioloop
import tornado.queues
import tornado.httpclient
class ResponseError(Exception):
pass
class Crawler(object):
client = tornado.httpclient.AsyncHTTPClient()
def __init__(self, groups, concurrency=10, retries=3, validators=None):
self.groups = groups
self.concurrency = concurrency
self.retries = retries
self.validators = validators or []
self.requests = tornado.queues.Queue()
self.responses = collections.defaultdict(list)
async def worker(self):
while True:
await self.consume()
async def validate(self, response):
for validator in self.validators:
validator(response)
async def save(self, response):
self.responses[response.request.group].append(response.body.decode('utf-8'))
async def consume(self):
async for request in self.requests:
try:
response = await self.client.fetch(request, raise_error=False)
await self.validate(response)
await self.save(response)
except ResponseError:
if request.retries < self.retries:
request.retries += 1
await self.requests.put(request)
finally:
self.requests.task_done()
async def produce(self):
for group, urls in self.groups.items():
for url in urls:
request = tornado.httpclient.HTTPRequest(url)
request.group = group
request.retries = 0
await self.requests.put(request)
async def fetch(self):
await self.produce()
for __ in range(self.concurrency):
tornado.ioloop.IOLoop.current().spawn_callback(self.worker)
await self.requests.join()
class MainHandler(tornado.web.RequestHandler):
async def get(self):
urls = []
with open('urls') as f: # mock
for line in f:
urls.append(line.strip())
crawler = Crawler({'default': urls})
await crawler.fetch()
self.write(crawler.responses)
if __name__ == '__main__':
app = tornado.web.Application(
(tornado.web.url(r'/', MainHandler),), debug=True
)
app.listen(8000)
tornado.ioloop.IOLoop.current().start()

It looks to me like most of the memory usage is devoted to self.responses. Since you seem to be ordering responses by "group" before writing them to a file, I can understand why you do it this way. One idea is to store them in a database (MySQL or MongoDB or whatever) with the "group" as column or field value in the database record.
The database might be the final destination of your data, or else it might be a temporary place to store the data until crawler.fetch completes. Then, query all the data from the database, ordered by "group", and write it to the file.
This doesn't solve the problem, it just means that the database process is responsible for most of your memory usage, instead of the Python process. This may be preferable for you, however.