We Keep Coding

dynamically add and remove sockets

How would I add / remove sockets dynamically on the with / as statement. Or would i need a completely other aproach like asyncio.create_task?
import asyncio
from binance import AsyncClient, BinanceSocketManager
api_key = config.binance_key
api_secret = config.binance_secret
async def main():
client = await AsyncClient.create()
bm = BinanceSocketManager(client)
# start any sockets here, i.e a trade socket
ts1 = bm.symbol_ticker_socket('BNBBTC')
ts2 = bm.symbol_ticker_socket('ETHBUSD')
# then start receiving messages
async with ts1 as tscm1, ts2 as tscm2:
while True:
res1 = await tscm1.recv()
res2 = await tscm2.recv()
print(res1)
print(res2)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())

Use AsyncExitStack to combine and handle multiple asynchronous context managers.
An auxiliary coroutine read_ticker is used to read data from a ticker socket connection.
import asyncio
from binance import AsyncClient, BinanceSocketManager
from contextlib import AsyncExitStack
api_key = config.binance_key
api_secret = config.binance_secret
async def read_ticker(ts_cm):
"""Read ticker data from ticker socket connection"""
while True:
res = await ts_cm.recv()
print(res)
async def main():
client = await AsyncClient.create()
bm = BinanceSocketManager(client)
# start any sockets here, i.e a trade socket
ts1 = bm.symbol_ticker_socket('BNBBTC')
ts2 = bm.symbol_ticker_socket('ETHBUSD')
async with AsyncExitStack() as stack:
await asyncio.gather(*[read_ticker(await stack.enter_async_context(ts))
for ts in (ts1, ts2)])
if __name__ == "__main__":
asyncio.run(main())

asyncio.gather() is starting first two tasks but not the third one

I have a python function that is supposed to send packet asynchronously in a loop.
import asyncio
import logging
import pickle
from time import sleep
from decoder import decode, encode
from socket import *
UDP_PORT_NUMBER = 44006
HOST_ADDRESS = "192.168.204.139"
server_address = (HOST_ADDRESS, UDP_PORT_NUMBER)
def get_data_packet():
with open('../packets/packet_15', mode='rb') as fpt:
packet = pickle.load(fpt, encoding='bytes')
return packet
class GcuUdpClient:
#staticmethod
async def send_packets():
data = get_data_packet()
client_socket = socket(AF_INET, SOCK_DGRAM)
client_socket.sendto(data, server_address)
print('sending data packets')
await asyncio.sleep(5)
I have another python program that calls this class, along with two other classes.
import asyncio
from http_server import PostServer
from grpc_server import GrpcServer
from gcu_udp_client import GcuUdpClient
async def main():
await asyncio.gather(GrpcServer.run(), PostServer.run_server(), GcuUdpClient.send_packets())
if __name__ == '__main__':
asyncio.run(main())
This code starts up the first two meaning
GrpcServer.run()
and
PostServer.run_server()
However I can't get the last one started, to send out the packets. What am I missing.
Adding the code for http_server.py
import threading
import zlib
from http import HTTPStatus
from http.server import HTTPServer, BaseHTTPRequestHandler
from io import BytesIO
from socket import *
from time import sleep
#from GCUSimulator.generated.GDataPayload_pb2 import GDataPayload
from generated.GDataPayload_pb2 import GDataPayload
HTTP_PORT_NUMBER = 8085
HTTP_SERVER_IP="192.168.204.1"
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
def __init__(self, *args, **kwargs):
self.subscribed = False
super(SimpleHTTPRequestHandler, self).__init__(*args, *kwargs)
def do_GET(self):
self.send_response(200)
self.end_headers()
def do_POST(self):
content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
post_data = self.rfile.read(content_length) # <--- Gets the data itself
print("POST Path: {0} Headers: {1} Body: {2}".
format(str(self.path), str(self.headers), post_data.decode('utf-8')))
if str(self.path) == '/setSubscriber.php':
message = "Subscription Result:Subscribed,APIVersion:1.0.0"
self.send_response(HTTPStatus.OK)
self.end_headers()
self.wfile.write(message.encode('utf-8'))
self.subscribed = True
elif str(self.path) == '/removeSubscriber.php':
self.send_response(HTTPStatus.OK)
self.end_headers()
self.subscribed = False
def get_subscription_status(self):
return self.subscribed
class PostServer:
#staticmethod
async def run_server(server_class=HTTPServer, handler_class=SimpleHTTPRequestHandler, addr=HTTP_SERVER_IP, port=HTTP_PORT_NUMBER):
server_address = (addr, port)
print('Http Post server : {0}:{1}'.format(addr, port))
httpd = server_class(server_address, handler_class)
await httpd.serve_forever()
Adding the code for grpc_server.py
from concurrent import futures
import grpc
from generated import guidance_grpc_service_definitions_pb2_grpc
from gcu.ops_station_services import OPSStationServices
GRPC_PORT_NUMBER = 12074
class GrpcServer:
#staticmethod
async def run():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
guidance_grpc_service_definitions_pb2_grpc.add_OpRPCServicesServicer_to_server(OServices(), server)
server.add_insecure_port('[::]:{0}'.format(GRPC_PORT_NUMBER))
print('Grpc Server : {0}:{1}'.format('localhost', GRPC_PORT_NUMBER))
server.start()
# server.wait_for_termination()

ConnectionResetError exception when receiving EOF in AsyncIO framework

I have a very simple chatroom (chat_server.py and chat_client.py) written asynchronously using AsyncIO framework. The server simply receives messages and tell the client that I received your message, that's it. The code is from Łukasz Langa's tutorial.
chat_server.py
from typing import *
import asyncio
async def handle_connection(
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
) -> None:
addr = writer.get_extra_info("peername")
while message := await reader.read(100):
text = message.decode()
print(f"Received {text!r} from {addr!r}")
print(f"Sending {text!r}")
writer.write(message)
await writer.drain()
if text == "quit\n":
break
print("Closing the connection")
writer.close()
async def main() -> None:
server = await asyncio.start_server(handle_connection, "127.0.0.1", 8888)
addr = server.sockets[0].getsockname() if server.sockets else "unknown"
print(f"Serving on {addr}")
async with server:
await server.serve_forever()
if __name__ == "__main__":
asyncio.run(main())
chat_client1.py
from typing import *
import asyncio
import sys
async def send_fine(file: IO[str]) -> None:
reader, writer = await asyncio.open_connection("127.0.0.1", 8888)
for message in file:
writer.write(message.encode()) ###############
await writer.drain() ###############
data = await reader.read(100)
text = data.decode()
print(f"Received {text!r}")
if text == "quit\n":
break
print("Closing the connection")
writer.close()
if __name__ == "__main__":
asyncio.run(send_fine(sys.stdin))
Having the above code, I can cause EOF by ctrl+D and the chat_server.py will handle that correctly by closing the connection without any exception.
However, instead of sending line by line, if I send byte by byte, it raises a ConnectionResetError when I press ctrl+D.
I've put ####... to show which lines where changed:
chat_cleint2.py
from typing import *
import asyncio
import sys
async def send_fine(file: IO[str]) -> None:
reader, writer = await asyncio.open_connection("127.0.0.1", 8888)
for message in file:
for ch in message.encode(): ###############
await asyncio.sleep(0.1) ###############
writer.write(bytes([ch])) ###############
await writer.drain()
data = await reader.read(100)
text = data.decode()
print(f"Received {text!r}")
if text == "quit\n":
break
print("Closing the connection")
writer.close()
if __name__ == "__main__":
asyncio.run(send_fine(sys.stdin))
I'm mostly interested in why there is no ConnectionResetError in chat_client1.py but there is in chat_client2.py.

Pandas Concat 500 CSV Files Within S3

I do have around 500 CSV file within S3 Bucket
Note: Each file size is around 1 GB, Around ~6m of lines.
What am trying to do is to concatenate all those CSV files into a single file.
I'm trying to speed up the Process but am not sure what else i can do in such case!
Below is my code:
import trio
import boto3
import pandas as pd
from functools import partial
AWS_ID = 'Hidden'
AWS_SECRET = 'Hidden'
Bucket_Name = 'Hidden'
limiter = trio.CapacityLimiter(10)
async def read_object(bucket, object_csv, sender):
async with limiter, sender:
print(f'Reading {object_csv}')
test = bucket.Object(object_csv)
test = test.get()['Body']
data = await trio.to_thread.run_sync(partial(pd.read_csv, test, header=None))
await sender.send(data)
async def main():
async with trio.open_nursery() as nurse:
s3 = boto3.resource(
service_name='s3',
aws_access_key_id=AWS_ID,
aws_secret_access_key=AWS_SECRET,
)
bucket = s3.Bucket(Bucket_Name)
allfiles = [i.key for i in bucket.objects.all()]
sender, receiver = trio.open_memory_channel(0)
nurse.start_soon(rec, receiver)
async with sender:
for f in allfiles:
nurse.start_soon(read_object, bucket, f, sender.clone())
async def rec(receiver):
alldf = []
async with receiver:
async for df in receiver:
alldf.append(df)
final = pd.concat(alldf, ignore_index=True)
print(final)
if __name__ == "__main__":
try:
trio.run(main)
except KeyboardInterrupt:
exit('Job Cancelled!')
The part which is taking time:
data = await trio.to_thread.run_sync(partial(pd.read_csv, test, header=None))
For a single file it's taking about 2 minutes. Also I'm running the operation under threads! so it's taking a lot!
Update: New Code is Below:
limiter = trio.CapacityLimiter(10)
async def read_object(bucket, object_csv, sender):
async with limiter, sender:
print(f'Reading {object_csv}')
test = bucket.Object(object_csv)
test = test.get()['Body']
data = await trio.to_thread.run_sync(test.read)
await sender.send(data)
print(f'Done Reading {object_csv}')
async def main():
async with trio.open_nursery() as nurse:
s3 = boto3.resource(
service_name='s3',
aws_access_key_id=AWS_ID,
aws_secret_access_key=AWS_SECRET,
)
bucket = s3.Bucket(Bucket_Name)
sender, receiver = trio.open_memory_channel(0)
nurse.start_soon(rec, receiver)
async with sender:
for csv in bucket.objects.all():
nurse.start_soon(read_object, bucket, csv.key, sender.clone())
async def rec(receiver):
async with receiver, await trio.open_file('output.csv', 'wb') as f:
count = 0
async for df in receiver:
count += 1
await f.write(df)
await f.write(b"\n")
print(f'Collected {count}', flush=True, end='\r')
if __name__ == "__main__":
try:
trio.run(main)
except KeyboardInterrupt:
exit('Job Cancelled!')
Based on that code, is it possible to speed up the process more than that ?

Python asyncio / aiohttp error

I am writing a simple producer/consumer app to call multiple URL's asynchronously.
In the following code if I set the conn_count=1, and add 2 items to the Queue it works fine as only one consumer is created. But if I make conn_count=2 and add 4 items to the Queue only 3 request are being made. The other request fails with ClientConnectorError.
Can you please help be debug the reason for failure with multiple consumers? Thank You.
I am using a echo server I created.
Server:
import os
import logging.config
import yaml
from aiohttp import web
import json
def start():
setup_logging()
app = web.Application()
app.router.add_get('/', do_get)
app.router.add_post('/', do_post)
web.run_app(app)
async def do_get(request):
return web.Response(text='hello')
async def do_post(request):
data = await request.json()
return web.Response(text=json.dumps(data))
def setup_logging(
default_path='logging.yaml',
default_level=logging.INFO,
env_key='LOG_CFG'
):
path = default_path
value = os.getenv(env_key, None)
if value:
path = value
if os.path.exists(path):
with open(path, 'rt') as f:
config = yaml.safe_load(f.read())
logging.config.dictConfig(config)
else:
logging.basicConfig(level=default_level)
if __name__ == '__main__':
start()
Client:
import asyncio
import collections
import json
import sys
import async_timeout
from aiohttp import ClientSession, TCPConnector
MAX_CONNECTIONS = 100
URL = 'http://localhost:8080'
InventoryAccount = collections.namedtuple("InventoryAccount", "op_co customer_id")
async def produce(queue, num_consumers):
for i in range(num_consumers * 2):
await queue.put(InventoryAccount(op_co=i, customer_id=i * 100))
for j in range(num_consumers):
await queue.put(None)
async def consumer(n, queue, session, responses):
print('consumer {}: starting'.format(n))
while True:
try:
account = await queue.get()
if account is None:
queue.task_done()
break
else:
print(f"Consumer {n}, Updating cloud prices for account: opCo = {account.op_co!s}, customerId = {account.customer_id!s}")
params = {'opCo': account.op_co, 'customerId': account.customer_id}
headers = {'content-type': 'application/json'}
with async_timeout.timeout(10):
print(f"Consumer {n}, session state " + str(session.closed))
async with session.post(URL,
headers=headers,
data=json.dumps(params)) as response:
assert response.status == 200
responses.append(await response.text())
queue.task_done()
except:
e = sys.exc_info()[0]
print(f"Consumer {n}, Error updating cloud prices for account: opCo = {account.op_co!s}, customerId = {account.customer_id!s}. {e}")
queue.task_done()
print('consumer {}: ending'.format(n))
async def start(loop, session, num_consumers):
queue = asyncio.Queue(maxsize=num_consumers)
responses = []
consumers = [asyncio.ensure_future(loop=loop, coro_or_future=consumer(i, queue, session, responses)) for i in range(num_consumers)]
await produce(queue, num_consumers)
await queue.join()
for consumer_future in consumers:
consumer_future.cancel()
return responses
async def run(loop, conn_count):
async with ClientSession(loop=loop, connector=TCPConnector(verify_ssl=False, limit=conn_count)) as session:
result = await start(loop, session, conn_count)
print("Result: " + str(result))
if __name__ == '__main__':
conn_count = 2
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(run(loop, conn_count))
finally:
loop.close()
Reference:
https://pymotw.com/3/asyncio/synchronization.html
https://pawelmhm.github.io/asyncio/python/aiohttp/2016/04/22/asyncio-aiohttp.html
https://hackernoon.com/asyncio-for-the-working-python-developer-5c468e6e2e8e