We Keep Coding

Joining twitch irc through helix api, (5000+ channels) connection reset error

To start off im using an anonymous connection joining the channels which means there are no JOIN limits, I have tried different variations of sleeping, I started off just joining from a text however that had a lot of problems because it was connecting all the sockets before joining so I couldnt see what caused it. However this is the best version I have created so far, its pretty scuffed but I am just trying to understand what the issue is. If anyone has any insight on doing a big task like this I would appreciate it a lot!
(oauth and helix headers are from a random alt account I made for testing and its trying to join 10k channels in the example but stops around 2k-3k max)
import requests
import socket
import time
import threading
import random
connections_made = 0
sockets = []
def connect():
global sockets
global connections_made
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print("CONNECTING TO IRC")
sock.connect(('irc.chat.twitch.tv', 6667))
sock.send(bytes('PASS oauth:'+ '\r\n', 'utf-8'))
sock.send(bytes('NICK justinfan' + str(random.randint(10000,99999)) + '\r\n', 'utf-8'))
sockets.append(sock)
connections_made += 1
print(f"socket: {len(sockets)}")
for i in range(2):
connect() # initial for .recv reading
helix_headers = {'client-id': 'q6batx0epp608isickayubi39itsckt', 'authorization': 'Bearer rk0ixn6169ar7y5xey9msvk1h8zrs8'}
def request(channels_to_join,cursor):
request_amount = int(channels_to_join / 100) # 100 requests = 10000 channels
user_list = []
sock_numb = 0
total_chans_joined = 0
count_every_request = 0
for i in range(request_amount):
time.sleep(1)
# 3k channels with time.sleep(1) 1.5k channels with time.sleep(2) 30 seconds then connection reset error (when bulk joining 100 channels and waiting for the next request)
# waiting 30 seconds doesnt fix this either stop at about 500 channels so lasted 2.5minutes?
# waiting 60 seconds at 500 channels breaks
if count_every_request == 1: # for every 100 channels
connect()
count_every_request = 0
r = requests.get("https://api.twitch.tv/helix/streams?first=100&after=" + cursor,headers=helix_headers)
cursor = r.json()['pagination']['cursor']
count_every_request += 1
for everything in r.json()['data']:
user_list.append(everything['user_login'])
channel = everything['user_login']
# join channel
if sock_numb == connections_made: # makes it so when joining sockets it joins up to the amount of sockets that there are and then loops back
sock_numb = 0
print(f"JOINING #{channel} with socket: {sock_numb} total joined: {total_chans_joined}")
sockets[sock_numb].send(bytes('JOIN #' + channel + '\r\n', 'utf-8'))
total_chans_joined += 1
sock_numb += 1
def loop():
print("Looping")
try:
while True:
time.sleep(0.1)
for i in range(connections_made):
data = sockets[i].recv(4096).decode("utf-8",errors='replace').strip()
if data == "":
continue
print(data)
if "PING :tmi.twitch.tv" in data:
print("PONG")
sockets[i].send(bytes('PONG :tmi.twitch.tv' + '\r\n', 'utf-8'))
except Exception as e:
print(str(e) + " error in loop ")
pass
thread_loop = threading.Thread(target=loop)
thread_loop.start()
request(channels_to_join=10000,cursor = "eyJiIjp7IkN1cnNvciI6ImV5SnpJam80T0RrMU1TNDVNRFkwTWpnd09URTVNU3dpWkNJNlptRnNjMlVzSW5RaU9uUnlkV1Y5In0sImEiOnsiQ3Vyc29yIjoiZXlKeklqbzFNakF6TGpJM056UTFPVEUzT1RReE1Td2laQ0k2Wm1Gc2MyVXNJblFpT25SeWRXVjkifX0")

The likely problem is that your bot can't keep up with the message send buffer.
So you connect to many channels, but are not processing the incoming chat messages in a timely fashion. So the "queue" of messages to send from Twitch to You exceeds Twitch's buffer. And it DC's you
Or as per the IRC Rate limit guide you are sneding too many commands and getting Disconnected from the server.
Large chat bots will often split groups of channels over multiple connections to solve this issue.

IBM Stats subscription topic always returns Reason 2033: FAILED: MQRC_NO_MSG_AVAILABLE

I am attempting to port some old java code to python.
I am using pymqi to connect to a queue manager and query for all messageflow statistics topics using the topic string: $SYS/Broker/+/StatisticsAccounting/Archive/#
When using the existing java program messages are read from the topic without issue.
When using the new python code it is able to connect and query the topic without issue but always gives the message
Reason 2033: FAILED: MQRC_NO_MSG_AVAILABLE
Stats messages are published by the broker for each messageflow every 10 minutes, and I have left the new code running for over 30minutes, never having received a message.
I've also tried setting
get_opts['WaitInterval'] = pymqi.CMQC.MQWI_UNLIMITED
and sitting around for 20minutes rather than using a loop, but no luck.
Is there any IIB server config that might be impacting the messages that I am able to see, or are there other options I should be using within the client?
import pymqi
queue_manager = 'MYQM'
channel = 'MYAPP.SVRCONN'
host = 'MYHOST'
port = 'MYPORT'
topic_string = '$SYS/Broker/+/StatisticsAccounting/Archive/#'
conn_info = '%s(%s)' % (host, port)
user = ""
password = ""
qmgr = pymqi.QueueManager(None)
qmgr.connect_tcp_client(queue_manager, pymqi.CD(), channel, conn_info, user, password)
sub_desc = pymqi.SD()
sub_desc['Options'] = pymqi.CMQC.MQSO_CREATE + pymqi.CMQC.MQSO_RESUME + pymqi.CMQC.MQSO_MANAGED
sub_desc.set_vs('SubName', 'apptest')
sub_desc.set_vs('ObjectString', topic_string)
sub = pymqi.Subscription(qmgr)
sub.sub(sub_desc=sub_desc)
get_opts = pymqi.GMO(Options=pymqi.CMQC.MQGMO_WAIT)
get_opts['WaitInterval'] = 10000
md = pymqi.md()
keep_running = True
while keep_running:
try:
# Reset the MsgId, CorrelId & GroupId so that we can reuse
# the same 'md' object again.
md.MsgId = pymqi.CMQC.MQMI_NONE
md.CorrelId = pymqi.CMQC.MQCI_NONE
md.GroupId = pymqi.CMQC.MQGI_NONE
message = sub.get(None, md, get_opts)
print('Have message from Queue')
print(message)
except pymqi.MQMIError as e:
if e.comp == pymqi.CMQC.MQCC_FAILED and e.reason == pymqi.CMQC.MQRC_NO_MSG_AVAILABLE:
print("no message?")
print(e)
pass
else:
# Some other error condition.
raise
except (UnicodeDecodeError, ValueError) as e:
print('Message is not valid json')
print(e)
print(message)
continue
except KeyboardInterrupt:
print('Have received a keyboard interrupt')
keep_running = False
sub.close(sub_close_options=0,close_sub_queue=True)
qmgr.disconnect()

How to tell a beanstalkc receiver to wait X seconds before reserving a task?

I have 2 beanstalkc receivers watching the same tube "tubename".
I would like one beanstalkc receiver to have priority over the other. In order to achieve this, I would like to tell the lowest-priority beanstalkc receiver to wait for task being X seconds old before reserving them.
I found "reserve-with-timeout", but I neither really understand it nor do I managed to make it work successfully for my use case.
class MyBeanstalkReceiver():
def __init__(self, host=beanstalkc.DEFAULT_HOST, port=beanstalkc.DEFAULT_PORT,
tube="default", timeout=1):
self.tube = tube
self.host = host
self.port = port
self.timeout = timeout
def run(self):
while True:
self.run_once()
def run_once(self):
job = self._get_task()
try:
body = job.body
data = json.loads(body)
self.job(data)
except Exception as e:
job.delete()
def job(self, data):
print(data)
def beanstalk(self):
beanstalk = beanstalkc.Connection(host=self.host, port=self.port)
beanstalk.use(self.tube)
beanstalk.watch(self.tube)
return beanstalk
def _get_task(self):
return self.beanstalk().reserve(self.timeout)
And my 2 beanstalkc receivers:
# receiver 1
w = MyBeanstalkReceiver(hosts=["localhost:14711"], tube="tubename", timeout=1)
w.run()
# receiver 2
w = MyBeanstalkReceiver(hosts=["localhost:14711"], tube="tubename", timeout=10000)
w.run()
Between the 2 receivers, with a timeout of 1 and 10000, nothing changes when I send tasks over the tube: both end up managing the same quantity of tasks put inside the tube "tubename".
Any idea on how to proceed to make "receiver 1" prioritary over "receiver 2"?

The timeout in reserve is for how long the client will wait before returning without a job.
You may be looking for put (with a delay), where the job is not released until it has been in the queue for at least n seconds.
There is also a priority per job. If the receiver could have seen them both at the same time, it will return any jobs with a higher priority (ie: closer to 0) rather than with a lower priority (with a larger number).

Beanstalkd does not differentiate between priorities of the clients or receivers.

Python/Matlab comm over TCP

Edit
Worked around this problem by switching to UDP. See that updated code snippets after the break.
I'm having trouble with my threaded tcp server in python. The goal is to establish a communication link between Matlab and a Python server to allow for querying a database. This example only works properly once, then I have to restart Matlab to get the message echoed back again.
The Python side:
import SocketServer
from threading import Thread
class service(SocketServer.BaseRequestHandler):
def handle(self):
self.data = 'dummy'
print "Client connected with ", self.client_address
while len(self.data):
self.data = self.request.recv(1024).strip()
print self.data
self.request.sendall(self.data.upper())
print "Client exited"
self.request.close()
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
pass
t = ThreadedTCPServer(('',1522), service)
t.serve_forever()
And the Matlab Code:
t = ThreadedTCPServer(('',1522), service)
t.serve_forever()
t = tcpip('localhost', 1522);
fopen(t);
% write a message
fwrite(t, 'This is a test message.');
% read the echo
bytes = fread(t, [1, t.BytesAvailable]);
char(bytes)
% close the connection
fclose(t);
So, while i'm expecting the data to get echoed back, this only occurs on the first run, i.e. right after I've started Matlab. If I run it again, I get the message on the python side, but it is not echoed again. Ideas?
The Updated Python Code:
x = np.array([[55, 1000, 45], [20, 3, 10]])
class UDP_Interrupt(SocketServer.BaseRequestHandler):
def setup(self):
pass
def handle(self):
data = self.request[0].strip()
print data
socket = self.request[1]
print "{}{} wrote:".format(self.client_address[0], self.client_address)
print data
print x
socket.sendto(x.tostring('C'), self.client_address)
#scipy.io.savemat('/Users/empire/Documents/MATLAB/hybridQuadSim/quaternionController/models/mapData.mat', mdict={'mapData': x})
def finish(self):
pass
class ThreadedUDPServer(SocketServer.ThreadingMixIn, SocketServer.UDPServer):
pass
if __name__ == "__main__":
map_server = ThreadedUDPServer((HOST, PORT), UDP_Interrupt)
# terminate with Ctrl-C
try:
server_thread = Thread(target=map_server.serve_forever)
server_thread.daemon = False
server_thread.start()
print "Server loop running in thread:", server_thread.name
except KeyboardInterrupt:
sys.exit(0)
And for Matlab:
% connect to the server
t = udp('localhost', 2002);
fopen(t);
% write a message
fwrite(t, 'This is a test message.');
% read the echo
bytes = fread(t, [t.BytesAvailable, 1], 'char');
temp = reshape(bytes, [8 6]);
z = [0,0,0,0,0,0]
for col = 1:6
bytepack=uint64(0);
for row = 1:8
temp(9-row, col)
bytepack = bitshift(temp(9 - row, col),8*(8-row));
z(col) = bitor(bytepack,z(col));
temp(row, col);
end;
end;
% close the connection
fclose(t);
This setup seems to work well enough, but would appreciate any feedback or corrections offered. I would like a more robust solution to the problem of reconstructing the numpy array into a matrix in Matlab.

python chat using poll

I need to built a python chat and I'm stacked in the very final step. I've built the server and the client and I have the following problem while running the code:
server.py 127.0.0.1
-in a separate window client.py 127.0.0.1
-another client
-type the nicknames to chat for both clients and get the correct answer 'yuppie' meaning you are connected
a client try to speak
message is not read by the other client until it doesn't print something, after printing it get the message printed on its screen correctly.
I'd like to get the message without being obliged to print something, it's pretty unrealistic!!! Code of client and server are below in 2 different classes. Thank you!
#! /usr/bin/env python
import socket,sys,select,re
PORT=1060
class Server():
def __init__(self,host):
#building listen_sock
self.listen_sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.listen_sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
self.listen_sock.bind((host,PORT))
self.listen_sock.listen(20)
#building dict for socket and socket state
self.sockets={self.listen_sock.fileno(): self.listen_sock}
self.socket_state={self.listen_sock.fileno():''}
#building poll object
self.poll=select.poll()
self.poll.register(self.listen_sock,select.POLLIN)
#users' list
self.users_list={}
#DON'T LOOK HERE
#initialize the sender
#self.sender=0
# self.users=re.compile("\s*\$(get users connected)$\s*",re.IGNORECASE)
# self.nick=re.compile("\s*\$\$(\w*\d*)\$\$\s*",re.IGNORECASE)
# self.quit=re.compile("\s*\$(quit)\$\s*",re.IGNORECASE)
#self.commands=[self.users,self.nick,self.quit]
#funcion to receive message from client (work well)
def recv_until(self,fd,suffix):
self.message=''
#checking the end of the message
while not self.message.endswith(suffix):
data=self.sockets[fd].recv(16)
if not data:
raise EOFError('socket closed before we saw %r' % suffix)
self.message+=data
self.message=self.message[:-1]
#delete client (work well)
def del_client(self,fd):
del self.users_list[fd]
del self.socket_state[fd]
self.poll.unregister(fd)
#print the remaining active connections
if not len(self.users_list):
print 'Anyone is connected, waiting for new connection'
else:
print self.users_list
#add new client and change the of the file descriptor for that client (work well)
def new_client(self,fd):
newsock, sockname = self.listen_sock.accept()
print 'new connection from ', newsock.getpeername()
newsock.setblocking(False)
#recording the new connection
fd=newsock.fileno()
self.sockets[fd]=newsock
self.poll.register(fd,select.POLLOUT)
self.socket_state[fd]='ask nick'
#DON'T LOOK HERE
# def handle_query(self,fd):
# for n,command in enumerate(self.commands):
# match=command.search(self.message)
# if n==1 and match:
# self.users_list[self.sockets[fd].getpeername()]=match.group(1)
# print self.users_list
# for value in self.users_list.values():
# self.sockets[fd].sendall(value+'\n')
#starting the main function of the class
def chat(self):
while True:
#here il where the code hangs up waitng and waiting (WORKS BAD)
#return a tuple, identify where (fd) the event (event) is happening
for fd,event in self.poll.poll():
#print the state of each socket and the poll object
print self.socket_state
print self.poll.poll()
#starting the state machine
#remove closed sockets
if event & (select.POLLHUP | select.POLLERR |
select.POLLNVAL):
#deleting the socket closed at fd
self.del_client(fd)
#if the socket referred to is our listen_sock and we have a new connection request
elif self.sockets[fd] is self.listen_sock:
#recording the new entry!
self.new_client(fd)
#managing all the situation where it is necessary to answer to a client
#and changing the state of the socket and that of the sockets[fd]
elif event & select.POLLOUT:
if self.socket_state[fd]=='ask nick':
self.sockets[fd].sendall('identify\n')
self.poll.modify(self.sockets[fd],select.POLLIN)
self.socket_state[fd]='get user'
if self.socket_state[fd]=='invalid nick':
self.sockets[fd].sendall('invalid nick\n')
for value in self.users_list.values():
self.sockets[fd].sendall('\n'+value+'\n')
self.socket_state[fd]='ask nick'
if self.socket_state[fd]=='connected':
print '3'
self.sockets[fd].sendall('yuppie\n')
self.poll.modify(self.sockets[fd],select.POLLIN)
self.socket_state[fd]='ready to communicate'
if self.socket_state[fd]=='ready to receive':
self.sockets[fd].sendall(self.message)
print '4'
self.poll.modify(self.sockets[fd],select.POLLIN)
self.socket_state[fd]='ready to communicate'
#managing all the situation where it is necessary to get values from clients
elif event & select.POLLIN:
if self.socket_state[fd]=='get user':
self.recv_until(fd,'\n')
if self.message not in self.users_list.values():
self.users_list[fd]=self.message
self.poll.modify(self.sockets[fd],select.POLLOUT)
self.socket_state[fd]='connected'
else:
self.poll.modify(self.sockets[fd],select.POLLOUT)
self.socket_state[fd]='invalid nick'
if self.socket_state[fd]=='ready to communicate':
self.recv_until(fd,'\n')
print '5'
for i in self.users_list.keys():
if i!=fd:
self.poll.modify(self.sockets[i],select.POLLOUT)
self.socket_state[i]='ready to receive'
if __name__ == '__main__':
se=Server(sys.argv[1])
se.chat()
#! /usr/bin/env python
import sys,socket,select,threading,time
s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
HOST=sys.argv.pop()
PORT=1060
class Client():
def setup(self):
server_address=(HOST,PORT)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect(server_address)
def chat(self):
while True:
time.sleep(1)
text=raw_input('>>> ')
self.sock.sendall(text+'\n')
def rec(self):
while True:
mess=self.sock.recv(16)
if mess:
print '$$$ ', mess,
def start(self):
l=threading.Thread(target=self.rec)
t=threading.Thread(target=self.chat)
t.start()
l.start()
if __name__=='__main__':
cl=Client()
cl.setup()
cl.start()

Next time take a look at http://www.zeromq.org/, it has a nice python binding http://zeromq.github.com/pyzmq/. It's perfect for this kind of stuff.