Python threads didn't want to close - python

I have a problem with my program.
I managed to locate the source of problem. Problem is in the method "checkPort". Without it the standard textbook methods for closing/terminating threads works very well.
Am I missing something? Is in the checkPort method something that prevents to successfully join the threads? Its always stuck on thread.join().
Part of the main program:
try:
queue = Queue.Queue()
for i in range(MAX_THREAD_COUNT):
t = checkPort_IPv6_thread(queue)
t.daemon = True
threads.append(t)
t.start()
cur.execute("SELECT * FROM ipv6")
results = cur.fetchall()
for row in results:
queue.put((row[0], row[2]))
queue.join()
for thread in threads:
thread.stop()
thread.join()
except Exception as e:
sys.stderr.write("Error: " + str(e))
print
print "\nChecking ports for IPv6 - DONE"
Here is the thread class where I call checkPort method:
class checkPort_IPv6_thread(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
self.keepRunning = True
def run(self):
while self.keepRunning:
args = self.queue.get()
id = args[0]
address = args[1]
port443 = 0
port21 = 0
port80 = 0
#---------------- Check ports for IPv6 ----------------
if str(address) != "N/A":
port443 = checkPort("TCP",str(address), 443)
port21 = checkPort("TCP",str(address), 21)
port80 = checkPort("TCP",str(address), 80)
lock.acquire()
try:
cur.execute("UPDATE ipv6 SET port_443=" + str(port443) + " WHERE id_ipv6 =" + str(id))
cur.execute("UPDATE ipv6 SET port_21=" + str(port21) + " WHERE id_ipv6 =" + str(id))
cur.execute("UPDATE ipv6 SET port_80=" + str(port80) + " WHERE id_ipv6 =" + str(id))
db.commit()
except Exception as e:
sys.stderr.write("Error: " + str(e))
except:
db.rollback()
lock.release()
self.queue.task_done()
def stop(self):
self.keepRunning = False
And the checkPort method:
def checkPort(typ, address, port):
if typ == "TCP":
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
pom = 0 # 0/1 = True/False
try:
s.settimeout(2) # timeout 1.5 sekundy
s.connect((str(address), port))
s.settimeout(None)
#time.sleep(0.5)
pom = 1
print str(address) + " >> on port: " + str(port) + " >> Connection was successfull"
except socket.timeout:
print str(address) + " >> on port: " + str(port) + " >> * Error: Timed out *"
except socket.error as e:
if e.errno == 10061:
print str(address) + " >> on port: " + str(port) + " >> * No connection could be made - target machine refused it *"
except Exception as ex:
sys.stderr.write("Error: " + str(ex))
return pom

The following may help you along with your program. It is written for Python 3.x. I believe that the main problem is on the line args = self.queue.get() of your program. It should be fixed down below.
import multiprocessing
import queue
import threading
import sys
import socket
MAX_THREAD_COUNT = multiprocessing.cpu_count()
def main(cur):
cur.execute("""UPDATE ipv6
SET port_21 = 0,
port_80 = 0,
port_443 = 0
WHERE address = 'N/A'""")
q = queue.Queue()
for _ in range(MAX_THREAD_COUNT):
CheckPort(q).start()
cur.execute("""SELECT id_ipv6,
address
FROM ipv6
WHERE address != 'N/A'""")
for row in cur.fetchall():
q.put(row)
q.join()
for thread in threading.enumerate():
if isinstance(thread, CheckPort):
thread.stop()
thread.join()
class CheckPort(threading.Thread):
def __init__(self, q):
super().__init__()
self.__q = q
self.__run = True
def stop(self):
self.__run = False
def run(self):
while self.__run:
try:
id, address = self.__q.get(True, 1)
except queue.Empty:
continue
with lock:
try:
cur.execute('''UPDATE ipv6
SET port_21 = ?,
port_80 = ?,
port_443 = ?
WHERE id_ipv6 = ?''',
self.check_port(address, 21),
self.check_port(address, 80),
self.check_port(address, 443),
id)
db.commit()
except Exception as error:
print('Error:', error, file=sys.stdout)
self.__q.task_done()
#staticmethod
def check_port(address, port):
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
sock.settimeout(2)
try:
sock.connect((address, port))
except socket.timeout:
return 0
else:
sock.shutdown(socket.SHUT_RDWR)
sock.close()
return 1
if __name__ == '__main__':
try:
main(cur)
except Exception as error:
print('Error:', error, file=sys.stdout)

From an OP's edit to his question:
Solution:
Thanks to Noctis Skytower the solution is to catch queue.empty exception:
try:
id, address = self.queue.get(True, 1)
except Queue.Empty:
continue

Related

OSError: [WinError 10022] An invalid argument was supplied. Process finished with exit code 1

I was following a tutorial and got stuck here. When I ran the same code for the first time, It went well. But from the second time, it shows error in the same code. I restarted my computer and then it ran once and from the second time, it again shows error.
#THE CODE:
import socket
from _thread import *
import sys
server = "192.168.0.102"
port = 5555
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((server, port))
except socket.error as e:
str(e)
s.listen(2)
print("Waiting for a connection , Server Started")
def read_pos(str):
str = str.split(",")
return int(str[0]), int(str[1])
def make_pos(tup):
return str(tup[0]) + "," + str(tup[1])
pos = [(0, 0), (100, 100)]
def threaded_client(conn, player):
conn.send(str.encode(make_pos(pos[player])))
reply = ""
while True:
try:
data = read_pos(conn.recv(2048).decode())
pos[player] = data
if not data:
print("Disconnected")
break
else:
if player == 1:
reply = pos[0]
else:
reply = pos[1]
print("Received : ", data)
print("Sending: ", reply)
conn.sendall(str.encode(make_pos(reply)))
except:
break
print("Lost Connection")
conn.close()
currentPlayer = 0
while True:
conn, addr = s.accept()
print("Connected to : ", addr)
start_new_thread(threaded_client, (conn, currentPlayer))
currentPlayer += 1
Try using a different IP address. The code worked correctly when I used 127.0.0.1 which is the local machine.
server = "127.0.0.1"
port = 5555

Multithreading does not work for bi-directional udp communication python

Output at console server
I am trying to write bi-directional UDP communication using multithread but it crashes after sending two messages. Also i am new to threading so please post your solution on this.
Thanks
Server side:
import threading
from threading import Thread
import socket
from socket import *
import time
import pymongo
from datetime import datetime
from time import ctime
#broadcast works for this program
import netifaces
import os
import re
import struct
class cont():
def get_msg(self):
UDP = "192.168.1.27"
port = 4343
address = UDP, port
self.sock = socket(AF_INET, SOCK_DGRAM)
self.sock.bind(address)
while True:
r = self.sock.recvfrom(1000)
print("controller1: %s" % (r[0]))
reply = input('Main controller : ')
client_address = r[1]
self.sock.sendto(bytearray(reply, "utf-8"), client_address)
t2 = threading.Thread(target=self.get_msg, args=(reply,))
t2.start()
if __name__=='__main__':
c=cont()
#c.broad(msg="")
c.get_msg()
Client side:
UDP=""
port=4343
address=UDP,port
client=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
while(True):
msg=input("Controller1")
client.sendto(bytearray(msg,"utf-8"),address)
reply=client.recvfrom(1000)
recved=str(reply)
print("Main Controller:% s" % recved))
Output required :
Server Console:
Client:b'hello'
Server:b'hi
Client Console:
Client: b'hello'
Server : (b'hi',('ip',port)
Here is a TCP class I made for communicating with my robots, can be easily modified for UDP. Might seem like a lot of code, but it's what it takes for "reliable" "two way" communication, without blocking your main program. I use processes instead of threads because threads in python aren't "real" threads due to the global interpreter lock.
import socket
from multiprocessing import Process, Queue, Event, Value
import traceback
class SocketComm(object):
def __init__(self,port):
self.address = ""
self.otherAddress = object
self.port = port
self.finished = Value("b", True)
self.inbox = Queue()
self.connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.getMessagesProcess = Process(target=self.getMessages)
self.getMessagesProcess._stop_event = Event()
self.getMessagesProcess.daemon = True
self.connected = False
self.timeout = 3
return
def setupLine(self, addr):
self.address = addr
if self.address is "": #i.e. server on raspberry pi
try:
self.connection.settimeout(self.timeout)
self.connection.bind((self.address, self.port))
print("binding with port: " + str(self.port))
self.connection.listen(1)
self.connection, self.otherAddress = self.connection.accept()
print("connected to client at: " + self.otherAddress[0])
except socket.error as e:
print(str(e))
return False
else:
try:
#print("connecting to port: " + str(self.port))
self.connection.connect((self.address, self.port)) # i.e. client
print("connected to server")
except socket.error as e:
#print(str(e))
return False
self.getMessagesProcess.start()
self.connected = True
self.finished.value = False
print("inbox at: " + str(id(self.inbox)))
return True
def sendMessage(self, msg):
try:
self.connection.send(str.encode(msg))
#print("sent: " + str(msg))
except Exception as e:
pass
#print(str(e))
#traceback.print_exc()
#print("exception caught.")
return
def getMessages(self):
#print("getting messages now")
self.connection.settimeout(1)
while(not self.finished.value):
#print("checking inbox")
#print("inbox length: " + str(len(self.inbox)))
try:
received = self.connection.recv(1024)
decoded = received.decode('utf-8')
if len(decoded) > 0:
if(decoded == "end"):
self.finished.value = True
else:
self.inbox.put(decoded)
print("received: " + str(decoded))
except socket.error as e:
if(type(e).__name__ == "timeout"):
pass
else:
print("endpoint closed.")
self.finished.value = True
return
def closeConnection(self):
if(self.connected):
self.finished.value = True
self.getMessagesProcess._stop_event.set()
self.sendMessage("end")
try:
self.getMessagesProcess.join()
except:
print("process already finished.")
self.connection.close()
return
##
##if(__name__ == "__main__"):
## robotClient = SocketComm(5555)
## robotClient.setupLine("127.0.0.1")
## while(robotClient.finished.value == False):
## val = input("enter something: ")
## if(len(val) > 0):
## robotClient.sendMessage(val)
##
##
##if(__name__ == "__main__"):
## try:
## robotServer = SocketComm(5555)
## print("waiting for client to connect...")
## robotServer.setupLine("")
## print("connected!")
## while(robotServer.finished.value == False):
## val = input("enter something: ")
## if(len(val) > 0):
## robotServer.sendMessage(val)
## except:
## pass
## finally:
## robotServer.closeConnection()
## sys.exit(0)

When I write "dir" I get this error TypeError: descriptor 'encode' requires a 'str' and the reason is "if len(str.encode(cmd)) > 0

from __future__ import print_function
import socket
import sys
def socket_create():
try:
global host
global port
global s
host = ''
port = 9999
s = socket.socket()
except socket.error as msg:
print("Socket creation error: " + str(msg))
def socket_bind():
try:
global host
global port
global s
print("Binding socket to port: " + str(port))
s.bind((host,port))
s.listen(5)
except socket.error as msg:
print("Socket binding error: " + str(msg) + "\n" + "Retrying...")
socket_bind()
def socket_accept():
conn, address = s.accept()
print("Connection has been established | " + "IP " + address[0] + "| Port " + str(address[1]))
send_commands(conn)
conn.close()
def send_commands(conn):
while True:
cmd = input()
if cmd == 'quit':
conn.close()
s.close()
sys.exit()
if len(str.encode(cmd)) > 0:
conn.send(str.encode(cmd))
client_response = str(conn.recv(1024), "utf-8")
print(client_response, end ="")
def main():
socket_create()
socket_bind()
socket_accept()
main()
Use raw_input not input (cf. https://docs.python.org/2/library/functions.html). Have a look at the value and the type of what is returned to you by input.
Also str takes only one argument (few lines down, in your if).

" Threading Error" Couldn't run the shell after converting single client shell into multi client shell by adding "threads"

"This code runs but won't be able to do anything,when I run this code what I am expecting is that I am going to get my shell but I get nothing it all worked fine before threading but when i edited this to convert it into multi_client then it all happened"
import socket
import threading
from queue import Queue
import time
NUMBER_OF_THREADS = 2
JOB_NUMBER = [1,2]
queue = Queue()
all_connections = []
all_addresses = []
def socket_create():
try:
global host
global port
global s
host = ''
port = 9999
s = socket.socket()
except socket.error as msg:
print("Socket creation error: " + str(msg))
def socket_bind():
try:
global host
global port
global s
s.bind((host,port))
s.listen(5)
except socket.error as msg:
print("Socket binding error: " + str(msg))
time.sleep(5)
socket_bind()
def accept_connections():
for c in all_connections:
c.close()
del all_connections[:]
del all_addresses[:]
while 1:
try:
conn, address = s.accept()
conn.setblocking(1)
all_connections.append(conn)
all_addresses.append(address)
print("\nConnection has been established: " + address[0])
except:
print("Errorr accepting connections")
def start_turtle():
while True:
cmd = input('turtle> ')
if cmd == 'list':
list_connections()
elif 'select' in cmd:
conn = get_target(cmd)
if conn is not None:
send_target_commands(conn)
else:
print("Command not Recognized")
def list_connections():
results = ''
for i, conn in enumerate(all_connections):
try:
conn.send(str.encode(' '))
conn.recv(20480)
except:
del all_connections[i]
del all_addresses[i]
continue
results += str(i) + ' ' + str(all_addresses[i][0]) + ' ' + str(all_addresses[i][1]) + '\n'
print('------ Clients -----' + '\n' + results)
def get_target(cmd):
try:
target = cmd.replace('select ', '')
target = int(target)
conn = all_connections[target]
print("You are now connected to " + str(all_addresses[target][0]))
print(str(all_addresses[target][0]) + "> ", end = "")
return conn
except:
print("Now a valid selection")
return None
def send_target_commands(conn):
while True:
try:
cmd = input()
if len(str.encode(cmd)) > 0:
conn.send(str.encode(cmd))
client_response = str(conn.recv(20480), "utf-8")
print(client_response, end="")
if cmd == 'quit':
break
except:
print("Connection was lost")
break
def create_workers():
for _ in range(NUMBER_OF_THREADS):
t = threading.Thread(target=work)
t.daemon = True
t.start()
def work():
while True:
x = queue.get()
if x == 1:
socket_create()
socket_bind()
accept_connections()
if x == 2:
start_turtle()
queue.task_done()
def create_jobs():
for x in JOB_NUMBER:
queue.put(x)
queue.join()
create_workers()
create_jobs()

Python script stuck at queue.join()

I am trying to implement a server for handling many clients (from thenewboston python reverse shell tutorials). I have the exact same code but when i run the script it gets stuck at queue.join(). How to make it work? I am unable to figure it out.
Here is the code
import socket
import sys
import threading
from queue import Queue
NUMBER_OF_THREADS = 2
JOB_NUMBER = [1, 2]
queue = Queue()
all_connections = []
all_addresses = []
# thread 1
# create socket (allows two computers to connect)
def socket_create():
try:
global host # ip address of the server
global port # port is to identify the kind of data
global s
host = ''
port = 9999
s = socket.socket()
except socket.error as msg:
print("Socket creation error: " + str(msg))
return
# bind socket to port and wait for connection from client
def socket_bind():
try:
global host
global port
global s
print("Binding socket to port: " + str(port))
s.bind((host, port))
s.listen(5)
# 5 is the no. of conections that can be made before server starts rejecting other requests
except socket.error as msg:
print("Socket binding error: " + str(msg) + "\n" + "Retrying...")
socket_bind()
return
# accept connections from multiple clients and save to list
def accept_connections():
for c in all_connections:
c.close()
del all_connections[:]
del all_addresses[:]
while 1:
try:
conn, address = s.accept()
conn.setblocking(1)
all_connections.append(conn)
all_addresses.append(address)
print("\nConnection has been establish: " + address[0])
except:
print("Error accepting connections")
return
# thread 2
# custom command promt for sending commands remotely
def start_turtle():
while True:
cmd = input('turtle> ')
if cmd == 'list':
list_connections()
elif 'select' in cmd:
conn = get_target(cmd)
if conn is not None:
send_target_commands(conn)
else:
print("Command not recognized")
return
# listing all the connections with indexing in the custom promt
def list_connections():
results = ''
for i, conn in enumerate(all_connections):
try:
conn.send(str.encode(' '))
conn.recv(20480)
except:
del all_connections[i]
del all_addresses[i]
continue
results += str(i) + ' ' + str(all_addresses[i][0]) + ' ' + str(all_addresses[i][1]) + '\n'
print('-----Clients-----' + '\n' + results)
return
# select a target client
def get_target(cmd):
try:
target = cmd.replace('select ', '')
target = int(target)
conn = all_connections[target]
print("You are now connected to " + str(all_addresses[target][0]))
print(str(all_addresses[target][0]) + '> ', end="")
return conn
except:
print("Not a valid selection")
return None
return
# connect with remote target client
def send_target_commands(conn):
while True:
try:
cmd = input()
if len(str.encode(cmd)) > 0:
conn.send(str.encode(cmd))
client_response = str(conn.recv(20480), "utf-8")
print(client_response, end="")
if cmd == "quit":
break
except:
print("Connection was lost")
break
return
# create worker threads
def create_workers():
for _ in range(NUMBER_OF_THREADS):
t = threading.Thread(target=work)
t.daemon = True
t.start
return
# do the next job in the queue (one handles connections, other sends commands)
def work():
while True:
x = queue.get()
if x == 1:
socket_create()
socket_bind()
accept_connections()
if x == 2:
start_turtle()
queue.task_done()
return
# create jobs for later extracting them and assigning them to the threads
def create_jobs():
for x in JOB_NUMBER:
queue.put(x)
queue.join()
return
def main():
create_workers()
create_jobs()
if __name__ == '__main__':
main()
Since you are using infinite loops (while True) at start_turtle and (while 1) at accept_connections they are not returning.
Since they don't return the func work never calls queue.task_done(), so the queue stuck joining.
I'm afraid you need to do one of the following:
start both start_turtle and accept_connections in parallel processes or threads.
Be sure they should call the queue.task_done().
For instance, you may include the queue as parameter and call it before starting the infinite loops (second option).
def work():
while True:
x = queue.get()
if x == 1:
socket_create()
socket_bind()
accept_connections(queue) # call queue.task_done() there
if x == 2:
start_turtle(queue) # call queue.task_done() in start_turtle
return
def start_turtle(queue):
queue.task_done() # Join one item from the queue
while True:
cmd = input('turtle> ')
if cmd == 'list':
list_connections()
elif 'select' in cmd:
conn = get_target(cmd)
if conn is not None:
send_target_commands(conn)
else:
print("Command not recognized")
return
On the other hand, in your create_workers you don't call the start method of the thread so your workers didn't really start.
Perhaps this is a typo.
def create_workers():
for _ in range(NUMBER_OF_THREADS):
t = threading.Thread(target=work)
t.daemon = True
# t.start # Not starting the Thread
t.start() # You need to call the start method
return

Categories