We Keep Coding

Multi threading not executing Tasks simultaneously

I wrote a python code for my project that stores file name of files on the SFTP server in a list and does the same for local directory and compares the 2 lists and then download the uncommon files. code takes its server credentials from a separate text file of format
CONTENTS OF TEXT FILE (config9.txt)
127.0.0.1,username,password,log22.txt,/new45,C:\Users\udit\Desktop\ftp2\bob16
my program executed one by one for different IP's. I wanted it to run it parallely for all IP's using multithreading. I modified my code but it still executes it one by one. Whats the error in it?
import os
import pysftp
import csv
import socket
from stat import S_IMODE, S_ISDIR, S_ISREG
import time
import threading
from threading import Thread
from time import sleep
import os.path
import shutil
import fileinput
import lock
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
def mainprocess():
with open("config9.txt", "r") as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
print(row)
IP=row[0]
myUsername=row[1]
myPassword=row[2]
txtfile=row[3]
remotepath=row[4]
localpath=row[5]
with pysftp.Connection(host=IP, username=myUsername, password=myPassword, cnopts=cnopts) as sftp:
r = str(socket.gethostbyaddr(IP))
print("connection successful with " + r)
def get_file2(sftp, remotedir):
result = []
for entry in sftp.listdir_attr(remotedir):
remotepath = remotedir + "/" + entry.filename
mode = entry.st_mode
if S_ISDIR(mode):
result += get_file2(sftp, remotepath)
elif S_ISREG(mode):
if (time.time() - entry.st_mtime) // (24 * 3600) > 0:
result.append(entry.filename)
return result
remote_path = remotepath
d = get_file2(sftp, remote_path)
def process():
myName = []
with open(filename, 'r+') as f:
for name in f.readlines():
while '\n' in name:
name = name.replace('\n', '')
myName.append(name)
print(myName)
print(len(myName))
import os
filtered_list = [string for string in d if string not in myName]
print("filtered list:", filtered_list)
print(len(filtered_list))
local_path =localpath
def compare_files(sftp, remotedir, remotefile, localdir, preserve_mtime=True):
remotepath = remotedir + "/" + remotefile
localpath = os.path.join(localdir, remotefile)
mode = sftp.stat(remotepath).st_mode
if S_ISDIR(mode):
try:
os.mkdir(localpath, mode=777)
except OSError:
pass
compare_files(sftp, remotepath, localpath, preserve_mtime)
elif S_ISREG(mode):
sftp.get(remotepath, localpath, preserve_mtime=True)
for file in filtered_list:
compare_files(sftp, remote_path, file, local_path, preserve_mtime=False)
with open(filename, 'a') as f:
for item in filtered_list:
f.write("%s\n" % item)
filename=txtfile
try:
file = open(filename, 'r')
process()
except IOError:
file = open(filename, 'w')
process()
t=time.time()
t1=threading.Thread(target=mainprocess(),args=())
t1.start()
t1.join()

save error compile message in txt file in python3

Could you help me that when I compile my program and if I have an error how can I redirect this error message in text file, before compilation terminate?
I wrote this code, But my problem is i want when i have an error after that my COMP_ERR.txt file create and write error inside this file.
but in my code this create before
import urllib.request
import os
import tempfile
import sys
import fileinput
original_stderr = sys.stderr
f_error = open("COMP_ERR.txt", "w")
sys.stderr = f_error
try:
url = sys.argv[1]
path_snip_file = sys.argv[2]
#url = 'http://pages.di.unipi.it/corradini/Didattica/AP-17/PROG-ASS/03/ClassWithTest.java'
#path_snip_file = "C:/Users/user/AppData/Local/Programs/Python/Python36/snip1.java"
path_remote_file_inOurComp = "C:/Users/user/AppData/Local/Programs/Python/Python36/ClassWithTest.java"
path_remote_file_inOurCom = "C:/Users/user/AppData/Local/Programs/Python/Python36/ClassWithTest1.java"
gt_url = urllib.request.urlretrieve(url)
print("the URL is: ")
print(gt_url)
link = input("input the path of file: ")
"""
f = open(link, "r")
for line in f:
print(line, end='')
f.close()
"""
#--------------------]
#copy snipper java file inside remote file
#[--------------------
with open(path_remote_file_inOurComp, 'w') as modify_file:
with open (link, 'r') as r_file:
for line in r_file:
if " public static void main(" not in line:
modify_file.write(line)
else:
with open(path_snip_file, 'r') as snip_file:
for lines in snip_file:
modify_file.write(lines)
modify_file.write('\n'+line)
#-------------------------------------------------------------
#refactor
#-------------------------------------------------------------
with open(path_remote_file_inOurCom, 'w') as ft:
with open(path_remote_file_inOurComp, 'r') as file_n:
for line in file_n:
line = line.replace("System.out.println(", "System.out.println(insertLength(")
line = line.replace(";", ");")
ft.write(line)
except IndexError:
print("Not enough input! ! !")
sys.stderr = original_stderr
f_error.close()

Check if it is useful to you
import sys
try:
raise
except Exception as err:
**exc_type, exc_obj, exc_tb = sys.exc_info()** # this is to get error line number and description.
file_name = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] # to get File Name.
error_string="ERROR : Error Msg:{},File Name : {}, Line no : {}\n".format(err,file_name,exc_tb.tb_lineno))
file_log = open("error_log.log", "a")
file_log.write(error_string)
file_log.close()

logf = open("download.log", "w")
for download in download_list:
try:
# code to process download here
except Exception as e: # most generic exception you can catch
logf.write("Failed to download {0}: {1}\n".format(str(download), str(e)))
# optional: delete local version of failed download
finally:
# optional clean up code
pass
apart from this approach, you can use logging library to save each and every logs of your application.
Following is the method 2 of the problem of saving logs.
import logging
logging.basicConfig(filename='gunicon-server.log',level=logging.DEBUG,
format='[%(levelname)s]: [%(asctime)s] [%(message)s]', datefmt='%m/%d/%Y %I:%M:%S %p')
try:
# code to process download here
except Exception as e: # most generic exception you can catch
logging.error(str(e))
finally:
# optional clean up code
pass

import sys
sys.stderr = open('errorlog.txt', 'w')
# do whatever
sys.stderr.close()
sys.stderr = sys.__stderr__
see this for more details : https://ubuntuforums.org/showthread.php?t=849752

QPixmap Line Being Skipped

I have the following piece of code:
def run(self):
filename = QtGui.QFileDialog.getSaveFileName(self, "Save file", "", ".inp")
if filename == "":
pass
elif filename != "":
nfile = open(filename, 'w')
self.pixmap1 = QtGui.QPixmap('yellow.jpg')
self.canvas1.setPixmap(self.pixmap1)
self.cursor.movePosition(self.cursor.Start, self.cursor.MoveAnchor)
self.cursor.movePosition(self.cursor.End, self.cursor.KeepAnchor)
text = self.cursor.selection()
text1 = text.toPlainText()
nfile.write(text1)
nfile.close()
fileInfo = QtCore.QFileInfo(filename)
name = fileInfo.baseName()
import os
os.system("rungms {}.inp 13-64 {} {} {}.out".format(name, self.cores_val, self.clusters_val, name))
from itertools import islice
import sys
with open("{}.out".format(name)) as searchfile:
for line in searchfile:
if 'TERMINATED NORMALLY' in line:
self.pixmap1 = QtGui.QPixmap('green.jpg')
self.canvas1.setPixmap(self.pixmap1)
elif 'job aborted' in line:
self.pixmap1 = QtGui.QPixmap('red.jpg')
self.canvas1.setPixmap(self.pixmap1)
else:
pass
else:
pass
My problem is that the two lines involving QPixmap('yellow.jpg') are not being read after saving my file. However, if I move these two lines in between the first and second lines of the code (i.e. before filename = QtGui...), then they do work.
Can someone explain to me why this is the case?

Reading from multiple files and storing data in a list

I am trying to read print search for all files in a directory and store contents in each file in a list to be used.
My problem is when i use print to debug if the file exists, it prints out the current file or first file in the list. However, It complains that file is not found when i try to read from this file
import re
import os
# Program to extract emails from text files
def path_file():
#path = raw_input("Please enter path to file:\n> ")
path = '/home/holy/thinker/leads/'
return os.listdir('/home/holy/thinker/leads') # returns a list like ["file1.txt", 'image.gif'] # need to remove trailing slashes
# read a file as 1 big string
def in_file():
print path_file()
content = []
for a_file in path_file(): # ['add.txt', 'email.txt']
print a_file
fin = open(a_file, 'r')
content.append(fin.read()) # store content of each file
print content
fin.close()
return content
print in_file()
# this is the error i get
""" ['add.txt', 'email.txt']
add.txt
Traceback (most recent call last):
File "Extractor.py", line 24, in <module>
print in_file()
File "Extractor.py", line 17, in in_file
fin = open(a_file, 'r')
IOError: [Errno 2] No such file or directory: 'add.txt'
"""
The error I get is aboive

os.listdir will return you only file name. You have to directory name on before that file name.
Its trying to open add.txt in same directory where you ran your program. Please add directory name before file name.
def path_file():
#path = raw_input("Please enter path to file:\n> ")
path = '/home/holy/thinker/leads/'
return [os.path.join(path, x) for x in os.listdir(path)]

you should use the full path of the file you want to read.
so please do fin = open(os.path.join(r'/home/holy/thinker/leads/', a_file), 'r')

Here's a rewrite using glob to limit which files are considered;
import glob
import os
import re
import sys
if sys.hexversion < 0x3000000:
# Python 2.x
inp = raw_input
else:
# Python 3.xrange
inp = input
def get_dir(prompt):
while True:
dir_name = inp(prompt)
dir_name = os.path.join(os.getcwd(), dir_name)
if os.path.isdir(dir_name):
return dir_name
else:
print("{} does not exist or is not a directory".format(dir_name))
def files_in_dir(dir_name, file_spec="*.txt"):
return glob.glob(os.path.join(dir_name, file_spec))
def file_iter(files):
for fname in files:
with open(fname) as inf:
yield fname, inf.read()
def main():
email_dir = get_dir("Please enter email directory: ")
email_files = files_in_dir(email_dir, "*.eml")
print(email_files)
content = [txt for fname,txt in file_iter(email_files)]
print(content)
if __name__=="__main__":
main()
and a trial run looks like
Please enter email directory: c:\temp
['c:\\temp\\file1.eml', 'c:\\temp\\file2.eml']
['file1 line one\nfile1 line two\nfile1 line three',
'file2 line one\nfile2 line two']

Not able to raise exception in the code

Trying to resolve the issue But no luck yet. Can some body tell me what's the issue. Tried re indenting the code.
I am not able to print the File Not found text in the exception block in the code.
Is this the indentation issue ?
Code Snippet:
from xlutils.copy import copy
from xlrd import open_workbook
import xlwt
import os
import shutil
import glob
def openexcel_main():
book = open_workbook('input.xls',formatting_info=True)
sheet = book.sheet_by_index(0)
wb = copy(book)
w_sheet = wb.get_sheet(0)
folder_name=['do_not_delete','internal_builds']
for j in range (0,2):
folder=folder_name.pop()
for i in range (1,(sheet.nrows)):
cell_test_group = sheet.cell(i,0)
data=str(cell_test_group.value)
print '#####################################'
print data
list=[]
source_path='/mnt/'+folder+'/pybuild/'+data+'/MAIN/'
if os.path.exists(source_path):
try:
os.chdir(source_path)
all_subdirs = [d for d in os.listdir('.') if os.path.isdir(d)]
for dirs in all_subdirs:
dir = os.path.join('/mnt/'+folder+'/pybuild/'+data+'/MAIN/', dirs)
os.chdir(dir)
current = os.getcwd()
new = str(current).split("/")[6]
list.append(new)
list.sort()
val=list
for i in range (1,4):
if val==[]:
break
else:
print i
current_build_number=val.pop()
print 'Current_Build:'+current_build_number
source_path_copy = r""+ source_path+"/"+current_build_number+"/"
print 'Copying From:'+ source_path_copy
dest_path = r"/home/builds_repo/"+folder+"/pybuild/"+data+"/MAIN/"+current_build_number+"/"
os.chdir(source_path_copy)
file_name=(glob.glob('*[_bin].*')).pop()
print 'File_Copied:'+ file_name
if not os.path.exists(dest_path):
os.makedirs(dest_path)
shutil.copyfile(source_path_copy + file_name, dest_path + file_name)
except:
print'File Not Found ..'
raise
def main():
openexcel_main()
main()

Try some good editors like pyscripter ,Emacs to make your pythonic life easy :)
I have tried to intend your code ...
from xlutils.copy import copy
from xlrd import open_workbook
import xlwt
import os
import shutil
import glob
def openexcel_main():
book = open_workbook('input.xls',formatting_info=True)
sheet = book.sheet_by_index(0)
wb = copy(book)
w_sheet = wb.get_sheet(0)
folder_name=['do_not_delete','internal_builds']
for j in range (0,2):
folder=folder_name.pop()
for i in range (1,(sheet.nrows)):
cell_test_group = sheet.cell(i,0)
data=str(cell_test_group.value)
print '#####################################'
print data
list=[]
source_path='/mnt/'+folder+'/pybuild/'+data+'/MAIN/'
if os.path.exists(source_path):
try:
os.chdir(source_path)
all_subdirs = [d for d in os.listdir('.') if os.path.isdir(d)]
for dirs in all_subdirs:
dir = os.path.join('/mnt/'+folder+'/pybuild/'+data+'/MAIN/', dirs)
os.chdir(dir)
current = os.getcwd()
new = str(current).split("/")[6]
list.append(new)
list.sort()
val=list
for i in range (1,4):
if val==[]:
break
else:
print i
current_build_number=val.pop()
print 'Current_Build:'+current_build_number
source_path_copy = r""+ source_path+"/"+current_build_number+"/"
print 'Copying From:'+ source_path_copy
dest_path = r"/home/builds_repo/"+folder+"/pybuild/"+data+"/MAIN/"+current_build_number+"/"
os.chdir(source_path_copy)
file_name=(glob.glob('*[_bin].*')).pop()
print 'File_Copied:'+ file_name
if not os.path.exists(dest_path):
os.makedirs(dest_path)
shutil.copyfile(source_path_copy + file_name, dest_path + file_name)
except Exception ,e: #Use Exception if not sure which exception will raise
print'File Not Found ..',e
#raise
def main():
openexcel_main()
if __name__ == '__main__': #Use main
main()
Line:
print'File Not Found ..'
, should be in else loop of
os.path.exists(source_path):
to check and print source path is not exists