We Keep Coding

decrease time execution of a python code

how can i decrease the time execution of this code:
tab_voyelle is a table of 2669433 different words
for elem in tab_voyelle:
words_match = []
elem = elem.strip()
Tab_elem_out = CheckImpli(elem,tab_voyelle)
if len(Tab_elem_out) != 0:
MonFichINImpli = open('/home/user/Bureau/MesTravaux/MatchingCorpus/in_imply_result.txt','w')
for i in range(len(Tab_elem_out)):
MonFichINImpli.write(elem + ' ' + Tab_elem_out[i] + '\n')
MonFichINImpli.close()
subprocess.call(['java', '-jar', '/home/user/Bureau/MesTravaux/MatchingCorpus/dist/ImpliMorphVer4.jar','-i', 'in_imply_result.txt','-o','Result_Impli.txt'])
words_match = MatchingArabicWords('Result_Impli.txt','in_imply_result.txt')
if len(words_match) != 0 :
MonFichierResultat.write('<mot id="'+elem+'">'+'\n')
for valeur in words_match:
valeur = valeur.strip()
MonFichierResultat.write('<val>'+valeur+'</val>'+'\n')
t = tab_voyelle[tab_voyelle.index(valeur)]
del tab_voyelle[tab_voyelle.index(valeur)]
MonFichierResultat.write('</mot>' + '\n')
if len(words_match) == 0:
MonFichierResultat.write('<mot id="'+elem+'">'+'\n'+'<val></val>\n'+'</mot>' + '\n')
MonFichierResultat.write('</matching>' + '\n')
MonFichierResultat.close()

python multiprocessing map_async does take much more time than the sequential

I want to find optimum path for the Travelling salesman problem, it works fine with the sequential algorithm but I have a problem with the parallel algorithm, for example when I run sequentially everything is ok but in parallel, it took 200x more time in 4 cores computer.
Here is my sequential code:
#!/usr/local/bin/python
#Traveling Salesman Solution
#Scott Stevenson, Jack Koppenaal, John Costantino
import time, itertools, math
#Get the city file
def getFile():
cityFile = 'in.txt'
try:
f = open(cityFile,'r')
return f
except Exception as e:
print(cityFile + ' could not be opened: ' + str(e))
def find_max():
file = open('in.txt','r')
max = file.read().split()
return max[1]
def initfiles(path):
file = open(path, 'r')
max = file.read().splitlines()
file.close()
file = open(path, 'r')
num = file.read().split()
file.close()
final = list()
final.append(str(num[0] + ' ' + num[1] + ' OK'))
if (num[2] != 'OK'):
print('File has been edited.')
file_write = open(path, 'wt')
for i in range(1, int(num[0]) + 1):
final.append(str(i) + ' ' + str(max[i]))
for line in range(len(final)):
file_write.write(str(final[line]) + '\n')
file_write.close()
#Get distance between 2 cities
#Cities stored as [ident, x, y]
def getDistance(city1, city2):
return math.sqrt((int(city2[1]) - int(city1[1]))**2 + (int(city2[2])-int(city1[2]))**2)
#Get the cities from a specified city file
def getCities(cityFile):
for line in cityFile:
try:
#Lines split by a space char will look like:
#[ident, x-coord, y-coord]
ident = line.split(' ')[0]
x = line.split(' ')[1]
y = line.split(' ')[2].strip('\n')
#print(x,y,ident)
#If the ident is not an int (not a city) skip it otherwise add it
if ident.isdigit():
#Cities are just lists with values (almost a pseudo-class)
city = [int(ident), int(x), int(y)]
cities.append(city)
except:
#The ident was not an int so we skip (pass) it and move on to the next
pass
#A function for bruteforcing the TSP problem based on a list of cities
def bruteForce(cities):
global maxweight
#Tours are also stored as pseudo-class lists
#tour[0] is the path and tour[1] is the weight
#Make a start tour with a weight of infinity so all other tours will be smaller
tour = [[], float("inf")]
permparm = []
#In order to get all permutations we need an array containing the values 1 through n
#These values are the idents of the cities (their 0th element) so we get and add them
for city in cities:
permparm.append(city[0])
#We now generate all permutations of n length from the array containing 1 - n idents
#and loop through them looking for the smallest distance
for perm in list(itertools.permutations(permparm, len(permparm))):
#Get the total weight of the permutation
dist = getWeight(perm)
#Make a new tour to represent the current permutation
thisTour = [perm, dist]
#If the current tour is shorter than the old tour, point the old tour to the new one
if thisTour[1] < tour[1] and thisTour[1]<=int (maxweight):
tour = thisTour
return tour
#Once we have gone through every permutation we have the shortest tour so return it
return tour
#A function to get the total weight of a path
#This function is messy because of an off-by-1 error introduced by the tour file starting at 1 instead of 0
def getWeight(perm):
#Set the initial distance to 0
dist = 0
#We now need to calculate and add the distance between each city in the path from 0 to n
for index in range(len(perm)):
try:
#Pass the 2 cities to the distance formula and add the value to dist
dist += getDistance(cities[perm[index]-1], cities[perm[index+1]-1])
except:
#We don't need to check bounds because the final pass will throw an out-of-bounds
#exception so we just catch it and skip that calculation
pass
#All TSP solutions are cycles so we now have to add the final city back to the initial city to the total dist
#Python has a nifty convention where list[-1] will return the last element, list[-2] will return second to last, etc.
dist += getDistance(cities[perm[-1]-1], cities[perm[0]-1])
#We now have the total distance so return it
#if (int(dist)<=80.0):
return dist
#A function to write the output of a tour to a file in a specified format
def toFile(tour):
loc = input('Enter the location where you would like to save the tour:\n')
fname = cityFile.name
#Index for the last file separator to get ONLY the file name not its path
sep_index = 0
#Linux/OSX files use /'s to separate dirs so get the position of the last one in the name
if '/' in fname:
sep_index = fname.rindex('/')+1
#Windows files use \'s to separate dirs so get the position of the last one in the name
if '\\' in fname:
sep_index = fname.rindex('\\')+1
#Create the header for the output file
header = ('NAME : ' + str(fname[sep_index:-4]) + '.opt.tour\n'
'COMMENT : Optimal tour for ' + str(fname[sep_index:]) + ' (' + str(tour[1]) + ')\n'
'TYPE : Tour\n'
'DIMENSON : ' + str(len(tour[0])) + '\n'
'TOUR_SECTION\n')
#Create the trailer for the output file
trailer = "-1\nEOF\n"
#Create the output file and write the results to it
try:
f = open(loc,'w')
f.write(header)
for city in tour[0]:
f.write(str(city) + '\n')
f.write(trailer)
f.close()
print ('Successfully saved tour data to: ' + loc)
except Exception as e:
print (loc + ' could not be written to: ' + str(e))
#-------------------The actual script begins here-----------------------
cities = []
cityFile = getFile()
maxweight = find_max()
initfiles('in.txt')
#Start the stopwatch
start = time.time()
getCities(cityFile)
opt_tour = bruteForce(cities)
#Stop the stopwatch
finish = time.time()
print ('The optimum tour is: %s (%f)' % (opt_tour[0], opt_tour[1]))
print ('This solution took %0.3f seconds to calculate.' % (finish-start))
And my parallel code:
#!/usr/local/bin/python
import time, itertools, math
from multiprocessing import Pool,Manager
thisTour = None
tour = None
dist = None
totalct = 0
#Get the city file
def getFile():
cityFile = 'in.txt'
try:
f = open(cityFile,'r')
return f
except Exception as e:
print(cityFile + ' could not be opened: ' + str(e))
def find_max():
file = open('in.txt','r')
max = file.read().split()
return max[1]
def initfiles(path):
global totalct
file = open(path, 'r')
max = file.read().splitlines()
totalct = len(max) - 2
file.close()
file = open(path, 'r')
num = file.read().split()
file.close()
final = list()
#print(totalct)
final.append(str(num[0] + ' ' + num[1] + ' OK'))
if (num[2] != 'OK'):
file_write = open(path, 'wt')
for i in range(1, int(num[0]) + 1):
# print(max[i])
final.append(str(i) + ' ' + str(max[i]))
# print(i)
# print(len(final))
for line in range(len(final)):
# print(final[line])
file_write.write(str(final[line]) + '\n')
print('File has been edited.')
file_write.close()
#Get distance between 2 cities
#Cities stored as [ident, x, y]
def getDistance(city1, city2):
return math.sqrt((int(city2[1]) - int(city1[1]))**2 + (int(city2[2])-int(city1[2]))**2)
#Get the cities from a specified city file
def getCities(cityFile):
for line in cityFile:
try:
#Lines split by a space char will look like:
#[ident, x-coord, y-coord]
ident = line.split(' ')[0]
x = line.split(' ')[1]
y = line.split(' ')[2].strip('\n')
#If the ident is not an int (not a city) skip it otherwise add it
if ident.isdigit():
#Cities are just lists with values (almost a pseudo-class)
city = [int(ident), int(x), int(y)]
cities.append(city)
except:
#The ident was not an int so we skip (pass) it and move on to the next
pass
#A function for bruteforcing the TSP problem based on a list of cities
def bruteForce(perm):
global maxweight
global tour
global thisTour
dist = getWeight(perm)
thisTour = [perm, dist]
#If the current tour is shorter than the old tour, point the old tour to the new one
if thisTour[1] < tour[1] and thisTour[1] <= int(maxweight):
tour = thisTour
return tour
#Once we have gone through every permutation we have the shortest tour so return it
return tour
#A function to get the total weight of a path
#This function is messy because of an off-by-1 error introduced by the tour file starting at 1 instead of 0
def getWeight(perm):
#Set the initial distance to 0
dist = 0
#We now need to calculate and add the distance between each city in the path from 0 to n
for index in range(len(perm)):
try:
#Pass the 2 cities to the distance formula and add the value to dist
dist += getDistance(cities[perm[index]-1], cities[perm[index+1]-1])
except:
#We don't need to check bounds because the final pass will throw an out-of-bounds
#exception so we just catch it and skip that calculation
pass
#All TSP solutions are cycles so we now have to add the final city back to the initial city to the total dist
#Python has a nifty convention where list[-1] will return the last element, list[-2] will return second to last, etc.
dist += getDistance(cities[perm[-1] - 1], cities[perm[0] - 1])
#We now have the total distance so return it
return dist
#A function to write the output of a tour to a file in a specified format
def toFile(tour):
loc = input('Enter the location where you would like to save the tour:\n')
fname = cityFile.name
#Index for the last file separator to get ONLY the file name not its path
sep_index = 0
#Linux/OSX files use /'s to separate dirs so get the position of the last one in the name
if '/' in fname:
sep_index = fname.rindex('/')+1
#Windows files use \'s to separate dirs so get the position of the last one in the name
if '\\' in fname:
sep_index = fname.rindex('\\')+1
#Create the header for the output file
header = ('NAME : ' + str(fname[sep_index:-4]) + '.opt.tour\n'
'COMMENT : Optimal tour for ' + str(fname[sep_index:]) + ' (' + str(tour[1]) + ')\n'
'TYPE : Tour\n'
'DIMENSON : ' + str(len(tour[0])) + '\n'
'TOUR_SECTION\n')
#Create the trailer for the output file
trailer = "-1\nEOF\n"
#Create the output file and write the results to it
try:
f = open(loc,'w')
f.write(header)
for city in tour[0]:
f.write(str(city) + '\n')
f.write(trailer)
f.close()
print ('Successfully saved tour data to: ' + loc)
except Exception as e:
print (loc + ' could not be written to: ' + str(e))
#-------------------The actual script begins here-----------------------
def permutations():
allperm = list(itertools.permutations(act, len(act)))
for i in allperm:
allpermlist.append(list(i))
print('permutations compute has been finished.')
return allpermlist
def allct ():
for i in range(1, totalct):
act.append(i)
def init():
global thisTour
global tour
global dist
if __name__ == "__main__":
cities = []
cityFile = getFile()
maxweight = find_max()
initfiles('in.txt')
manager = Manager()
allpermlist = manager.list()
tour = [[], float("inf")]
allpermlist = []
act = []
allct()
permutations()
#Start the stopwatch
start = time.time()
getCities(cityFile)
with Pool(processes=8, initializer=init) as p:
opt_tour = p.map_async(bruteForce, allpermlist, chunksize=2048)
opt_tour.wait()
print(opt_tour.get())
p.close()
#p.join()
finish = time.time()
print('This solution took %0.3f seconds to calculate.' % (finish-start))
And my in.txt :
9 47 OK
1 13 15
2 4 21
3 7 17
4 8 11
5 10 14
6 2 15
7 14 11
8 15 20
9 13 17

printing output in python to txt file

wrote a simple program and would like to print the output to a text file. I'm trying the following at the very top of my script:
python SagicorPremiumCalculator2.py > textfile.txt
But this is giving me a syntax error in idle. How is my syntax incorrect?
python SagicorPremiumCalculator2.py > textfile.txt
python SagicorPremiumCalculator2.py > textfile.txt
import openpyxl, os, sys
wb = openpyxl.load_workbook('example.xlsx')
Millar_sheet = wb.get_sheet_by_name('Millar')
client = []
output = []
print(os.getcwd())
for rowOfCellObjects in Millar_sheet['A2':'AA13']:
for cellObj in rowOfCellObjects:
#if cellObj.value != None:
#print(cellObj.coordinate, cellObj.value)
client.append(cellObj.value)
#print(client[2]) #client name
print(client[0]) #policy number
print(client[9]) #license plate
#output.append(client[0]) #1
#output.append(client[9]) #2
if client[12] != "Not Applicable":
float(client[12])
print('S.I. = ' + '$' + str("%0.2f" % client[12]))
#output.append("%0.2f" % client[12]) #3
else:
print('S.I. ' + client[12])
print('Basic = ' + '$' + str("%0.2f" % client[13]))
#output.append("%0.2f" % client[13])#3
if client[14] != None:
print('Loading = ' + str(100*client[14]) + '%')
else:
print('No Loading')
val = client[13]
if client[14] == None:
val = client[15] #if there is no loading percentage, val becomes subtotal
else:
val += (val*client[14]) # a = a + b, adds 150 to 1000
print('Subtotal = ' + '$' + str("%0.2f" % val))#good
if client[16] != None:
ATD = val*client[16] #discount amount
print('ATD = ' + str(100*client[16]) + '%, -$' + str("%0.2f" % ATD))
val -= (val*client[16])
print('$' + str("%0.2f" % val))
if client[17] != None:
val -= (val*client[17])
SP = val*client[17] #Discount amount
print('SP = ' + str(100*client[17]) + '%, -$' + str("%0.2f" % SP))
print('$' + str("%0.2f" % val))
if client[18] != None:
val = (val+client[18])
PAB = client[18]
print('PAB = +$' + str(client[18]))
print('$' + str("%0.2f" % val))
if client[19] != None:
NCD = val*client[19]
val -= (val*client[19])#discount amount
print('NCD = ' + str(100*client[19]) +'%, -$' + str("%0.2f" % NCD))
print('$' + "%0.2f" % val)
if client[20] != None:
val = (val+client[20])
print('W/S = +$' + str(client[20]))
print('$' + "%0.2f" % val)
if client[21] != None:
val = (val+client[21])
print('LOU = $' + str(client[21]))
print('$' + "%0.2f" % val)
if val < 750: #Applies minimum premium if val is under 750
print("Minimum premium applied")
val = 750
print('Pre-tax Premium is ' + '$' + str("%0.2f" % val)) #good
if client[23] == 0: #Client over 65, not being charged tax
print('According to Sagicor speadsheet, client over 65')
else:
PreTax = val*0.06
val = (PreTax+val)
print('Premium Tax = +$' + str("%0.2f" % PreTax)) #good
print('$' + "%0.2f" % val)
if client[25] != None:
print('RS = +$' + str(client[25]))
val = (val+client[25])
print('Total = $' + str("%0.2f" % val))
#if val == client[26]:
#print('Sagicor\'s total agrees with calculated total - top')
#else:
#print('Premium does not agree - top')
else:
print('Roadside assistance NOT included')
print('Total = $' + str("%0.2f" % val))
#if val == client[26]:
#print('Sagicor\'s total agrees with calculated total - bottom')
#else:
#print('Premium does not agree - bottom')
client = []
output = []
print('--- END OF ROW ---')

If you want to write the output of your script in a file I see two options.
The first one is to handle the writing in the python script :
#!/usr/bin/python
#I am here assuming that you use a Unix-like system (i.e. Not Windows )
file = open('textfile.txt','w')
... some code ...
file.write(myData +'\n')
#Don't forget to close the file at the end of the script
file.close()
N.B: the 'w' option creates the file if it not exists, but it aslo delete previous version of the file. If you want to add lines at the end of an existing file, you should use 'a' instead.
The '\n' is just here to create a new line.
The other option is the following : Simple use print statament in your script
#My Script
... some code ...
print MyData
and then, run your script in a shell not in IDLE with the following command :
python SagicorPremiumCalculator2.py > textfile.txt
If you are running Windows, you should definitely use the first option (because I am not sure the bash-style commands works in cmd ).
I hope this helps !

Make a diff of 2 text file quickly using python

I have 2 big text files (right now 17MB but could be GB), as such I don't want to load them in the ram because their size could exceed my ram capacity.
The code I wrote for now is this :
def stopIfFileExist(filename):
if os.path.isfile(filename):
raise Exception("%s already exist" %filename)
def compareDump(before_filename, after_filename, diff_filename):
"""
Compare 2 dumps generated via makeDump(output_filename) and generate
a file containing the differences
-before_filename : (string) filename of the first dump
-after_filename : (string) filename of the second dump
-diff_filename : (string) filename of the diff
"""
stopIfFileExist(diff_filename)
num_lines = sum(1 for line in open(after_filename))
one_percent = num_lines/float(100)
diff = []
start = time.time()
with open(after_filename, "r") as afterFile:
counter = 0
for a_line in afterFile:
print "completion : %.9f percents" %(counter/float(one_percent))
counter = counter + 1
diff.append(a_line)
with open(before_filename, "r") as beforeFile:
for b_line in beforeFile:
if a_line.rstrip() == b_line.rstrip():
diff.pop()
break
end = time.time()
print "task completed in %s seconds" %(end - start)
with open(diff_filename, "a") as diffFile:
for line in diff:
diffFile.write(line)
what I'd like to do is remove from the beforeFile a line that was sucessfully compared (eg, when the if a_line.rstrip() == b_line.rstrip(): is triggered)
However since I am currently reading the file I don't see how to do it.
Any ideas?
Thanks.

I was able to diff two 20 megabyte files in a little over 3 minutes using the following test code.
Every 10,000 lines I put a random number, which you can see diff'd in the results.
import random
import difflib
import os
import time
start = time.time()
NUM_LINES = int(10000000 / 4)
t1 = 'test1'
t2 = 'test2'
if os.path.exists(t1):
os.remove(t1)
if os.path.exists(t2):
os.remove(t2)
with open(t1, 'w+') as f1:
for number in range(1, NUM_LINES):
if number % 10000 == 0:
r = random.randint(1, number)
else:
r = 1
f1.write(str(number * r) + '\n')
else:
f1.seek(0)
with open(t2, 'w+') as f2:
for number in range(1, NUM_LINES):
if number % 10000 == 0:
r = random.randint(1, number)
else:
r = 1
f2.write(str(number * r) + '\n')
else:
f2.seek(0)
t1 = f1.readlines()
t2 = f2.readlines()
for l in difflib.unified_diff(t1, t2, lineterm=''):
print(l.strip())
print('Execution took: {:.2f} seconds'.format(time.time() - start))
I pasted the output on github, as it is obscenely long.

Output 2 dim array 'list of lists" to text file in python

Simple question - I am creating a two dim array (ddist = [[0]*d for _ in [0]*d]) using lists in the code below. It outputs distance using gis data. I just want a simple way to take the result of my array/list and output to a text file keeping the same N*N structure. I have used output from print statements in the past but not a good solution in this case.
I am new to python by way of SAS.
def match_bg():
#as the name suggests this function will match the variations of blockgroups with grid travel time. Then output into two arras time and distance.
count = -1
countwo = -1
ctime = -1
ddist = [[0]*d for _ in [0]*d] #cratesan N*N array list
dtime = -1
while count < 10:
count = count +1
#j[count][7] = float(j[count][7])
#j[count][6] = float(j[count][6])
while countwo < d:
countwo = countwo+1
if count < 1:
#change values in bg file
j[countwo][7] = float(j[countwo][7])
j[countwo][6] = float(j[countwo][6])
#print j[count], j[countwo]
while ctime < RowsT:
#print ctime, lenth, t[ctime][0], count, countwo
ctime = ctime + 1
#takes both verations of big zone which should be end of the file and matches to travetime file - note 0 and 1 for t[] should be same for different files
if ((j[count][lenth-1] == t[ctime][0]) and (j[countwo][lenth-1] == t[ctime][1])) or ((j[countwo][lenth-1] == t[ctime][0]) and (j[count][lenth-1] == t[ctime][1])):
if t[ctime][0] != t[ctime][1]:
#jkdljf
x1=3963*j[count][7]*(math.pi/180)
x2=3963*j[countwo][7]*(math.pi/180)
y1=math.cos(j[count][6]*math.pi/180)*3963*j[count][7]*(math.pi/180)
y2=math.cos(j[countwo][6]*math.pi/180)*3963*j[countwo][7]*(math.pi/180)
dist=math.sqrt(pow(( x1-x2), 2) + pow((y1-y2), 2))
dtime = dist/t[ctime][11]
print countwo, count
ddist[count-1][countwo-1] = dist/t[ctime][lenth]
print dtime, "ajusted time", "not same grid"
print
elif j[count][5] != j[countwo][5]:
#ljdkjfs
x1=3963*j[count][7]*(math.pi/180)
x2=3963*j[countwo][7]*(math.pi/180)
y1=math.cos(j[count][6]*math.pi/180)*3963*j[count][7]*(math.pi/180)
y2=math.cos(j[countwo][6]*math.pi/180)*3963*j[countwo][7]*(math.pi/180)
dist=math.sqrt(pow(( x1-x2), 2) + pow((y1-y2), 2)) # could change to calculation
dtime = (dist/.65)/(t[ctime][10]/60.0)
print dtime, dist, "not in the same bg", j[count], j[countwo], t[ctime]
elif j[count][5] == j[countwo][5]:
if t[count][7] < 3000000:
dtime = 3
elif t[count][7] < 20000000:
dtime = 8
else:
dtime = 12
print dtime, "same bg"
print t[ctime][0], t[ctime], 1, j[count], j[countwo]
else:
print "error is skip logic", j[count], j[countwo], t[ctime]
break
#elif (j[countwo][lenth-1] == t[ctime][0]) and (j[count][lenth-1] == t[ctime][1]):
#print t[ctime][0], t[ctime], 2, j[count], j[countwo]
#break
ctime = -1
countwo = -1

that's what you could to output your 2-d list (or any 2d list for that matter):
with open(outfile, 'w') as file:
file.writelines('\t'.join(str(j) for j in i) + '\n' for i in top_list)