So I have a rather general question I was hoping to get some help with. I put together a Python program that runs through and automates workflows at the state level for all the different counties. The entire program was created for research at school - not actual state work. Anyways, I have two designs shown below. The first is an updated version. It takes about 40 minutes to run. The second design shows the original work. Note that it is not a well structured design. However, it takes about five minutes to run the entire program. Could anybody give any insight why there are such differences between the two? The updated version is still ideal as it is much more reusable (can run and grab any dataset in the url) and easy to understand. Furthermore, 40 minutes to get about a hundred workflows completed is still a plus. Also, this is still a work in progress. A couple minor issues still need to be addressed in the code but it is still a pretty cool program.
Updated Design
import os, sys, urllib2, urllib, zipfile, arcpy
from arcpy import env
path = os.getcwd()
def pickData():
myCount = 1
path1 = 'path2URL'
response = urllib2.urlopen(path1)
print "Enter the name of the files you need"
numZips = raw_input()
numZips2 = numZips.split(",")
myResponse(myCount, path1, response, numZips2)
def myResponse(myCount, path1, response, numZips2):
myPath = os.getcwd()
for each in response:
eachNew = each.split(" ")
eachCounty = eachNew[9].strip("\n").strip("\r")
try:
myCountyDir = os.mkdir(os.path.expanduser(myPath+ "\\counties" + "\\" + eachCounty))
except:
pass
myRetrieveDir = myPath+"\\counties" + "\\" + eachCounty
os.chdir(myRetrieveDir)
myCount+=1
response1 = urllib2.urlopen(path1 + eachNew[9])
for all1 in response1:
allNew = all1.split(",")
allFinal = allNew[0].split(" ")
allFinal1 = allFinal[len(allFinal)-1].strip(" ").strip("\n").strip("\r")
numZipsIter = 0
path8 = path1 + eachNew[9][0:len(eachNew[9])-2] +"/"+ allFinal1
downZip = eachNew[9][0:len(eachNew[9])-2]+".zip"
while(numZipsIter <len(numZips2)):
if (numZips2[numZipsIter][0:3].strip(" ") == "NWI") and ("remap" not in allFinal1):
numZips2New = numZips2[numZipsIter].split("_")
if (numZips2New[0].strip(" ") in allFinal1 and numZips2New[1] != "remap" and numZips2New[2].strip(" ") in allFinal1) and (allFinal1[-3:]=="ZIP" or allFinal1[-3:]=="zip"):
urllib.urlretrieve (path8, allFinal1)
zip1 = zipfile.ZipFile(myRetrieveDir +"\\" + allFinal1)
zip1.extractall(myRetrieveDir)
#maybe just have numzips2 (raw input) as the values before the county number
#numZips2[numZipsIter][0:-7].strip(" ") in allFinal1 or numZips2[numZipsIter][0:-7].strip(" ").lower() in allFinal1) and (allFinal1[-3:]=="ZIP" or allFinal1[-3:]=="zip"
elif (numZips2[numZipsIter].strip(" ") in allFinal1 or numZips2[numZipsIter].strip(" ").lower() in allFinal1) and (allFinal1[-3:]=="ZIP" or allFinal1[-3:]=="zip"):
urllib.urlretrieve (path8, allFinal1)
zip1 = zipfile.ZipFile(myRetrieveDir +"\\" + allFinal1)
zip1.extractall(myRetrieveDir)
numZipsIter+=1
pickData()
#client picks shapefiles to add to map
#section for geoprocessing operations
# get the data frames
#add new data frame, title
#check spaces in ftp crawler
os.chdir(path)
env.workspace = path+ "\\symbology\\"
zp1 = os.listdir(path + "\\counties\\")
def myGeoprocessing(layer1, layer2):
#the code in this function is used for geoprocessing operations
#it returns whatever output is generated from the tools used in the map
try:
arcpy.Clip_analysis(path + "\\symbology\\Stream_order.shp", layer1, path + "\\counties\\" + layer2 + "\\Streams.shp")
except:
pass
streams = arcpy.mapping.Layer(path + "\\counties\\" + layer2 + "\\Streams.shp")
arcpy.ApplySymbologyFromLayer_management(streams, path+ '\\symbology\\streams.lyr')
return streams
def makeMap():
#original wetlands layers need to be entered as NWI_line or NWI_poly
print "Enter the layer or layers you wish to include in the map"
myInput = raw_input();
counter1 = 1
for each in zp1:
print each
print path
zp2 = os.listdir(path + "\\counties\\" + each)
for eachNew in zp2:
#print eachNew
if (eachNew[-4:] == ".shp") and ((myInput in eachNew[0:-7] or myInput.lower() in eachNew[0:-7])or((eachNew[8:12] == "poly" or eachNew[8:12]=='line') and eachNew[8:12] in myInput)):
print eachNew[0:-7]
theMap = arcpy.mapping.MapDocument(path +'\\map.mxd')
df1 = arcpy.mapping.ListDataFrames(theMap,"*")[0]
#this is where we add our layers
layer1 = arcpy.mapping.Layer(path + "\\counties\\" + each + "\\" + eachNew)
if(eachNew[7:11] == "poly" or eachNew[7:11] =="line"):
arcpy.ApplySymbologyFromLayer_management(layer1, path + '\\symbology\\' +myInput+'.lyr')
else:
arcpy.ApplySymbologyFromLayer_management(layer1, path + '\\symbology\\' +eachNew[0:-7]+'.lyr')
# Assign legend variable for map
legend = arcpy.mapping.ListLayoutElements(theMap, "LEGEND_ELEMENT", "Legend")[0]
# add wetland layer to map
legend.autoAdd = True
try:
arcpy.mapping.AddLayer(df1, layer1,"AUTO_ARRANGE")
#geoprocessing steps
streams = myGeoprocessing(layer1, each)
# more geoprocessing options, add the layers to map and assign if they should appear in legend
legend.autoAdd = True
arcpy.mapping.AddLayer(df1, streams,"TOP")
df1.extent = layer1.getExtent(True)
arcpy.mapping.ExportToJPEG(theMap, path + "\\counties\\" + each + "\\map.jpg")
# Save map document to path
theMap.saveACopy(path + "\\counties\\" + each + "\\map.mxd")
del theMap
print "done with map " + str(counter1)
except:
print "issue with map or already exists"
counter1+=1
makeMap()
Original Design
import os, sys, urllib2, urllib, zipfile, arcpy
from arcpy import env
response = urllib2.urlopen('path2URL')
path1 = 'path2URL'
myCount = 1
for each in response:
eachNew = each.split(" ")
myCount+=1
response1 = urllib2.urlopen(path1 + eachNew[9])
for all1 in response1:
#print all1
allNew = all1.split(",")
allFinal = allNew[0].split(" ")
allFinal1 = allFinal[len(allFinal)-1].strip(" ")
if allFinal1[-10:-2] == "poly.ZIP":
response2 = urllib2.urlopen('path2URL')
zipcontent= response2.readlines()
path8 = 'path2URL'+ eachNew[9][0:len(eachNew[9])-2] +"/"+ allFinal1[0:len(allFinal1)-2]
downZip = str(eachNew[9][0:len(eachNew[9])-2])+ ".zip"
urllib.urlretrieve (path8, downZip)
# Set the path to the directory where your zipped folders reside
zipfilepath = 'F:\Misc\presentation'
# Set the path to where you want the extracted data to reside
extractiondir = 'F:\Misc\presentation\counties'
# List all data in the main directory
zp1 = os.listdir(zipfilepath)
# Creates a loop which gives use each zipped folder automatically
# Concatinates zipped folder to original directory in variable done
for each in zp1:
print each[-4:]
if each[-4:] == ".zip":
done = zipfilepath + "\\" + each
zip1 = zipfile.ZipFile(done)
extractiondir1 = extractiondir + "\\" + each[:-4]
zip1.extractall(extractiondir1)
path = os.getcwd()
counter1 = 1
# get the data frames
# Create new layer for all files to be added to map document
env.workspace = "E:\\Misc\\presentation\\symbology\\"
zp1 = os.listdir(path + "\\counties\\")
for each in zp1:
zp2 = os.listdir(path + "\\counties\\" + each)
for eachNew in zp2:
if eachNew[-4:] == ".shp":
wetlandMap = arcpy.mapping.MapDocument('E:\\Misc\\presentation\\wetland.mxd')
df1 = arcpy.mapping.ListDataFrames(wetlandMap,"*")[0]
#print eachNew[-4:]
wetland = arcpy.mapping.Layer(path + "\\counties\\" + each + "\\" + eachNew)
#arcpy.Clip_analysis(path + "\\symbology\\Stream_order.shp", wetland, path + "\\counties\\" + each + "\\Streams.shp")
streams = arcpy.mapping.Layer(path + "\\symbology\\Stream_order.shp")
arcpy.ApplySymbologyFromLayer_management(wetland, path + '\\symbology\\wetland.lyr')
arcpy.ApplySymbologyFromLayer_management(streams, path+ '\\symbology\\streams.lyr')
# Assign legend variable for map
legend = arcpy.mapping.ListLayoutElements(wetlandMap, "LEGEND_ELEMENT", "Legend")[0]
# add the layers to map and assign if they should appear in legend
legend.autoAdd = True
arcpy.mapping.AddLayer(df1, streams,"TOP")
legend.autoAdd = True
arcpy.mapping.AddLayer(df1, wetland,"AUTO_ARRANGE")
df1.extent = wetland.getExtent(True)
# Export the map to a pdf
arcpy.mapping.ExportToJPEG(wetlandMap, path + "\\counties\\" + each + "\\wetland.jpg")
# Save map document to path
wetlandMap.saveACopy(path + "\\counties\\" + each + "\\wetland.mxd")
del wetlandMap
print "done with map " + str(counter1)
counter1+=1
Have a look at this guide:
https://wiki.python.org/moin/PythonSpeed/PerformanceTips
Let me quote:
Function call overhead in Python is relatively high, especially compared with the execution speed of a builtin function. This strongly suggests that where appropriate, functions should handle data aggregates.
So effectively this suggests, to not factor out something as a function that is going to be called hundreds of thousands of times.
In Python functions won't be inlined, and calling them is not cheap. If in doubt use a profiler to find out how many times is each function called, and how long does it take on average. Then optimize.
You might also give PyPy a shot, as they have certain optimizations built in. Reducing the function call overhead in some cases seems to be one of them:
Python equivalence to inline functions or macros
http://pypy.org/performance.html
Related
Hello I am having a difficult time running a python script on startup that request user input in terminal to name a directory. The python script ask for a file name then creates a directory and csv file to store pictures and information of each picture such as GPS data in the csv code. My code works fine when I run it in Geany but I have tried every imaginable way to start the code on bootup. The code will work if I change direcname = str(input("name your file: ")) to direcname=str("file_name"). I have spent days trying to figure this out, I can not find a single way to open terminal on boot that works for my script.
#import packages
from gpiozero import Button, LED
from picamera import PiCamera
import os
import datetime
from gps import *
#define gpio pins and variables
pwd = os.getcwd()
camera = PiCamera()
led = LED(13)
previewbtn = Button(26, hold_time=2)
counter = 1
#GPS stuff
gpsd = gps(mode=WATCH_ENABLE|WATCH_NEWSTYLE)
#make new directory and create text file within
direcname = str(input("name your file: "))
newpath = pwd + '/' + direcname
os.makedirs(newpath)
txtfile = open(newpath + '/' + direcname + '.csv', 'w+')
txtfile.write('img, date/time, lat, lon, alt(m)')
txtfile.close()
#define functions
def capture():
global counter
camera.capture(newpath + '/' + direcname + str(counter) + '.jpg')
txtfile = open(newpath + '/' + direcname + '.csv', 'a')
txtfile.write("\n")
txtfile.write( direcname + str(counter) + ',' + str(datetime.datetime.now()) +
',' + lat1 + ',' + lon1 + ','+ alt1)
txtfile.close()
counter += 1
#run function
try:
while True:
#Setting lat,lon, and alt as variables
report = gpsd.next()
if report['class'] == 'TPV':
if getattr(report,'lat',0.0)!=0:
lat1 = str(getattr(report,'lat',0.0))
if getattr(report,'lon',0.0)!=0:
lon1 = str(getattr(report,'lon',0.0))
if getattr(report,'alt','nan')!= 'nan':
alt1 = str(getattr(report,'alt','nan'))
else:
lat1 = "ERROR"
lon1 = "ERROR"
alt1 = "ERROR"
#Everything else
led.source = previewbtn
previewbtn.when_pressed = camera.start_preview
previewbtn.when_held = capture
previewbtn.when_released = camera.stop_preview
except(KeyboardInterrupt, SystemExit):
print("Done.\nExiting")
If you are using window manager then you can add your python filename.py & in .xinitrc which starts executing anything which is written in that script.
If using any desktop environment then it comes with a default display manager which does the same things. Each display manager has their own way of auto-starting things.
And i recommend starting it while login rather than starting on boot.
The title might not be relevant for my question becuase I don't actually want a wireless file transfering script, I need a file manager type.
I want something with which I can connect my phone with my pc (eg: hotspot and wifi) and then I would like to show text file browser (I have the code for that) by sending lists of all files and folders using os.listdir(), whenever the selected option is a file (os.path.isdir() == False), I would like to transfer the file and run it(like: picture, video, etc).
The file Browser code which I wrote runs on windows and also Android (after making a few changes) using qpython.
My code is
import os
def FileBrowser(cwd = os.getcwd()):
while True:
if cwd[-1:] != "\\":
cwd = cwd + "\\"
files = os.listdir(cwd)
count = 1
tmpp = ""
print("\n\n" + "_"*50 +"\n\n")
print(cwd + "\n")
for f in files:
if os.path.isdir(cwd + f) == True:
s1 = str(count) + ". " + f
tmps1 = 40 - (len(s1)+5)
t2 = int(tmps1/3)
s1 = s1 + " " * t2 + "-" * (tmps1 - t2)
print(s1 + "<dir>")
else:
print(str(count) + ". " + f + tmpp)
count = count + 1
s = raw_input("Enter the file/Directory: ")
if s == "...":
tmp1 = cwd.count("\\")
tmp2 = cwd.rfind("\\")
if tmp1 > 1:
cwd = cwd[0:tmp2]
tmp2 = cwd.rfind("\\")
cwd = cwd[0:tmp2+1]
continue
else:
continue
else:
s = int(s) - 1
if os.path.isdir(cwd + files[s]) == True:
cwd = cwd + files[s] + "\\"
continue
else:
f1 = files[s]
break
return f1
def main():
fb = FileBrowser()
main()
A very naive approach using Python is to go to the root of the directory you want to be served and use:
python -m SimpleHTTPServer
The connect to it on port 8000.
you may need to socket programming. creating a link (connection) between your PC and you smart phone and then try to transfer files
Task 1: I've completed.
Write a python script to generate three sets of 50 files (150 files total) with random dates in the file name from within the last 50 days:
IT421_unixdate.txt, IT151_unixdate.txt, IT620_unixdate.txt
Task 2:
Write a python script to:
Check if the folders IT421, IT151, IT620 exist, if they do not, create them. Move the files that begin with ITx to the folder ITx
If the file already exists in the folder, rename it like the following example:
IT421_1456991403.txt becomes IT421_1456991403(1).txt, IT421_1456991403(1).txt becomes IT421_1456991403(2).txt, and so on….
I am stuck on task2 part 2
#!/usr/local/bin/python3
import random
import time
import os
folders = ["IT421", "IT151", "IT620"]
#task2(1)
for i in range(0,len(folders)):
if not os.path.exists(folders[i]):
os.mkdir(folders[i])
#task1
for i in range (0,50):
for classIT in folders:
pastFiftyDays = int(time.time()) - 432000
currentTime = int(time.time())
timeStamp = str(random.randrange(pastFiftyDays, currentTime))
#Task2(2)
if classIT == "IT151":
open('IT151/' + classIT + "_" + timeStamp + .txt".format(classIT), "w")
elif classIT == "IT421":
open('IT421/' + classIT + "_" + timeStamp + ".txt".format(classIT), "w")
else:
open('IT620/' + classIT + "_" + timeStamp + ".txt".format(classIT), "w")
This is somewhere along the lines of what I've been trying.
#!/usr/local/bin/python3
import os
import fnmatch
folders = ["file_a", "file_z", "file_c"]
for i in range(0,len(folders)):
x = folders[i]
for filename in os.listdir('./temp'):
if fnmatch.fnmatch(filename, folders[i] + '.txt'):
open('temp/' + x + "(1).txt".format(x), "w")
else:
open('temp/' + x + ".txt".format(x), "w")
I am new to python and have been working and searching for days for ways to solve this problem. Any help would be appreciated. Thank you.
I have several shapefiles in a folder, each suffixed by _LINES or _AREAS.
I would like to get rid of these suffixes
California_Aqueduct_LINES.shp --> California_Aqueduct.shp
California_Aqueduct_LINES.dbf --> California_Aqueduct.dbf
California_Aqueduct_LINES.prj --> California_Aqueduct.prj
California_Aqueduct_LINES.shx--> California_Aqueduct.shx
Subdivision_AREAS.dbf --> Subdivision.dbf
Subdivision_AREAS.prj --> Subdivision.prj
Subdivision_AREAS.SHP --> Subdivision.SHP
Subdivision_AREAS.shx --> Subdivision.shx
Can i do this way:
#!/usr/bin/python
ls = ['California_Aqueduct_LINES.shp',
'Subdivision_AREAS.dbf',
'Subdivision_AREAS.SHP']
truncate = set(['LINES', 'AREAS'])
for p in [x.split('_') for x in ls]:
pre, suf = p[-1].split('.')
if pre in truncate:
print '_'.join(p[:-1]) + '.' + suf
else:
print '_'.join(p[:-1]) + p[-1]
Output:
California_Aqueduct.shp
Subdivision.dbf
Subdivision.SHP
Here is something I've put together in ArcPY
import os, sys, arcpy
InFolder = sys.argv[1] # make this a hard set path if you like
arcpy.env.workspace = InFolder
CapitalKeywordsToRemove = ["_AREAS","_LINES"]# MUST BE CAPITALS
DS_List = arcpy.ListFeatureClasses("*.shp","ALL") # Get a list of the feature classes (shapefiles)
for ThisDS in DS_List:
NewName = ThisDS # set the new name to the old name
# replace key words, searching is case sensitive but I'm trying not to change the case
# of the name so the new name is put together from the original name with searching in
# upper case, use as many key words as you like
for CapKeyWord in CapitalKeywordsToRemove:
if CapKeyWord in NewName.upper():
# remove the instance of CapKeyWord without changing the case
NewName = NewName[0:NewName.upper().find(CapKeyWord)] + NewName[NewName.upper().find(CapKeyWord) + len(CapKeyWord):]
if NewName != ThisDS:
if not arcpy.Exists(NewName):
arcpy.AddMessage("Renaming " + ThisDS + " to " + NewName)
arcpy.Rename_management(ThisDS , NewName)
else:
arcpy.AddWarning("Cannot rename, " + NewName + " already exists")
else:
arcpy.AddMessage("Retaining " + ThisDS)
If you don't have arcpy let me know and I'll change it to just straight python... there's a bit more to it but it's not difficult.
I am very new in python, but I have been able to make few useful python codes (at least useful for my work). I would like to combine two of my codes, but I have a hard time making it work, I think I am completely lost in how the code should looks like.
The first code basically takes a file, read it, extract to columns from it, and then write the columns in a new file. I repeat this with several files:
import sys
import re
filetowrite = sys.argv[1]
filetoread = sys.argv[2]
newfile = str(filetowrite) + ".txt"
openold = open(filetoread,"r")
opennew = open(newfile,"w")
rline = openold.readlines()
number = int(len(rline))
start = 0
for i in range (len(rline)) :
if "2theta" in rline[i] :
start = i
opennew.write ("q" + "\t" + "I" + "\n")
opennew.write ("1/A" + "\t" + "1/cm" + "\n")
opennew.write (str(filetowrite) + "\t" + str(filetowrite) + "\n")
for line in rline[start + 1 : number] :
words = line.split()
word1 = (words[1])
word2 = (words[2])
opennew.write (word1 + "\t" + word2 + "\n")
openold.close()
opennew.close()
The second code takes the new previously created files and combine them in a way in which the columns are next to each other in the final file.
import sys
from itertools import izip
filenames = sys.argv[2:]
filetowrite = sys.argv[1]
newfile = str(filetowrite) + ".txt"
opennew = open(newfile, "w")
files = map(open, filenames)
for lines in izip(*files):
opennew.write(('\t'.join(i.strip() for i in lines))+"\n")
Any help in how to proceed to make a single code out of these two codes is highly appreciated.
All the best
Make each file into a function in one larger file, then call the functions as necessary. Make use of __main__ to do that.
import sys
import re
from itertools import izip
def func_for_file1():
# All of the code from File 1 goes here.
def func_for_file2():
# ALl of the code from File 2 goes here.
def main():
# Decide what order you want to call these methods.
func_for_file1()
func_for_file2()
if __name__ == '__main__':
main()