We Keep Coding

wait until thread pool process is finished before submitting the next process

I am trying to pause a thread until the initial thread finishes.
My thread_pool starts like:
thread_pool_executor = futures.ThreadPoolExecutor(max_workers=5)
def onStart(self,event):
self._quit = False
self.btn_output.Disable()
self.btn_watch.Disable()
self.btn_start.Disable()
thread_pool_executor.submit(self.monitor_folder)
thread_pool_executor.submit(self.active_listening)
def monitor_folder(self):
#wx.CallAfter(self.print1)
while self._quit == False:
#path_to_watch = os.path.abspath (".")
path_to_watch = self.lbl_watch.GetLabel()
# FindFirstChangeNotification sets up a handle for watching
# file changes. The first parameter is the path to be
# watched; the second is a boolean indicating whether the
# directories underneath the one specified are to be watched;
# the third is a list of flags as to what kind of changes to
# watch for. We're just looking at file additions / deletions.
#
change_handle = win32file.FindFirstChangeNotification (
path_to_watch,
0,
win32con.FILE_NOTIFY_CHANGE_FILE_NAME
)
#
# Loop forever, listing any file changes. The WaitFor... will
# time out every half a second allowing for keyboard interrupts
# to terminate the loop.
try:
old_path_contents = dict ([(f, None) for f in os.listdir (path_to_watch)])
while 1:
result = win32event.WaitForSingleObject (change_handle, 500)
# If the WaitFor... returned because of a notification (as
# opposed to timing out or some error) then look for the
# changes in the directory contents.
if result == win32con.WAIT_OBJECT_0:
new_path_contents = dict ([(f, None) for f in os.listdir (path_to_watch)])
added = [f for f in new_path_contents if not f in old_path_contents]
deleted = [f for f in old_path_contents if not f in new_path_contents]
if added:
print ("Added: ", ", ".join (added))
#Get file type
a_string = ", ".join (added)
length = len(a_string)
fType = a_string[length - 4:]
if fType == ".csv" or fType == ".txt":
data_file = a_string
thread_pool_executor.submit(self.process_csv,a_string) # Pause thread, run process_csv, and resume
if deleted: print ("Deleted: ", ", ".join (deleted))
old_path_contents = new_path_contents
win32file.FindNextChangeNotification (change_handle)
finally:
win32file.FindCloseChangeNotification (change_handle)
I get an error on thread_pool_executor.submit(self.process_csv,a_string) if two or more files are added to the directory. How can I wait until thread_pool_executor.submit(self.process_csv,a_string) is finished running before submitting the next process?
I want these 2 to remain running though.
thread_pool_executor.submit(self.monitor_folder)
thread_pool_executor.submit(self.active_listening)
EDIT:
When two or more files are detected
Message=(-2146777998, 'OLE error 0x800ac472', None, None)
Source=C:\Users\Apache Paint\AppData\Local\Temp\gen_py\3.8\00020813-0000-0000-C000-000000000046x0x1x9.py
StackTrace:
File "C:\Users\Apache Paint\AppData\Local\Temp\gen_py\3.8\00020813-0000-0000-C000-000000000046x0x1x9.py", line 41189, in Open
ret = self.oleobj.InvokeTypes(1923, LCID, 1, (13, 0), ((8, 1), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17), (12, 17)),Filename
File "C:\Users\Apache Paint\source\repos\Monitor_Folder\Monitor_Folder.py", line 247, in excel_to_pdf
wb = excel.Workbooks.Open(filepath)
File "C:\Users\Apache Paint\source\repos\Monitor_Folder\Monitor_Folder.py", line 233, in process_csv (Current frame)
self.excel_to_pdf(df, a_string)
Full Code:
import os
import win32file
import win32event
import win32con
import pythoncom
from win32com import client
import ctypes
import pandas as pd
import csv
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from openpyxl import load_workbook
import wx
import glob
from os.path import splitext
from concurrent import futures
import sys
import datetime
import time
import re
import configparser
config = configparser.ConfigParser()
thread_pool_executor = futures.ThreadPoolExecutor(max_workers=5)
directory = os.getcwd()
v_dict = {'ProjectNumber' : ['1|F6'],'Yield' : ['1|F5'],'Coil Lot Number' : ['1|D6'],'Coil Thickness' : ['1|D5'],'Web Profile' : ['1|H7'],'Operation Number' : ['1|F7'],'Label Readable?' :
['1|D26','2|E26','3|F26','4|G26','5|H26'],'ICCES Number?' : ['1|D27','2|E27','3|F27','4|G27','5|H27'],'Date of Test' : ['1|D7'],'Start Time of Test' :
['1|H5','1|D10','1|E10','1|F10','1|G10'],'End Time of Test' : ['1|H6','2|H10'],'Part Length' : ['1|D12','2|E12','3|F12','4|G12','5|H12'],'Web Width' :
['1|D13','2|E13','3|F13','4|G13','5|H13'],'Flare Far' : ['1|D14','2|E14','3|F14','4|G14','5|H14'],'Flare Near' : ['1|D15','2|E15','3|F15','4|G15','5|H15'],'Hole Location Width' :
['1|D16','2|E16','3|F16','4|G16','5|H16'],'Hole Location Length' : ['1|D17','2|E17','3|F17','4|G17','5|H17'],'Crown' : ['1|D18','2|E18','3|F18','4|G18','5|H18'],'Camber' :
['1|D19','2|E19','3|F19','4|G19','5|H19'],'Bow' : ['1|D20','2|E20','3|F20','4|G20','5|H20'],'Twist' : ['1|D21','2|E21','3|F21','4|G21','5|H21'],'Flange Width Far' :
['1|D22','2|E22','3|F22','4|G22','5|H22'],'Flange Width Near' : ['1|D23','2|E23','3|F23','4|G23','5|H23'],'Lip Length Far' : ['1|D24','2|E24','3|F24','4|G24','5|H24'],'Lip Length Near' :
['1|D25','2|E25','3|F25','4|G25','5|H25']}
class MainFrame(wx.Frame):
def __init__(self, parent, title):
super(MainFrame, self).__init__(parent, title=title,size=(600,400))
global template
font = wx.Font(12, wx.MODERN, wx.ITALIC, wx.BOLD, True, u'Arial Narrow',)
font1 = wx.Font(8, wx.MODERN, wx.NORMAL, wx.NORMAL, False, u'Arial Narrow')
b_font = wx.Font(12, wx.MODERN, wx.ITALIC, wx.BOLD, False, u'Arial Narrow')
self.panel = wx.Panel(self)
self.panel.SetBackgroundColour("light gray")
#Create sizers
vbox = wx.BoxSizer(wx.VERTICAL)
hbox1 = wx.BoxSizer(wx.HORIZONTAL)
hbox2 = wx.BoxSizer(wx.HORIZONTAL)
#Create widgets
self.st1 = wx.StaticText(self.panel, label='Script is not running.',style = wx.ALIGN_CENTRE)
self.lbl_watch = wx.StaticText(self.panel, label= os.path.abspath ("."), style=wx.ALIGN_LEFT)
self.lbl_output = wx.StaticText(self.panel, label=os.path.abspath ("."))
self.tc = wx.TextCtrl(self.panel, style= wx.TE_MULTILINE | wx.SUNKEN_BORDER | wx.TE_READONLY )
self.btn_start = wx.Button(self.panel, label='Run Script', size=(100, 30))
self.btn_watch = wx.Button(self.panel, label='Select Folder to Watch')
self.btn_output = wx.Button(self.panel, label='Select Output Folder ')
self.btn_start.SetBackgroundColour(wx.Colour(198, 89, 17))
self.st1.SetForegroundColour((255,0,0)) # set text color
self.tc.SetFont(font1)
self.st1.SetFont(font)
self.btn_start.SetFont(b_font)
self.btn_start.Bind(wx.EVT_BUTTON, self.onStart)
self.btn_output.Bind(wx.EVT_BUTTON, self.choose_output)
self.btn_watch.Bind(wx.EVT_BUTTON, self.choose_watch)
hbox1.Add(self.btn_watch )
hbox1.Add(self.lbl_watch, 0 , wx.ALL | wx.EXPAND, 5)
hbox2.Add(self.btn_output)
hbox2.Add(self.lbl_output, 0 , wx.ALL | wx.EXPAND, 5)
vbox.Add(self.st1,-1 , wx.ALIGN_CENTRE | wx.ALL, 5)
vbox.Add(self.btn_start, 0, wx.ALIGN_CENTRE | wx.ALL, 5)
vbox.Add(self.tc,2, wx.EXPAND| wx.ALL, 10)
vbox.Add(hbox1,0, wx.EXPAND| wx.ALL, 10)
vbox.Add(hbox2,0, wx.EXPAND| wx.ALL, 10)
self.panel.SetSizer(vbox)
self.Centre()
self.Show()
template = self.resource_path('template.xlsx')
self.write_config()
def write_config(self):
#Write Config
global config
global config_default
if not os.path.exists(self.resource_path('config.ini')):
#config["DEFAULT"] = {
# "watch_folder": os.path.abspath ("."),
# "output_folder": os.path.abspath (".")}
config['DEFAULT'] = {'watch_folder': os.path.abspath ("."), 'output_folder': os.path.abspath (".")}
config.write(open(self.resource_path('config.ini'), 'w'))
#Check if sections are blank
config.read(self.resource_path("config.ini"))
config_default = config["DEFAULT"]
if config_default["output_folder"] == '':
config_default["output_folder"] = os.path.abspath (".")
with open(self.resource_path('config.ini'), 'w') as conf:
config.write(conf)
if config_default["watch_folder"] == '':
config_default["watch_folder"] = os.path.abspath (".")
with open(self.resource_path('config.ini'), 'w') as conf:
config.write(conf)
#Update labels/paths
self.lbl_watch.SetLabel(config_default["watch_folder"])
self.lbl_output.SetLabel(config_default["output_folder"])
def onStart(self,event):
self._quit = False
self.btn_output.Disable()
self.btn_watch.Disable()
self.btn_start.Disable()
thread_pool_executor.submit(self.monitor_folder)
thread_pool_executor.submit(self.active_listening)
def choose_output(self, event):
message = 'Select Output Folder'
f_path = self.set_dir(message)
if f_path == '':
dlg = wx.MessageBox('Do you want to revert path to the default directory?' + '\n' + '\n' + 'Selecting "Yes" will revert the path to the application directory.','Revert Output Filepath?',wx.YES_NO | wx.ICON_QUESTION)
if dlg == wx.YES:
#Update output folder
f_path = os.getcwd()
else:
#f_path = self.lbl_output.SetLabel(f_path)
return
#Update output folder
self.lbl_output.SetLabel(f_path)
config_default["output_folder"] = f_path
#Write changes back to file
with open(self.resource_path('config.ini'), 'w') as conf:
config.write(conf)
def choose_watch(self, event):
message = 'Select Watch Folder'
f_path = self.set_dir(message)
print(f_path)
if f_path == '':
dlg = wx.MessageBox('Do you want to revert path to the default directory?' + '\n' + '\n' + 'Selecting "Yes" will revert the path to the application directory.','Revert Watch Filepath?',wx.YES_NO | wx.ICON_QUESTION)
if dlg == wx.YES:
#Update watch folder
f_path = os.getcwd()
else:
#f_path = self.lbl_watch.SetLabel(f_path)
return
#Update watch folder
self.lbl_watch.SetLabel(f_path)
config_default["watch_folder"] = f_path
#Write changes back to file
with open(self.resource_path('config.ini'), 'w') as conf:
config.write(conf)
def set_dir(self, message):
dlg = wx.DirDialog(
self, message=message,
style=wx.DD_DEFAULT_STYLE)
# Show the dialog and retrieve the user response.
if dlg.ShowModal() == wx.ID_OK:
# load directory
path = dlg.GetPath()
else:
path = ''
# Destroy the dialog.
dlg.Destroy()
return path
def get_pdf_path(self):
folder = self.lbl_output.GetLabel()
if folder == '':
folder = os.getcwd()
self.lbl_output.SetLabel(folder)
config_default["output_folder"] = folder
fileName = 'Test Data Report ' + str(datetime.datetime.now().strftime("%Y_%m_%d__%H.%M.%S")) + '.pdf'# + "\n" %H:%M:%S"
#fileName = fileName.replace(':','.')
path =os.path.join(folder, fileName)
return path
def getDirectory(self, filename): # For Excel Template
# Construct path for file
current_work_dir = os.getcwd()
path = os.path.join(current_work_dir, filename)
return path
def getDirectoryCSV(self, filename): # For Watch Folder
# Construct path for file
current_work_dir = self.lbl_watch.GetLabel()
if current_work_dir == '':
current_work_dir = os.getcwd()
self.lbl_watch.SetLabel(current_work_dir)
config_default["watch_folder"] = current_work_dir
path = os.path.join(current_work_dir, filename)
return path
def process_csv(self,a_string):
data_file = self.getDirectoryCSV(a_string)
print(data_file)
df = pd.read_fwf(data_file, header=None)
df = df[0].str.split(',', expand=True)
df.set_index(0, inplace = True)
df.fillna("", inplace=True)
self.excel_to_pdf(df, a_string)
def excel_to_pdf(self, df,a_string):
# Open Microsoft Excel
pythoncom.CoInitialize()
excel = client.Dispatch("Excel.Application")
excel.Visible = False
excel.ScreenUpdating = False
excel.DisplayAlerts = False
excel.EnableEvents = False
# Read Excel File
filepath = self.getDirectory(template)
print (filepath)
wb = excel.Workbooks.Open(filepath)
work_sheets = wb.Worksheets('Form')
#Write to sheet
for key, items in v_dict.items():
row_id = key
for item in items:
cel = str(item.split('|')[1])
col_num = int(str(item.split('|')[0]))
work_sheets.Range(cel).Value = df.loc[row_id][col_num]
#Format
#print_area = 'C1:I64'
work_sheets.PageSetup.Zoom = False
work_sheets.PageSetup.FitToPagesTall = 1
work_sheets.PageSetup.TopMargin = 10
work_sheets.PageSetup.BottomMargin = 10
work_sheets.PageSetup.RightMargin = 10
work_sheets.PageSetup.LeftMargin = 10
#work_sheets.PageSetup.PrintArea = print_area
# Convert into PDF File
pdf_path = self.get_pdf_path()
work_sheets.ExportAsFixedFormat(0, pdf_path)
excel.ScreenUpdating = True
excel.DisplayAlerts = True
excel.EnableEvents = True
wb.Close(SaveChanges=False)
excel.Quit()
text = 'PDF CREATED: ' + pdf_path
self.tc.AppendText(text + "\n" + "\n")
def df_to_pdf(self,df):
#Convert to PDF
fig, ax =plt.subplots(figsize=(12,4))
ax.axis('tight')
ax.axis('off')
the_table = ax.table(cellText=df.values,colLabels=df.columns,loc='center')
pp = PdfPages("csv_data.pdf") # ADD DATE
pp.savefig(fig, bbox_inches='tight')
pp.close()
def active_listening(self):
font = wx.Font(12, wx.MODERN, wx.ITALIC, wx.BOLD, False, u'Arial Narrow',)
self.st1.SetForegroundColour((0,128,0))
self.st1.SetFont(font)
m = 'Listening'
self.st1.SetLabel(m)
i = 1
while self._quit == False:
time.sleep(1)
if i <= 3:
m = m + "."
self.st1.SetLabel(m)
i = i + 1
else:
i = 1
m = 'Listening'
def monitor_folder(self):
#wx.CallAfter(self.print1)
while self._quit == False:
#path_to_watch = os.path.abspath (".")
path_to_watch = self.lbl_watch.GetLabel()
# FindFirstChangeNotification sets up a handle for watching
# file changes. The first parameter is the path to be
# watched; the second is a boolean indicating whether the
# directories underneath the one specified are to be watched;
# the third is a list of flags as to what kind of changes to
# watch for. We're just looking at file additions / deletions.
#
change_handle = win32file.FindFirstChangeNotification (
path_to_watch,
0,
win32con.FILE_NOTIFY_CHANGE_FILE_NAME
)
#
# Loop forever, listing any file changes. The WaitFor... will
# time out every half a second allowing for keyboard interrupts
# to terminate the loop.
try:
old_path_contents = dict ([(f, None) for f in os.listdir (path_to_watch)])
while 1:
result = win32event.WaitForSingleObject (change_handle, 500)
# If the WaitFor... returned because of a notification (as
# opposed to timing out or some error) then look for the
# changes in the directory contents.
if result == win32con.WAIT_OBJECT_0:
new_path_contents = dict ([(f, None) for f in os.listdir (path_to_watch)])
added = [f for f in new_path_contents if not f in old_path_contents]
deleted = [f for f in old_path_contents if not f in new_path_contents]
if added:
print ("Added: ", ", ".join (added))
#Get file type
a_string = ", ".join (added)
length = len(a_string)
fType = a_string[length - 4:]
if fType == ".csv" or fType == ".txt":
data_file = a_string
thread_pool_executor.submit(self.process_csv,a_string) # Pause thread, run process_csv, and resume
if deleted: print ("Deleted: ", ", ".join (deleted))
old_path_contents = new_path_contents
win32file.FindNextChangeNotification (change_handle)
finally:
win32file.FindCloseChangeNotification (change_handle)
def resource_path(self, relative_path):
""" Get absolute path to resource, works for dev and for PyInstaller """
try:
# PyInstaller creates a temp folder and stores path in _MEIPASS
base_path = sys._MEIPASS
except Exception:
base_path = os.path.abspath(".")
return os.path.join(base_path, relative_path)
def main():
app = wx.App()
ex = MainFrame(None, title='File Monitor')
ex.Show()
app.MainLoop()
if __name__ == '__main__':
main()

This is a classic use case for threading.Semaphore
Inside each function that can't be run concurrently, Instantiate it globally (in the main thread) using Semaphore(1). Use acquire() and release() methods accordingly to serialize access to your "shared resource" (inside the main monitir_folder function or the process_csv functions, depends on where you're getting the error - in this case, CSV processing internal state
As the number of permits is 1, an RLock will also make a good fit, but A Semaphore better express what you are trying to achieve.
Example:
# Global declaration
semaphore = threading.Semaphore(1)
...
# Inside the monitor_folder function
semaphore.acquire()
try:
thread_pool_executor.submit(self.process_csv,a_string)
finally:
semaphore.release()

Writing into table in Word doc using python docx

I'm trying to write into tables in a word template my company uses for statement of works,
I already have working code that takes our template file and saves it as a new filename in a different location. The filepath+filename is new_SOWdst_file_name.
This keeps crashing usually for a failure error about this being not writeable.
Pretty new to python. Any advice appreciated.
# doc = Document(new_SOWdst_file_name)
# f = open(new_SOWdst_file_name, 'r')
# doc = Document(f)
# if customType == ca :
# sys.exit()
# elif customType == sc :
# SOWNum = doc.tables[0].cell(4,2).text
# SOWAuthor = doc.tables[0].cell(6,2).text
# SOWDescriptor = doc.tables[0].cell(8,2).text
# SOWPartNum = doc.tables[0].cell(9,2).text
# SOWDate = doc.tables[0].cell(13,1).text
# SOWNum = 'SOW-'+ PartNum
# SOWAuthor = 'Cody Nelson'
# SOWDescriptor = Description
# SOWPartNum = PartNum
# SOWDate = now.strftime("%b %d, %Y")
# doc.save(f)
# f.close()
# doc.close()
# sys.exit()
# elif customType == cc :
# sys.exit()

Don't open the file before reading or writing it. python-docx takes care of all those details for you. All you need is:
document = Document(new_SOWdst_file_name)
# ... do your work with `document`
document.save("new-file-name.docx")

doc = Document(new_SOWdst_file_name)
doc.tables[0].cell(13,1).text = now.strftime("%b %d, %Y")
doc.tables[0].cell(9,2).text = PartNum
doc.tables[0].cell(8,2).text = Description
doc.tables[0].cell(6,2).text = 'Cody Nelson'
doc.tables[0].cell(4,2).text = 'SOW-'+ PartNum
doc.save(new_SOWdst_file_name)
Not sure why this works and the other doesn't.
But, it works.

PYMEL Maya UI crazyness

I'm having trouble with a script i judged simple, yet here I am! The thing is I have no real problem with the ACTUAL script, but with the Maya UI.
It seems I can't avoid the creation of multiple windows, nor delete it using deleteUI("nemeOfUI"). thats driving me crazy. I looked up on line and it seems I'm doing it alright and I just can't finde the answer.
Here the whole code.
Attention to lines 15 16 17 18, 110, 208.
I know the code isnt' the cleanest but its versin 1.2. Once it is working I'll do the cleaning! Thank you!
import pymel.core as pm
from functools import partial
import os.path
try:
pm.deleteUI(changeTexWindow)
except:
print''
global sizeChoice
def drawUi():
if (pm.window("ChangeTextureFiles_v1.2", query = True, exists = True)):
pm.deleteUI("ChangeTextureFiles_v1.2")
changeTexWindow = pm.window("ChangeTextureFiles_v1.2",
sizeable = False,
width = 300,
height = 175,
minimizeButton = False,
maximizeButton = False
)
uiTabs = pm.tabLayout(width = 300,
height = 175,
parent = changeTexWindow
)
uiColumn1 = pm.columnLayout("Change Texture Files",
rowSpacing = 5,
parent = changeTexWindow,
width = 300
)
uiColumn2 = pm.columnLayout("Help",
rowSpacing = 5,
parent = changeTexWindow,
width = 300
)
uiRowExtra = pm.rowLayout(numberOfColumns = 1, parent = uiColumn1, h = 2)
uiRow0 = pm.rowLayout(numberOfColumns = 3, parent = uiColumn1, h = 15)
pm.separator(style = "none", height = 10, width = 2, horizontal = False)
pm.text("Change texture...")
uiRow1 = pm.rowLayout(numberOfColumns = 3, parent = uiColumn1, h = 15)
uiRow2 = pm.rowLayout(numberOfColumns = 3, parent = uiColumn1, h = 15)
uiRow3 = pm.rowLayout(numberOfColumns = 3, parent = uiColumn1, h = 15)
options = pm.radioCollection(parent = uiRow1)
opt1 = pm.radioButton(parent = uiRow1, label = "From all objects in the scene", data=1)
opt2 = pm.radioButton(parent = uiRow2, label = "From only selected objects", data=2)
opt3 = pm.radioButton(parent = uiRow3, label = "From all objects in scene but selected", data=3)
options = cmds.radioCollection( options, edit=True, select=opt1 )
uiRow4 = pm.rowLayout(numberOfColumns = 3, parent = uiColumn1)
uiRow5 = pm.rowLayout(numberOfColumns = 2, parent = uiColumn1)
pm.text('Type the size of texture you want:', parent = uiRow5, annotation = "Enter values as '1,2 3...'. Check Help tab. ", width = 215)
sizeChoice = pm.textField(parent = uiRow5, width = 60, annotation = "Enter values as '1,2 3...'. Check Help tab. ")
uiRow6 = pm.rowLayout(numberOfColumns = 3, parent = uiColumn1)
allB = pm.button(label = "Apply", width = 115, command = partial(passValue, options, sizeChoice, 0))
selB = pm.button(label = "Apply and close", width = 115, command = partial(passValue, options, sizeChoice, 1))
otherB = pm.button(label = "Cancel", width = 54, command = "closeUi()")
helpTxt0 = pm.text("",parent= uiColumn2, height = 2)
helpTxt1 = pm.text("Created by Mendel Reis at", parent= uiColumn2, width = 300, height = 12)
helpTxt2 = pm.text("FACULDADE MELIES", parent= uiColumn2, width = 300,height = 12)
helpTxt3 = pm.text("www.melies.com.br", parent= uiColumn2, width = 300, height = 12)
helpTxt01 = pm.text(" ",parent= uiColumn2, height = 5)
helpTxt4 = pm.text("HOW TO USE:", parent= uiColumn2, width = 300, height = 12)
helpTxt5 = pm.text("Will change files thru naming convention:", parent= uiColumn2, width = 300, height = 12)
helpTxt6 = pm.text("file_name.SIZE.extension", parent= uiColumn2, width = 300, height = 12)
helpTxt7 = pm.text("Input the SIZE as you wish.", parent= uiColumn2, width = 300, height = 12)
helpTxt8 = pm.text("Only use . (DOT) to isolate the SIZE tag.", parent= uiColumn2, width = 300, height = 12)
pm.showWindow(changeTexWindow)
def passValue(options, sizeChoice, closeAfter, *args):
radioCol = cmds.radioCollection(options, query=True, sl=True)
selOption = cmds.radioButton(radioCol, query=True, data=True)
newSize = pm.textField(sizeChoice, query = True, tx = True)
print selOption
print newSize
if (selOption == 1):
changeAll(newSize)
elif (selOption == 2):
changeSelected(newSize)
elif (selOption == 3):
changeAllBut(newSize)
if (closeAfter):
try:
del sizeChoice
except:
print''
pm.deleteUI("ChangeTextureFiles_v1.2")
def changeAll(newSize):
allTex = pm.ls(type = "file")
notFound = [] #array for texture files without a size change match
for tex in allTex: #get info from texture file to be changed
path = tex.getAttr("fileTextureName") #get the full path of texture file
name = path.split("/")[-1].split('.')[0] #get the name of the file
size = path.split("/")[-1].split('.')[-2] #get the user specified, thru naming convention, texture file size info
extension = path.split("/")[-1].split('.')[-1] #get the texture file extension
if (size != newSize): #check if texture file needs to be changed
old = name + "." + size + "." + extension #concatenate the old name
new = name + "." + newSize + "." + extension #concatenate the new name
newTex = path.replace(old, new, 1) #create string for new fileTextureName
if (os.path.isfile(newTex)): #check if requered file exists
tex.setAttr("fileTextureName", newTex) #change the textureFileName
else:
notFound.append(name+'.' + extension) #create a log with failed attempts
def changeSelected(newSize):
objs = pm.selected()
for item in objs:
a = pm.listRelatives(item)[0]
b = pm.listConnections(a, type = "shadingEngine")
sgInfo = pm.listConnections(b, type='materialInfo')
fileNode = pm.listConnections(sgInfo[0], type='file')
textureFile = pm.getAttr(fileNode[0].fileTextureName)
name = textureFile.split("/")[-1].split('.')[0] #get the name of the file
print "NAME", name
size = textureFile.split("/")[-1].split('.')[-2] #get the user specified, thru naming convention, texture file size info
print "SIZE", size
extension = textureFile.split("/")[-1].split('.')[-1] #get the texture file extension
print "EXTENSION", extension
if (size != newSize): #check if texture file needs to be changed
old = name + "." + size + "." + extension #concatenate the old name
new = name + "." + newSize + "." + extension #concatenate the new name
newTex = textureFile.replace(old, new, 1) #create string for new fileTextureName
if (os.path.isfile(newTex)): #check if requered file exists
fileNode[0].setAttr("fileTextureName", newTex) #change the textureFileName
else:
print "Did not find a texture to replace. Check naming convention: enter size as '1K', '2K'..."
def changeAllBut(newSize):
allObjs = pm.ls(type = "mesh")
selection = pm.selected()
selObjs = []
for item in selection:
a = pm.listRelatives(item)[0]
selObjs.append(a)
for item in allObjs:
if item in selObjs:
allObjs.remove(item)
for item in allObjs:
a = item
b = pm.listConnections(a, type = "shadingEngine")
sgInfo = pm.listConnections(b, type='materialInfo')
fileNode = pm.listConnections(sgInfo[0], type='file')
textureFile = pm.getAttr(fileNode[0].fileTextureName)
name = textureFile.split("/")[-1].split('.')[0] #get the name of the file
print "NAME", name
size = textureFile.split("/")[-1].split('.')[-2] #get the user specified, thru naming convention, texture file size info
print "SIZE", size
extension = textureFile.split("/")[-1].split('.')[-1] #get the texture file extension
print "EXTENSION", extension
if (size != newSize): #check if texture file needs to be changed
old = name + "." + size + "." + extension #concatenate the old name
new = name + "." + newSize + "." + extension #concatenate the new name
newTex = textureFile.replace(old, new, 1) #create string for new fileTextureName
if (os.path.isfile(newTex)): #check if requered file exists
fileNode[0].setAttr("fileTextureName", newTex) #change the textureFileName
else:
print "Did not find a texture to replace. Check naming convention: enter size as '1K', '2K'..."
def closeUi():
try:
del sizeChoice
except:
print''
pm.deleteUI("ChangeTextureFiles_v1.2")
drawUi()

This question is a bit dated, but maybe an answer is still helpful:
The problem lies in the window objects name:"ChangeTextureFiles_v1.2". Windows etc. are Maya objects and they to not allow spaces or points. You can try it if you simply create a polycube and try to rename it to "polycube_v1.2".
For your window you can use the title flag instead:
WINNAME="CTexWin"
changeTexWindow = pm.window(WINNAME,
title = "ChangeTextureFiles_v1.2",
sizeable = False,
width = 300,
height = 175,
minimizeButton = False,
maximizeButton = False
)
And if you check for existence just use the WINNAME:
if pm.window(WINNAME, exists = True):
pm.deleteUI(WINNAME)

How to read .evtx file using python?

Guys do anyone know how to read event log file in C:\Windows\System32\winevt\Logs with .evtx extension?
I have already tried to open it using notepad and read using python but notepad says access is denied...
Do anyone know how to do it? Thanks in advance..

This is how you would read the file "Forwarded Events" from the event viewer. You need admin access so I would run it as admin but I it will prompt you for a password if you don't.
import win32evtlog
import xml.etree.ElementTree as ET
import ctypes
import sys
def is_admin():
try:
return ctypes.windll.shell32.IsUserAnAdmin()
except:
return False
if is_admin():
# open event file
query_handle = win32evtlog.EvtQuery(
'C:\Windows\System32\winevt\Logs\ForwardedEvents.evtx',
win32evtlog.EvtQueryFilePath)
read_count = 0
a = 1
while a == 1:
a += 1
# read 1 record(s)
events = win32evtlog.EvtNext(query_handle, 1)
read_count += len(events)
# if there is no record break the loop
if len(events) == 0:
break
for event in events:
xml_content = win32evtlog.EvtRender(event, win32evtlog.EvtRenderEventXml)
# parse xml content
xml = ET.fromstring(xml_content)
# xml namespace, root element has a xmlns definition, so we have to use the namespace
ns = '{http://schemas.microsoft.com/win/2004/08/events/event}'
substatus = xml[1][9].text
event_id = xml.find(f'.//{ns}EventID').text
computer = xml.find(f'.//{ns}Computer').text
channel = xml.find(f'.//{ns}Channel').text
execution = xml.find(f'.//{ns}Execution')
process_id = execution.get('ProcessID')
thread_id = execution.get('ThreadID')
time_created = xml.find(f'.//{ns}TimeCreated').get('SystemTime')
#data_name = xml.findall('.//EventData')
#substatus = data_name.get('Data')
#print(substatus)
event_data = f'Time: {time_created}, Computer: {computer}, Substatus: {substatus}, Event Id: {event_id}, Channel: {channel}, Process Id: {process_id}, Thread Id: {thread_id}'
print(event_data)
user_data = xml.find(f'.//{ns}UserData')
# user_data has possible any data
else:
ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, " ".join(sys.argv), None, 1)
input()

.evtx is the extension for Windows Eventlog files. It contains data in a special binary format designed by Microsoft so you cannot simply open it in a text editor.
The are open source tools to read .evtx and the NXLog EE can also read .evtx files. (Disclaimer: I'm affiliated with the latter).

I modified the accepted answer a bit as following, so it becomes reusable:
import xml.etree.ElementTree as Et
import win32evtlog
from collections import namedtuple
class EventLogParser:
def __init__(self, exported_log_file):
self.exported_log_file = exported_log_file
def get_all_events(self):
windows_events = []
query_handle = win32evtlog.EvtQuery(str(self.exported_log_file),
win32evtlog.EvtQueryFilePath | win32evtlog.EvtQueryReverseDirection)
while True:
raw_event_collection = win32evtlog.EvtNext(query_handle, 1)
if len(raw_event_collection) == 0:
break
for raw_event in raw_event_collection:
windows_events.append(self.parse_raw_event(raw_event))
return windows_events
def parse_raw_event(self, raw_event):
xml_content = win32evtlog.EvtRender(raw_event, win32evtlog.EvtRenderEventXml)
root = Et.fromstring(xml_content)
ns = "{" + root.tag.split('}')[0].strip('{') + "}"
system = root.find(f'{ns}System')
event_id = system.find(f'{ns}EventID').text
level = system.find(f'{ns}Level').text
time_created = system.find(f'{ns}TimeCreated').get('SystemTime')
computer = system.find(f'{ns}Computer').text
WindowsEvent = namedtuple('WindowsEvent',
'event_id, level, time_created, computer')
return WindowsEvent(event_id, level, time_created, computer)

I use the "python-evtx" library, you can install it using this command:
pip install python-evtx
In my case, I'm not interested in reading records with the "Information" level.
import os
import codecs
from lxml import etree
import Evtx.Evtx as evtx
def evtxFile(absolutePath, filenameWithExt, ext, _fromDate, _toDate):
print("Reading: " + filenameWithExt)
outText = ""
channel = ""
#read the windows event viewer log and convert its contents to XML
with codecs.open(tempFilePath, "a+", "utf-8", "ignore") as tempFile:
with evtx.Evtx(absolutePath) as log:
for record in log.records():
xmlLine = record.xml()
xmlLine = xmlLine.replace(" xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"", "")
xmlParse = etree.XML(xmlLine)
level = parseXMLtoString(xmlParse, ".//Level/text()")
if not level == "0" and not level == "4":
providerName = parseXMLtoString(xmlParse, ".//Provider/#Name")
qualifiers = parseXMLtoString(xmlParse, ".//EventID/#Qualifiers")
timestamp = parseXMLtoString(xmlParse, ".//TimeCreated/#SystemTime")
eventID = parseXMLtoString(xmlParse, ".//EventID/text()")
task = parseXMLtoString(xmlParse, ".//Task/text()")
keywords = parseXMLtoString(xmlParse, ".//Keywords/text()")
eventRecordID = parseXMLtoString(xmlParse, ".//EventRecordID/text()")
channel = parseXMLtoString(xmlParse, ".//Channel/text()")
computer = parseXMLtoString(xmlParse, ".//Computer/text()")
message = parseXMLtoString(xmlParse, ".//Data/text()")
if level == "1":
level = "Critical"
elif level == "2":
level = "Error"
elif level == "3":
level = "Warning"
date = timestamp[0:10]
time = timestamp[11:19]
time = time.replace(".", "")
_date = datetime.strptime(date, "%Y-%m-%d").date()
if _fromDate <= _date <= _toDate:
message = message.replace("<string>", "")
message = message.replace("</string>", "")
message = message.replace("\r\n", " ")
message = message.replace("\n\r", " ")
message = message.replace("\n", " ")
message = message.replace("\r", " ")
outText = date + " " + time + "|" + level + "|" + message.strip() + "|" + task + "|" + computer + "|" + providerName + "|" + qualifiers + "|" + eventID + "|" + eventRecordID + "|" + keywords + "\n"
tempFile.writelines(outText)
with codecs.open(tempFilePath, "r", "utf-8", "ignore") as tempFile2:
myLinesFromDateRange = tempFile2.readlines()
#delete the temporary file that was created
os.remove(tempFilePath)
if len(myLinesFromDateRange) > 0:
createFolder("\\filtered_data_files\\")
outFilename = "windows_" + channel.lower() + "_event_viewer_logs" + ext
myLinesFromDateRange.sort()
#remove duplicate records from the list
myFinalLinesFromDateRange = list(set(myLinesFromDateRange))
myFinalLinesFromDateRange.sort()
with codecs.open(os.getcwd() + "\\filtered_data_files\\" + outFilename, "a+", "utf-8", "ignore") as linesFromDateRange:
linesFromDateRange.seek(0)
if len(linesFromDateRange.read(100)) > 0:
linesFromDateRange.writelines("\n")
linesFromDateRange.writelines(myFinalLinesFromDateRange)
del myLinesFromDateRange[:]
del myFinalLinesFromDateRange[:]
else:
print("No data was found within the specified date range.")
print("Closing: " + filenameWithExt)
I hope it helps you or someone else in the future.
EDIT:
The "tempFilePath" can be anything you want, for example:
tempFilePath = os.getcwd() + "\\tempFile.txt"
I collected some information first before calling the "evtxFile" function:
The "From" and the "To" dates are in the following format: YYYY-MM-DD
Converted the dates to "date" data type:
_fromDate = datetime.strptime(fromDate, "%Y-%m-%d").date()
_toDate = datetime.strptime(toDate, "%Y-%m-%d").date()
Divided the directory where the .evtx files are located into different parts:
def splitDirectory(root, file):
absolutePathOfFile = os.path.join(root, file)
filePathWithoutFilename = os.path.split(absolutePathOfFile)[0]
filenameWithExt = os.path.split(absolutePathOfFile)[1]
filenameWithoutExt = os.path.splitext(filenameWithExt)[0]
extension = os.path.splitext(filenameWithExt)[1]
return absolutePathOfFile, filePathWithoutFilename, filenameWithExt, filenameWithoutExt, extension
for root, subFolders, files in os.walk(directoryPath):
for f in files:
absolutePathOfFile, filePathWithoutFilename, filenameWithExt,
filenameWithoutExt, extension = splitDirectory(root, f)
if extension == ".evtx":
evtxFile(absolutePathOfFile, filenameWithExt, ".txt", _fromDate, _toDate)

Why Python program execution slows down when using functions?

So I have a rather general question I was hoping to get some help with. I put together a Python program that runs through and automates workflows at the state level for all the different counties. The entire program was created for research at school - not actual state work. Anyways, I have two designs shown below. The first is an updated version. It takes about 40 minutes to run. The second design shows the original work. Note that it is not a well structured design. However, it takes about five minutes to run the entire program. Could anybody give any insight why there are such differences between the two? The updated version is still ideal as it is much more reusable (can run and grab any dataset in the url) and easy to understand. Furthermore, 40 minutes to get about a hundred workflows completed is still a plus. Also, this is still a work in progress. A couple minor issues still need to be addressed in the code but it is still a pretty cool program.
Updated Design
import os, sys, urllib2, urllib, zipfile, arcpy
from arcpy import env
path = os.getcwd()
def pickData():
myCount = 1
path1 = 'path2URL'
response = urllib2.urlopen(path1)
print "Enter the name of the files you need"
numZips = raw_input()
numZips2 = numZips.split(",")
myResponse(myCount, path1, response, numZips2)
def myResponse(myCount, path1, response, numZips2):
myPath = os.getcwd()
for each in response:
eachNew = each.split(" ")
eachCounty = eachNew[9].strip("\n").strip("\r")
try:
myCountyDir = os.mkdir(os.path.expanduser(myPath+ "\\counties" + "\\" + eachCounty))
except:
pass
myRetrieveDir = myPath+"\\counties" + "\\" + eachCounty
os.chdir(myRetrieveDir)
myCount+=1
response1 = urllib2.urlopen(path1 + eachNew[9])
for all1 in response1:
allNew = all1.split(",")
allFinal = allNew[0].split(" ")
allFinal1 = allFinal[len(allFinal)-1].strip(" ").strip("\n").strip("\r")
numZipsIter = 0
path8 = path1 + eachNew[9][0:len(eachNew[9])-2] +"/"+ allFinal1
downZip = eachNew[9][0:len(eachNew[9])-2]+".zip"
while(numZipsIter <len(numZips2)):
if (numZips2[numZipsIter][0:3].strip(" ") == "NWI") and ("remap" not in allFinal1):
numZips2New = numZips2[numZipsIter].split("_")
if (numZips2New[0].strip(" ") in allFinal1 and numZips2New[1] != "remap" and numZips2New[2].strip(" ") in allFinal1) and (allFinal1[-3:]=="ZIP" or allFinal1[-3:]=="zip"):
urllib.urlretrieve (path8, allFinal1)
zip1 = zipfile.ZipFile(myRetrieveDir +"\\" + allFinal1)
zip1.extractall(myRetrieveDir)
#maybe just have numzips2 (raw input) as the values before the county number
#numZips2[numZipsIter][0:-7].strip(" ") in allFinal1 or numZips2[numZipsIter][0:-7].strip(" ").lower() in allFinal1) and (allFinal1[-3:]=="ZIP" or allFinal1[-3:]=="zip"
elif (numZips2[numZipsIter].strip(" ") in allFinal1 or numZips2[numZipsIter].strip(" ").lower() in allFinal1) and (allFinal1[-3:]=="ZIP" or allFinal1[-3:]=="zip"):
urllib.urlretrieve (path8, allFinal1)
zip1 = zipfile.ZipFile(myRetrieveDir +"\\" + allFinal1)
zip1.extractall(myRetrieveDir)
numZipsIter+=1
pickData()
#client picks shapefiles to add to map
#section for geoprocessing operations
# get the data frames
#add new data frame, title
#check spaces in ftp crawler
os.chdir(path)
env.workspace = path+ "\\symbology\\"
zp1 = os.listdir(path + "\\counties\\")
def myGeoprocessing(layer1, layer2):
#the code in this function is used for geoprocessing operations
#it returns whatever output is generated from the tools used in the map
try:
arcpy.Clip_analysis(path + "\\symbology\\Stream_order.shp", layer1, path + "\\counties\\" + layer2 + "\\Streams.shp")
except:
pass
streams = arcpy.mapping.Layer(path + "\\counties\\" + layer2 + "\\Streams.shp")
arcpy.ApplySymbologyFromLayer_management(streams, path+ '\\symbology\\streams.lyr')
return streams
def makeMap():
#original wetlands layers need to be entered as NWI_line or NWI_poly
print "Enter the layer or layers you wish to include in the map"
myInput = raw_input();
counter1 = 1
for each in zp1:
print each
print path
zp2 = os.listdir(path + "\\counties\\" + each)
for eachNew in zp2:
#print eachNew
if (eachNew[-4:] == ".shp") and ((myInput in eachNew[0:-7] or myInput.lower() in eachNew[0:-7])or((eachNew[8:12] == "poly" or eachNew[8:12]=='line') and eachNew[8:12] in myInput)):
print eachNew[0:-7]
theMap = arcpy.mapping.MapDocument(path +'\\map.mxd')
df1 = arcpy.mapping.ListDataFrames(theMap,"*")[0]
#this is where we add our layers
layer1 = arcpy.mapping.Layer(path + "\\counties\\" + each + "\\" + eachNew)
if(eachNew[7:11] == "poly" or eachNew[7:11] =="line"):
arcpy.ApplySymbologyFromLayer_management(layer1, path + '\\symbology\\' +myInput+'.lyr')
else:
arcpy.ApplySymbologyFromLayer_management(layer1, path + '\\symbology\\' +eachNew[0:-7]+'.lyr')
# Assign legend variable for map
legend = arcpy.mapping.ListLayoutElements(theMap, "LEGEND_ELEMENT", "Legend")[0]
# add wetland layer to map
legend.autoAdd = True
try:
arcpy.mapping.AddLayer(df1, layer1,"AUTO_ARRANGE")
#geoprocessing steps
streams = myGeoprocessing(layer1, each)
# more geoprocessing options, add the layers to map and assign if they should appear in legend
legend.autoAdd = True
arcpy.mapping.AddLayer(df1, streams,"TOP")
df1.extent = layer1.getExtent(True)
arcpy.mapping.ExportToJPEG(theMap, path + "\\counties\\" + each + "\\map.jpg")
# Save map document to path
theMap.saveACopy(path + "\\counties\\" + each + "\\map.mxd")
del theMap
print "done with map " + str(counter1)
except:
print "issue with map or already exists"
counter1+=1
makeMap()
Original Design
import os, sys, urllib2, urllib, zipfile, arcpy
from arcpy import env
response = urllib2.urlopen('path2URL')
path1 = 'path2URL'
myCount = 1
for each in response:
eachNew = each.split(" ")
myCount+=1
response1 = urllib2.urlopen(path1 + eachNew[9])
for all1 in response1:
#print all1
allNew = all1.split(",")
allFinal = allNew[0].split(" ")
allFinal1 = allFinal[len(allFinal)-1].strip(" ")
if allFinal1[-10:-2] == "poly.ZIP":
response2 = urllib2.urlopen('path2URL')
zipcontent= response2.readlines()
path8 = 'path2URL'+ eachNew[9][0:len(eachNew[9])-2] +"/"+ allFinal1[0:len(allFinal1)-2]
downZip = str(eachNew[9][0:len(eachNew[9])-2])+ ".zip"
urllib.urlretrieve (path8, downZip)
# Set the path to the directory where your zipped folders reside
zipfilepath = 'F:\Misc\presentation'
# Set the path to where you want the extracted data to reside
extractiondir = 'F:\Misc\presentation\counties'
# List all data in the main directory
zp1 = os.listdir(zipfilepath)
# Creates a loop which gives use each zipped folder automatically
# Concatinates zipped folder to original directory in variable done
for each in zp1:
print each[-4:]
if each[-4:] == ".zip":
done = zipfilepath + "\\" + each
zip1 = zipfile.ZipFile(done)
extractiondir1 = extractiondir + "\\" + each[:-4]
zip1.extractall(extractiondir1)
path = os.getcwd()
counter1 = 1
# get the data frames
# Create new layer for all files to be added to map document
env.workspace = "E:\\Misc\\presentation\\symbology\\"
zp1 = os.listdir(path + "\\counties\\")
for each in zp1:
zp2 = os.listdir(path + "\\counties\\" + each)
for eachNew in zp2:
if eachNew[-4:] == ".shp":
wetlandMap = arcpy.mapping.MapDocument('E:\\Misc\\presentation\\wetland.mxd')
df1 = arcpy.mapping.ListDataFrames(wetlandMap,"*")[0]
#print eachNew[-4:]
wetland = arcpy.mapping.Layer(path + "\\counties\\" + each + "\\" + eachNew)
#arcpy.Clip_analysis(path + "\\symbology\\Stream_order.shp", wetland, path + "\\counties\\" + each + "\\Streams.shp")
streams = arcpy.mapping.Layer(path + "\\symbology\\Stream_order.shp")
arcpy.ApplySymbologyFromLayer_management(wetland, path + '\\symbology\\wetland.lyr')
arcpy.ApplySymbologyFromLayer_management(streams, path+ '\\symbology\\streams.lyr')
# Assign legend variable for map
legend = arcpy.mapping.ListLayoutElements(wetlandMap, "LEGEND_ELEMENT", "Legend")[0]
# add the layers to map and assign if they should appear in legend
legend.autoAdd = True
arcpy.mapping.AddLayer(df1, streams,"TOP")
legend.autoAdd = True
arcpy.mapping.AddLayer(df1, wetland,"AUTO_ARRANGE")
df1.extent = wetland.getExtent(True)
# Export the map to a pdf
arcpy.mapping.ExportToJPEG(wetlandMap, path + "\\counties\\" + each + "\\wetland.jpg")
# Save map document to path
wetlandMap.saveACopy(path + "\\counties\\" + each + "\\wetland.mxd")
del wetlandMap
print "done with map " + str(counter1)
counter1+=1

Have a look at this guide:
https://wiki.python.org/moin/PythonSpeed/PerformanceTips
Let me quote:
Function call overhead in Python is relatively high, especially compared with the execution speed of a builtin function. This strongly suggests that where appropriate, functions should handle data aggregates.
So effectively this suggests, to not factor out something as a function that is going to be called hundreds of thousands of times.
In Python functions won't be inlined, and calling them is not cheap. If in doubt use a profiler to find out how many times is each function called, and how long does it take on average. Then optimize.
You might also give PyPy a shot, as they have certain optimizations built in. Reducing the function call overhead in some cases seems to be one of them:
Python equivalence to inline functions or macros
http://pypy.org/performance.html