Guys do anyone know how to read event log file in C:\Windows\System32\winevt\Logs with .evtx extension?
I have already tried to open it using notepad and read using python but notepad says access is denied...
Do anyone know how to do it? Thanks in advance..
This is how you would read the file "Forwarded Events" from the event viewer. You need admin access so I would run it as admin but I it will prompt you for a password if you don't.
import win32evtlog
import xml.etree.ElementTree as ET
import ctypes
import sys
def is_admin():
try:
return ctypes.windll.shell32.IsUserAnAdmin()
except:
return False
if is_admin():
# open event file
query_handle = win32evtlog.EvtQuery(
'C:\Windows\System32\winevt\Logs\ForwardedEvents.evtx',
win32evtlog.EvtQueryFilePath)
read_count = 0
a = 1
while a == 1:
a += 1
# read 1 record(s)
events = win32evtlog.EvtNext(query_handle, 1)
read_count += len(events)
# if there is no record break the loop
if len(events) == 0:
break
for event in events:
xml_content = win32evtlog.EvtRender(event, win32evtlog.EvtRenderEventXml)
# parse xml content
xml = ET.fromstring(xml_content)
# xml namespace, root element has a xmlns definition, so we have to use the namespace
ns = '{http://schemas.microsoft.com/win/2004/08/events/event}'
substatus = xml[1][9].text
event_id = xml.find(f'.//{ns}EventID').text
computer = xml.find(f'.//{ns}Computer').text
channel = xml.find(f'.//{ns}Channel').text
execution = xml.find(f'.//{ns}Execution')
process_id = execution.get('ProcessID')
thread_id = execution.get('ThreadID')
time_created = xml.find(f'.//{ns}TimeCreated').get('SystemTime')
#data_name = xml.findall('.//EventData')
#substatus = data_name.get('Data')
#print(substatus)
event_data = f'Time: {time_created}, Computer: {computer}, Substatus: {substatus}, Event Id: {event_id}, Channel: {channel}, Process Id: {process_id}, Thread Id: {thread_id}'
print(event_data)
user_data = xml.find(f'.//{ns}UserData')
# user_data has possible any data
else:
ctypes.windll.shell32.ShellExecuteW(None, "runas", sys.executable, " ".join(sys.argv), None, 1)
input()
.evtx is the extension for Windows Eventlog files. It contains data in a special binary format designed by Microsoft so you cannot simply open it in a text editor.
The are open source tools to read .evtx and the NXLog EE can also read .evtx files. (Disclaimer: I'm affiliated with the latter).
I modified the accepted answer a bit as following, so it becomes reusable:
import xml.etree.ElementTree as Et
import win32evtlog
from collections import namedtuple
class EventLogParser:
def __init__(self, exported_log_file):
self.exported_log_file = exported_log_file
def get_all_events(self):
windows_events = []
query_handle = win32evtlog.EvtQuery(str(self.exported_log_file),
win32evtlog.EvtQueryFilePath | win32evtlog.EvtQueryReverseDirection)
while True:
raw_event_collection = win32evtlog.EvtNext(query_handle, 1)
if len(raw_event_collection) == 0:
break
for raw_event in raw_event_collection:
windows_events.append(self.parse_raw_event(raw_event))
return windows_events
def parse_raw_event(self, raw_event):
xml_content = win32evtlog.EvtRender(raw_event, win32evtlog.EvtRenderEventXml)
root = Et.fromstring(xml_content)
ns = "{" + root.tag.split('}')[0].strip('{') + "}"
system = root.find(f'{ns}System')
event_id = system.find(f'{ns}EventID').text
level = system.find(f'{ns}Level').text
time_created = system.find(f'{ns}TimeCreated').get('SystemTime')
computer = system.find(f'{ns}Computer').text
WindowsEvent = namedtuple('WindowsEvent',
'event_id, level, time_created, computer')
return WindowsEvent(event_id, level, time_created, computer)
I use the "python-evtx" library, you can install it using this command:
pip install python-evtx
In my case, I'm not interested in reading records with the "Information" level.
import os
import codecs
from lxml import etree
import Evtx.Evtx as evtx
def evtxFile(absolutePath, filenameWithExt, ext, _fromDate, _toDate):
print("Reading: " + filenameWithExt)
outText = ""
channel = ""
#read the windows event viewer log and convert its contents to XML
with codecs.open(tempFilePath, "a+", "utf-8", "ignore") as tempFile:
with evtx.Evtx(absolutePath) as log:
for record in log.records():
xmlLine = record.xml()
xmlLine = xmlLine.replace(" xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"", "")
xmlParse = etree.XML(xmlLine)
level = parseXMLtoString(xmlParse, ".//Level/text()")
if not level == "0" and not level == "4":
providerName = parseXMLtoString(xmlParse, ".//Provider/#Name")
qualifiers = parseXMLtoString(xmlParse, ".//EventID/#Qualifiers")
timestamp = parseXMLtoString(xmlParse, ".//TimeCreated/#SystemTime")
eventID = parseXMLtoString(xmlParse, ".//EventID/text()")
task = parseXMLtoString(xmlParse, ".//Task/text()")
keywords = parseXMLtoString(xmlParse, ".//Keywords/text()")
eventRecordID = parseXMLtoString(xmlParse, ".//EventRecordID/text()")
channel = parseXMLtoString(xmlParse, ".//Channel/text()")
computer = parseXMLtoString(xmlParse, ".//Computer/text()")
message = parseXMLtoString(xmlParse, ".//Data/text()")
if level == "1":
level = "Critical"
elif level == "2":
level = "Error"
elif level == "3":
level = "Warning"
date = timestamp[0:10]
time = timestamp[11:19]
time = time.replace(".", "")
_date = datetime.strptime(date, "%Y-%m-%d").date()
if _fromDate <= _date <= _toDate:
message = message.replace("<string>", "")
message = message.replace("</string>", "")
message = message.replace("\r\n", " ")
message = message.replace("\n\r", " ")
message = message.replace("\n", " ")
message = message.replace("\r", " ")
outText = date + " " + time + "|" + level + "|" + message.strip() + "|" + task + "|" + computer + "|" + providerName + "|" + qualifiers + "|" + eventID + "|" + eventRecordID + "|" + keywords + "\n"
tempFile.writelines(outText)
with codecs.open(tempFilePath, "r", "utf-8", "ignore") as tempFile2:
myLinesFromDateRange = tempFile2.readlines()
#delete the temporary file that was created
os.remove(tempFilePath)
if len(myLinesFromDateRange) > 0:
createFolder("\\filtered_data_files\\")
outFilename = "windows_" + channel.lower() + "_event_viewer_logs" + ext
myLinesFromDateRange.sort()
#remove duplicate records from the list
myFinalLinesFromDateRange = list(set(myLinesFromDateRange))
myFinalLinesFromDateRange.sort()
with codecs.open(os.getcwd() + "\\filtered_data_files\\" + outFilename, "a+", "utf-8", "ignore") as linesFromDateRange:
linesFromDateRange.seek(0)
if len(linesFromDateRange.read(100)) > 0:
linesFromDateRange.writelines("\n")
linesFromDateRange.writelines(myFinalLinesFromDateRange)
del myLinesFromDateRange[:]
del myFinalLinesFromDateRange[:]
else:
print("No data was found within the specified date range.")
print("Closing: " + filenameWithExt)
I hope it helps you or someone else in the future.
EDIT:
The "tempFilePath" can be anything you want, for example:
tempFilePath = os.getcwd() + "\\tempFile.txt"
I collected some information first before calling the "evtxFile" function:
The "From" and the "To" dates are in the following format: YYYY-MM-DD
Converted the dates to "date" data type:
_fromDate = datetime.strptime(fromDate, "%Y-%m-%d").date()
_toDate = datetime.strptime(toDate, "%Y-%m-%d").date()
Divided the directory where the .evtx files are located into different parts:
def splitDirectory(root, file):
absolutePathOfFile = os.path.join(root, file)
filePathWithoutFilename = os.path.split(absolutePathOfFile)[0]
filenameWithExt = os.path.split(absolutePathOfFile)[1]
filenameWithoutExt = os.path.splitext(filenameWithExt)[0]
extension = os.path.splitext(filenameWithExt)[1]
return absolutePathOfFile, filePathWithoutFilename, filenameWithExt, filenameWithoutExt, extension
for root, subFolders, files in os.walk(directoryPath):
for f in files:
absolutePathOfFile, filePathWithoutFilename, filenameWithExt,
filenameWithoutExt, extension = splitDirectory(root, f)
if extension == ".evtx":
evtxFile(absolutePathOfFile, filenameWithExt, ".txt", _fromDate, _toDate)
Hi I have the below program which should give me the list of files created before a given amount of time from the current time
def filesOlderThan(path,day,hour,mins):
file_list = []
for root, dirs, files in os.walk(path):
for a in files:
path1 = root + "\\"+ a
sub_time = datetime.datetime.today() - datetime.timedelta(days=day,hours=hour,minutes=mins)
r_time = time.mktime(sub_time.utctimetuple())
c_time = os.path.getctime(path1)
if(r_time < c_time):
file_list.append(path1)
count = len(file_list)
return file_list,count
files,count = filesOlderThan("C:\\Arena\\sample",0,0,10)
print count
for a in files:
print a
When I run the program
C:\Arena>python getFilesOlderThan.py
0
The program works for values hour values, but does not work with the following input case
I set the minutes to 10
Create a file inside C:\Arena\Sample
Run the program, the program does not give me the file.
r_time = time.mktime(sub_time.utctimetuple())
You're comparing the file's timestamp in localtime withthe current time in UTC.
You should use timetuple() instead of utctimetuple.
You can make some minor improvements to your function, such as doing calculations using datetimes instead of timestamps, using datetime.now instead of today and using cross-platform os.join instead of hard-coded windows separators. It also seems unnecessary to calculate the len of the list inside the function - you can do it outside, if it's needed.
def filesOlderThan(path,day,hour,mins):
delta= datetime.timedelta(days=day,hours=hour,minutes=mins)
now= datetime.datetime.now()
file_list = []
for root, dirs, files in os.walk(path):
for a in files:
path1 = os.path.join(root, a)
c_time = datetime.datetime.fromtimestamp(os.path.getctime(path1))
if(now - delta < c_time):
file_list.append(path1)
return file_list
Finally, if you ever plan on sending these timestamps to another system, remember to convert them to utc (both the file's and the current). You'll probably want to use pytz for that.
i create this function but you can modified it
import datetime
import time
def diff(file):
today = datetime.datetime.today()
file_mtime = time.ctime(os.path.getmtime(file))
t_ob = time.strptime(file_mtime)
t = time.strftime("%Y-%m-%d %H-%M-%S", t_ob)
file_d = datetime.datetime.strptime(t, "%Y-%m-%d %H-%M-%S")
minutes = divmod((today-file_d).total_seconds(), 60)
if (today - file_d).days > 30:
return f"{round(((today - file_d).days)/30)} months"
elif (today - file_d).days != 0:
return f"{(today - file_d).days} days"
elif minutes[0] > 59:
return f"{round(minutes[0]/60)} huors"
elif minutes[0] != 0:
return f"{round(minutes[0])} minutes"
else:
return f"{round(minutes[1])} seconds"
>>>diff("file.txt")
I'm trying to return a variable (list) in python but it does not allows me to return it. Instead it goes into except block and returns somethhing else.
The code is like this: Initially I'm getting period_value(datetime), search_name(location name) and time_period(day,week,month,year) via Ajax calls. If its day, then its work fine, but in other cases(week,month,year) the result variable does gives me satisfied data but it does not returns the data and instead goes into except and returns something else. This seems to be a weird problem. Kindly check where I'm going wrong?
if request.POST.has_key('period_val') or request.POST.has_key('search_name'):
try : period_val = request.POST['period_val']
except : period_val = datetime.datetime.strftime((datetime.date.today()), '%m/%d/%Y')
try: time_period = request.POST['time_period']
except : time_period = 'day'
try : search_name = request.POST['search_name']
except : search_name = ""
return HttpResponse(json.dumps(get_widgets_data(request,period_val,time_period,search_name)), mimetype = "application/json")
get_widgets_data functions
def get_widgets_data(request,period_val,time_period,search_name):
if time_period == 'day':
file_name = datetime.datetime.strptime(period_val, '%m/%d/%Y')
current_file_name = file_name.strftime("%Y-%m-%d.json")
yester_date = file_name - timedelta(days = 1)
yester_file_name = yester_date.strftime("%Y-%m-%d.json")
elif time_period == 'week':
file_name = str(period_val).split(", ")
current_file_name = file_name[1] + "-w" + file_name[0].split(" ")[0]+".json"
if (int(file_name[0].split(" ")[0]) - 1) <= 0:
year = int(file_name[1]) - 1
week = datetime.date(year,12,31).isocalendar()[-1]
yester_file_name = file_name[1] + "-w" + file_name[0]+".json"
else :
yester_file_name = file_name[1] + "-w" + str(int(file_name[0].split(" ")[0]) - 1)+".json"
elif time_period == 'month':
file_name = datetime.datetime.strptime(period_val, '%B, %Y')
current_file_name = file_name.strftime("%Y-%m.json")
yester_month = int((current_file_name.split(".")[0]).split("-")[1]) - 1
yester_year = int((current_file_name.split(".")[0]).split("-")[0])
if yester_month == 0:
yester_year = yester_year - 1
yester_month = 12
yester_file_name = str(yester_year) + "-" + str(yester_month) + ".json"
elif time_period == 'year':
file_name = datetime.datetime.strptime(period_val, '%Y')
current_file_name = file_name.strftime("%Y.json")
yester_file_name = str(int(period_val)-1) + ".json"
with open( os.path.join(os.path.dirname(__file__),'../../../../service/report/data/processed/Akamai/energy_meter/',current_file_name)) as file_handler:
current_file_content = json.load(file_handler)
file_handler.close()
user_level_mapping = UserLevelMapping.objects.filter(user_id=request.user.id)
level = (user_level_mapping[0].level_id).split(':')[0]
l_id = (user_level_mapping[0].level_id).split(':')[1]
try : # try if this file exists or not else goes into except.
with open( os.path.join(os.path.dirname(__file__),'../../../../service/report/data/processed/Akamai/energy_meter/',yester_file_name)) as file_handler:
yester_file_content = json.load(file_handler)
file_handler.close()
device_level_mapping_values = device_level_mapping_func(level,l_id)
yester_time = list()
yester_file_datetime = yester_file_content["device"].keys()
yester_total = 0.0
for time in yester_file_datetime:
yester_time.append(time.split(" ")[1])
for date_time in current_file_content["device"].keys():
time = date_time.split(" ")[1]
if time in yester_time:
for device in device_level_mapping_values:
yester_total += float(yester_file_content["device"][date_time][str(device)])
current_total = float(current_file_content['total']['levels']['level'+str(level)][str(l_id)])
baseline_energy_yester = (yester_total) - current_total
results = (current_total,baseline_energy_yester)
print results # prints(123,345) but does not returns. instead goes into except for week,month and year and returns (123,Nan)
return results
except:
current_total = float(current_file_content['total']['levels']['level'+str(level)][str(l_id)])
baseline_energy_yester = "NaN"
results = (current_total,baseline_energy_yester)
return results
Using except without identifying the types you're catching is a dangerous antipattern. If the code in the try block encountered a KeyboardInterrupt, NameError, ImportError or IOError, is it really safe to assume that it's safe to continue and the recovery is simply to make sure that 'search_name' is empty?
try : search_name = request.POST['search_name']
except : search_name = ""
As an aside, you can ask a dict to return a default if no matching key is defined:
search_name = request.POST.get('search_name', None)
Instead it goes into except block and returns somethhing else.
In general if your code encounters an exception, something is "wrong". Wrongness can vary, so it might not be devastating, but if you're not certain which exception is occurring and why then it's best not to try to handle it at all. Let it fly all the way up the stack either to some code that has context to recover or gracefully fail, or better still in cases like this -- all the way up to the developer (you!) so you can address the failure in design/implementation.
To get value of a POST post data field try using
period_val = request.POST.get('period_val')
instead of
period_val = request.POST['period_val']
When copying large files using shutil.copy(), you get no indication of how the operation is progressing..
I have put together something that works - it uses a simple ProgressBar class (which simple returns a simple ASCII progress bar, as a string), and a loop of open().read() and .write() to do the actual copying. It displays the progress bar using sys.stdout.write("\r%s\r" % (the_progress_bar)) which is a little hackish, but it works.
You can see the code (in context) on github here
Is there any built-in module that will do this better? Is there any improvements that can be made to this code?
Two things:
I would make the default block size a lot larger than 512. I would start with 16384 and perhaps more.
For modularity, it might be better to have the copy_with_prog function not output the progress bar itself, but call a callback function so the caller can decide how to display the progress.
Perhaps something like this:
def copy_with_prog(src, dest, callback = None):
while True:
# copy loop stuff
if callback:
callback(pos, total)
prog = ProgressBar(...)
copy_with_prog(src, dest, lambda pos, total: prog.update(pos, total))
Overkill? Perhaps. But on almost any system, Linux, Mac, and With a quick wxWidgets install on Windows, you can have the real deal, with pause and cancel buttons in a gui setup. Macs ship with wxWidgets these days, and it's a common package on Linux.
A single file is very quick (it will immediately finish and look broken) so you might consider creating a fileSet job that ticks along once per file instead of once per block. Enjoy!
-Jim Carroll
"""
Threaded Jobs.
Any class that does a long running process can inherit
from ThreadedJob. This enables running as a background
thread, progress notification, pause and cancel. The
time remaining is also calculated by the ThreadedJob class.
"""
import wx.lib.newevent
import thread
import exceptions
import time
(RunEvent, EVT_RUN) = wx.lib.newevent.NewEvent()
(CancelEvent, EVT_CANCEL) = wx.lib.newevent.NewEvent()
(DoneEvent, EVT_DONE) = wx.lib.newevent.NewEvent()
(ProgressStartEvent, EVT_PROGRESS_START) = wx.lib.newevent.NewEvent()
(ProgressEvent, EVT_PROGRESS) = wx.lib.newevent.NewEvent()
class InterruptedException(exceptions.Exception):
def __init__(self, args = None):
self.args = args
#
#
class ThreadedJob:
def __init__(self):
# tell them ten seconds at first
self.secondsRemaining = 10.0
self.lastTick = 0
# not running yet
self.isPaused = False
self.isRunning = False
self.keepGoing = True
def Start(self):
self.keepGoing = self.isRunning = True
thread.start_new_thread(self.Run, ())
self.isPaused = False
#
def Stop(self):
self.keepGoing = False
#
def WaitUntilStopped(self):
while self.isRunning:
time.sleep(0.1)
wx.SafeYield()
#
#
def IsRunning(self):
return self.isRunning
#
def Run(self):
# this is overridden by the
# concrete ThreadedJob
print "Run was not overloaded"
self.JobFinished()
pass
#
def Pause(self):
self.isPaused = True
pass
#
def Continue(self):
self.isPaused = False
pass
#
def PossibleStoppingPoint(self):
if not self.keepGoing:
raise InterruptedException("process interrupted.")
wx.SafeYield()
# allow cancel while paused
while self.isPaused:
if not self.keepGoing:
raise InterruptedException("process interrupted.")
# don't hog the CPU
time.sleep(0.1)
#
#
def SetProgressMessageWindow(self, win):
self.win = win
#
def JobBeginning(self, totalTicks):
self.lastIterationTime = time.time()
self.totalTicks = totalTicks
if hasattr(self, "win") and self.win:
wx.PostEvent(self.win, ProgressStartEvent(total=totalTicks))
#
#
def JobProgress(self, currentTick):
dt = time.time() - self.lastIterationTime
self.lastIterationTime = time.time()
dtick = currentTick - self.lastTick
self.lastTick = currentTick
alpha = 0.92
if currentTick > 1:
self.secondsPerTick = dt * (1.0 - alpha) + (self.secondsPerTick * alpha)
else:
self.secondsPerTick = dt
#
if dtick > 0:
self.secondsPerTick /= dtick
self.secondsRemaining = self.secondsPerTick * (self.totalTicks - 1 - currentTick) + 1
if hasattr(self, "win") and self.win:
wx.PostEvent(self.win, ProgressEvent(count=currentTick))
#
#
def SecondsRemaining(self):
return self.secondsRemaining
#
def TimeRemaining(self):
if 1: #self.secondsRemaining > 3:
minutes = self.secondsRemaining // 60
seconds = int(self.secondsRemaining % 60.0)
return "%i:%02i" % (minutes, seconds)
else:
return "a few"
#
def JobFinished(self):
if hasattr(self, "win") and self.win:
wx.PostEvent(self.win, DoneEvent())
#
# flag we're done before we post the all done message
self.isRunning = False
#
#
class EggTimerJob(ThreadedJob):
""" A sample Job that demonstrates the mechanisms and features of the Threaded Job"""
def __init__(self, duration):
self.duration = duration
ThreadedJob.__init__(self)
#
def Run(self):
""" This can either be run directly for synchronous use of the job,
or started as a thread when ThreadedJob.Start() is called.
It is responsible for calling JobBeginning, JobProgress, and JobFinished.
And as often as possible, calling PossibleStoppingPoint() which will
sleep if the user pauses, and raise an exception if the user cancels.
"""
self.time0 = time.clock()
self.JobBeginning(self.duration)
try:
for count in range(0, self.duration):
time.sleep(1.0)
self.JobProgress(count)
self.PossibleStoppingPoint()
#
except InterruptedException:
# clean up if user stops the Job early
print "canceled prematurely!"
#
# always signal the end of the job
self.JobFinished()
#
#
def __str__(self):
""" The job progress dialog expects the job to describe its current state."""
response = []
if self.isPaused:
response.append("Paused Counting")
elif not self.isRunning:
response.append("Will Count the seconds")
else:
response.append("Counting")
#
return " ".join(response)
#
#
class FileCopyJob(ThreadedJob):
""" A common file copy Job. """
def __init__(self, orig_filename, copy_filename, block_size=32*1024):
self.src = orig_filename
self.dest = copy_filename
self.block_size = block_size
ThreadedJob.__init__(self)
#
def Run(self):
""" This can either be run directly for synchronous use of the job,
or started as a thread when ThreadedJob.Start() is called.
It is responsible for calling JobBeginning, JobProgress, and JobFinished.
And as often as possible, calling PossibleStoppingPoint() which will
sleep if the user pauses, and raise an exception if the user cancels.
"""
self.time0 = time.clock()
try:
source = open(self.src, 'rb')
# how many blocks?
import os
(st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime) = os.stat(self.src)
num_blocks = st_size / self.block_size
current_block = 0
self.JobBeginning(num_blocks)
dest = open(self.dest, 'wb')
while 1:
copy_buffer = source.read(self.block_size)
if copy_buffer:
dest.write(copy_buffer)
current_block += 1
self.JobProgress(current_block)
self.PossibleStoppingPoint()
else:
break
source.close()
dest.close()
except InterruptedException:
# clean up if user stops the Job early
dest.close()
# unlink / delete the file that is partially copied
os.unlink(self.dest)
print "canceled, dest deleted!"
#
# always signal the end of the job
self.JobFinished()
#
#
def __str__(self):
""" The job progress dialog expects the job to describe its current state."""
response = []
if self.isPaused:
response.append("Paused Copy")
elif not self.isRunning:
response.append("Will Copy a file")
else:
response.append("Copying")
#
return " ".join(response)
#
#
class JobProgress(wx.Dialog):
""" This dialog shows the progress of any ThreadedJob.
It can be shown Modally if the main application needs to suspend
operation, or it can be shown Modelessly for background progress
reporting.
app = wx.PySimpleApp()
job = EggTimerJob(duration = 10)
dlg = JobProgress(None, job)
job.SetProgressMessageWindow(dlg)
job.Start()
dlg.ShowModal()
"""
def __init__(self, parent, job):
self.job = job
wx.Dialog.__init__(self, parent, -1, "Progress", size=(350,200))
# vertical box sizer
sizeAll = wx.BoxSizer(wx.VERTICAL)
# Job status text
self.JobStatusText = wx.StaticText(self, -1, "Starting...")
sizeAll.Add(self.JobStatusText, 0, wx.EXPAND|wx.ALL, 8)
# wxGague
self.ProgressBar = wx.Gauge(self, -1, 10, wx.DefaultPosition, (250, 15))
sizeAll.Add(self.ProgressBar, 0, wx.EXPAND|wx.ALL, 8)
# horiz box sizer, and spacer to right-justify
sizeRemaining = wx.BoxSizer(wx.HORIZONTAL)
sizeRemaining.Add((2,2), 1, wx.EXPAND)
# time remaining read-only edit
# putting wide default text gets a reasonable initial layout.
self.remainingText = wx.StaticText(self, -1, "???:??")
sizeRemaining.Add(self.remainingText, 0, wx.LEFT|wx.RIGHT|wx.ALIGN_CENTER_VERTICAL, 8)
# static text: remaining
self.remainingLabel = wx.StaticText(self, -1, "remaining")
sizeRemaining.Add(self.remainingLabel, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL, 8)
# add that row to the mix
sizeAll.Add(sizeRemaining, 1, wx.EXPAND)
# horiz box sizer & spacer
sizeButtons = wx.BoxSizer(wx.HORIZONTAL)
sizeButtons.Add((2,2), 1, wx.EXPAND|wx.ADJUST_MINSIZE)
# Pause Button
self.PauseButton = wx.Button(self, -1, "Pause")
sizeButtons.Add(self.PauseButton, 0, wx.ALL, 4)
self.Bind(wx.EVT_BUTTON, self.OnPauseButton, self.PauseButton)
# Cancel button
self.CancelButton = wx.Button(self, wx.ID_CANCEL, "Cancel")
sizeButtons.Add(self.CancelButton, 0, wx.ALL, 4)
self.Bind(wx.EVT_BUTTON, self.OnCancel, self.CancelButton)
# Add all the buttons on the bottom row to the dialog
sizeAll.Add(sizeButtons, 0, wx.EXPAND|wx.ALL, 4)
self.SetSizer(sizeAll)
#sizeAll.Fit(self)
sizeAll.SetSizeHints(self)
# jobs tell us how they are doing
self.Bind(EVT_PROGRESS_START, self.OnProgressStart)
self.Bind(EVT_PROGRESS, self.OnProgress)
self.Bind(EVT_DONE, self.OnDone)
self.Layout()
#
def OnPauseButton(self, event):
if self.job.isPaused:
self.job.Continue()
self.PauseButton.SetLabel("Pause")
self.Layout()
else:
self.job.Pause()
self.PauseButton.SetLabel("Resume")
self.Layout()
#
#
def OnCancel(self, event):
self.job.Stop()
#
def OnProgressStart(self, event):
self.ProgressBar.SetRange(event.total)
self.statusUpdateTime = time.clock()
#
def OnProgress(self, event):
# update the progress bar
self.ProgressBar.SetValue(event.count)
self.remainingText.SetLabel(self.job.TimeRemaining())
# update the text a max of 20 times a second
if time.clock() - self.statusUpdateTime > 0.05:
self.JobStatusText.SetLabel(str(self.job))
self.statusUpdateTime = time.clock()
self.Layout()
#
#
# when a job is done
def OnDone(self, event):
self.ProgressBar.SetValue(0)
self.JobStatusText.SetLabel("Finished")
self.Destroy()
#
#
if __name__ == "__main__":
app = wx.PySimpleApp()
#job = EggTimerJob(duration = 10)
job = FileCopyJob("VeryBigFile.mp4", "/tmp/test_junk.mp4", 1024*1024*10)
dlg = JobProgress(None, job)
job.SetProgressMessageWindow(dlg)
job.Start()
dlg.ShowModal()
#
I have this shutil.copy() with progress bar made in a simple way just with built in modules.
If you are using utf-8 encoding you can get a progress like the second example in the gif image:
Progress bars examples for this:
Read the comments to change style and colors. The first and last examples don't need utf-8.
You can use the command CPprogress(SOURCE, DESTINATION) just where you had shutil.copy(src, dst):
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
CPprogress(SOURCE, DESTINATION)
I made this to give shutil.copy() [or shutil.copy2() in this case] a progress bar.
You can use CPprogress(SOURCE, DESTINATION) just like shutil.copy(src, dst). SOURCE must be a file path and DESTINATION a file or folder path.
It will give you a progress bar for each file copied. Just copy this code above the place where you want to use CPprogress(SOURCE, DESTINATION) in your code.
You can easily change the look of the progress bar:
- To keep the style and just change the colors, replace the colors values of progressCOLOR and finalCOLOR (orange code at the end of the lines).
- The use a solid block progress bar, # -*- coding: utf-8 -*- is required. Otherwise, you will get an encoding error. Some basic terminals, like xterm, may not show the progress bar because of the utf-8 characters.
To use this style, remove the comments #STYLE# in lines ###COLORS### - BlueCOLOR and endBLOCK.
In def getPERCECENTprogress() remove the comments #STYLE# AND COMMENT THE PREVIOUS line. Do the same in def CPprogress()
If you don't want the utf-8 encoding, delete the four lines beginning with #STYLE#.
NOTE: If you want to copy lots of small files, the copy process for file is so fast
that all you will see is a lot of lines scrolling in you terminal window - not enough time for a 'progress'.
In that case, I use an overall progress that shows only one progress bar to the complete job. nzX
'''
import os
import shutil
import sys
import threading
import time
######## COLORS ######
progressCOLOR = '\033[38;5;33;48;5;236m' #\033[38;5;33;48;5;236m# copy inside '' for colored progressbar| orange:#\033[38;5;208;48;5;235m
finalCOLOR = '\033[38;5;33;48;5;33m' #\033[38;5;33;48;5;33m# copy inside '' for colored progressbar| orange:#\033[38;5;208;48;5;208m
#STYLE#BlueCOLOR = '\033[38;5;33m'#\033[38;5;33m# copy inside '' for colored progressbar Orange#'\033[38;5;208m'# # BG progress# #STYLE#
#STYLE#endBLOCK = '' # ▌ copy OR '' for none # BG progress# #STYLE# requires utf8 coding header
########
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
CEND = '\033[0m'
def getPERCECENTprogress(source_path, destination_path):
time.sleep(.24)
if os.path.exists(destination_path):
while os.path.getsize(source_path) != os.path.getsize(destination_path):
sys.stdout.write('\r')
percentagem = int((float(os.path.getsize(destination_path))/float(os.path.getsize(source_path))) * 100)
steps = int(percentagem/5)
copiado = int(os.path.getsize(destination_path)/1000000)# Should be 1024000 but this get's equal to Thunar file manager report (Linux - Xfce)
sizzz = int(os.path.getsize(source_path)/1000000)
sys.stdout.write((" {:d} / {:d} Mb ".format(copiado, sizzz)) + (BOLD + progressCOLOR + "{:20s}".format('|'*steps) + CEND) + (" {:d}% ".format(percentagem))) # BG progress
#STYLE#sys.stdout.write((" {:d} / {:d} Mb ".format(copiado, sizzz)) + (BOLD + BlueCOLOR + "▐" + "{:s}".format('█'*steps) + CEND) + ("{:s}".format(' '*(20-steps))+ BOLD + BlueCOLOR + endBLOCK+ CEND) +(" {:d}% ".format(percentagem))) #STYLE# # BG progress# closer to GUI but less compatible (no block bar with xterm) # requires utf8 coding header
sys.stdout.flush()
time.sleep(.01)
def CPprogress(SOURCE, DESTINATION):
if os.path.isdir(DESTINATION):
dst_file = os.path.join(DESTINATION, os.path.basename(SOURCE))
else: dst_file = DESTINATION
print " "
print (BOLD + UNDERLINE + "FROM:" + CEND + " "), SOURCE
print (BOLD + UNDERLINE + "TO:" + CEND + " "), dst_file
print " "
threading.Thread(name='progresso', target=getPERCECENTprogress, args=(SOURCE, dst_file)).start()
shutil.copy2(SOURCE, DESTINATION)
time.sleep(.02)
sys.stdout.write('\r')
sys.stdout.write((" {:d} / {:d} Mb ".format((int(os.path.getsize(dst_file)/1000000)), (int(os.path.getsize(SOURCE)/1000000)))) + (BOLD + finalCOLOR + "{:20s}".format('|'*20) + CEND) + (" {:d}% ".format(100))) # BG progress 100%
#STYLE#sys.stdout.write((" {:d} / {:d} Mb ".format((int(os.path.getsize(dst_file)/1000000)), (int(os.path.getsize(SOURCE)/1000000)))) + (BOLD + BlueCOLOR + "▐" + "{:s}{:s}".format(('█'*20), endBLOCK) + CEND) + (" {:d}% ".format(100))) #STYLE# # BG progress 100%# closer to GUI but less compatible (no block bar with xterm) # requires utf8 coding header
sys.stdout.flush()
print " "
print " "
'''
#Ex. Copy all files from root of the source dir to destination dir
folderA = '/path/to/SOURCE' # SOURCE
folderB = '/path/to/DESTINATION' # DESTINATION
for FILE in os.listdir(folderA):
if not os.path.isdir(os.path.join(folderA, FILE)):
if os.path.exists(os.path.join(folderB, FILE)): continue # as we are using shutil.copy2() that overwrites destination, this skips existing files
CPprogress(os.path.join(folderA, FILE), folderB) # use the command as if it was shutil.copy2() but with progress
75 / 150 Mb |||||||||| | 50%
'''
If you want to use the Windows copy dialog with progress you can use these:
https://github.com/tjguk/winshell/
https://github.com/frmdstryr/pywinutils
If you want an overall progress, you can use something like this (made for another script). Note that in this case, the 'threading.Thread' that calls the progress bar was placed outside the 'for' loop. Also, the measures need be taken in a different way. This is the third example (non utf-8) from the gif image in the previous answer. It adds a files 'ToGo’ counting:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Ex.
CopyProgress('/path/to/SOURCE', '/path/to/DESTINATION')
I think this 'copy with overall progress' is very 'plastic' and can be easily adapted.
By default, it will RECURSIVELY copy the CONTENT of 'path/to/SOURCE' to 'path/to/DESTINATION/' keeping the directory tree.
Paying attention to comments, there are 4 main options that can be immediately change:
1 - The LOOK of the progress bar: see COLORS and the PAIR of STYLE lines in 'def getPERCECENTprogress'(inside and after the 'while' loop);
2 - The DESTINATION path: to get 'path/to/DESTINATION/SOURCE_NAME' as target, comment the 2nd 'DST =' definition on the top of the 'def CopyProgress(SOURCE, DESTINATION)' function;
3 - If you don't want to RECURSIVELY copy from sub-directories but just the files in the root source directory to the root of destination, you can use os.listdir() instead of os.walk(). Read the comments inside 'def CopyProgress(SOURCE, DESTINATION)' function to disable RECURSION. Be aware that the RECURSION changes(4x2) must be made in both os.walk() loops;
4 - Handling destination files: if you use this in a situation where the destination filename may already exist, by default, the file is skipped and the loop will jump to the next and so on. On the other way shutil.copy2(), by default, overwrites destination file if exists. Alternatively, you can handle files that exist by overwriting or renaming (according to current date and time). To do that read the comments after 'if os.path.exists(dstFILE): continue' both in the count bytes loop and the main loop. Be aware that the changes must match in both loops (as described in comments) or the progress function will not work properly.
'''
import os
import shutil
import sys
import threading
import time
progressCOLOR = '\033[38;5;33;48;5;236m' #BLUEgreyBG
finalCOLOR = '\033[48;5;33m' #BLUEBG
# check the color codes below and paste above
###### COLORS #######
# WHITEblueBG = '\033[38;5;15;48;5;33m'
# BLUE = '\033[38;5;33m'
# BLUEBG = '\033[48;5;33m'
# ORANGEBG = '\033[48;5;208m'
# BLUEgreyBG = '\033[38;5;33;48;5;236m'
# ORANGEgreyBG = '\033[38;5;208;48;5;236m' # = '\033[38;5;FOREGROUND;48;5;BACKGROUNDm' # ver 'https://i.stack.imgur.com/KTSQa.png' para 256 color codes
# INVERT = '\033[7m'
###### COLORS #######
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
CEND = '\033[0m'
FilesLeft = 0
def FullFolderSize(path):
TotalSize = 0
if os.path.exists(path):# to be safely used # if FALSE returns 0
for root, dirs, files in os.walk(path):
for file in files:
TotalSize += os.path.getsize(os.path.join(root, file))
return TotalSize
def getPERCECENTprogress(source_path, destination_path, bytes_to_copy):
dstINIsize = FullFolderSize(destination_path)
time.sleep(.25)
print " "
print (BOLD + UNDERLINE + "FROM:" + CEND + " "), source_path
print (BOLD + UNDERLINE + "TO:" + CEND + " "), destination_path
print " "
if os.path.exists(destination_path):
while bytes_to_copy != (FullFolderSize(destination_path)-dstINIsize):
sys.stdout.write('\r')
percentagem = int((float((FullFolderSize(destination_path)-dstINIsize))/float(bytes_to_copy)) * 100)
steps = int(percentagem/5)
copiado = '{:,}'.format(int((FullFolderSize(destination_path)-dstINIsize)/1000000))# Should be 1024000 but this get's closer to the file manager report
sizzz = '{:,}'.format(int(bytes_to_copy/1000000))
sys.stdout.write((" {:s} / {:s} Mb ".format(copiado, sizzz)) + (BOLD + progressCOLOR + "{:20s}".format('|'*steps) + CEND) + (" {:d}% ".format(percentagem)) + (" {:d} ToGo ".format(FilesLeft))) # STYLE 1 progress default #
#BOLD# sys.stdout.write(BOLD + (" {:s} / {:s} Mb ".format(copiado, sizzz)) + (progressCOLOR + "{:20s}".format('|'*steps) + CEND) + BOLD + (" {:d}% ".format(percentagem)) + (" {:d} ToGo ".format(FilesLeft))+ CEND) # STYLE 2 progress BOLD #
#classic B/W# sys.stdout.write(BOLD + (" {:s} / {:s} Mb ".format(copiado, sizzz)) + ("|{:20s}|".format('|'*steps)) + (" {:d}% ".format(percentagem)) + (" {:d} ToGo ".format(FilesLeft))+ CEND) # STYLE 3 progress classic B/W #
sys.stdout.flush()
time.sleep(.01)
sys.stdout.write('\r')
time.sleep(.05)
sys.stdout.write((" {:s} / {:s} Mb ".format('{:,}'.format(int((FullFolderSize(destination_path)-dstINIsize)/1000000)), '{:,}'.format(int(bytes_to_copy/1000000)))) + (BOLD + finalCOLOR + "{:20s}".format(' '*20) + CEND) + (" {:d}% ".format( 100)) + (" {:s} ".format(' ')) + "\n") # STYLE 1 progress default #
#BOLD# sys.stdout.write(BOLD + (" {:s} / {:s} Mb ".format('{:,}'.format(int((FullFolderSize(destination_path)-dstINIsize)/1000000)), '{:,}'.format(int(bytes_to_copy/1000000)))) + (finalCOLOR + "{:20s}".format(' '*20) + CEND) + BOLD + (" {:d}% ".format( 100)) + (" {:s} ".format(' ')) + "\n" + CEND ) # STYLE 2 progress BOLD #
#classic B/W# sys.stdout.write(BOLD + (" {:s} / {:s} Mb ".format('{:,}'.format(int((FullFolderSize(destination_path)-dstINIsize)/1000000)), '{:,}'.format(int(bytes_to_copy/1000000)))) + ("|{:20s}|".format('|'*20)) + (" {:d}% ".format( 100)) + (" {:s} ".format(' ')) + "\n" + CEND ) # STYLE 3 progress classic B/W #
sys.stdout.flush()
print " "
print " "
def CopyProgress(SOURCE, DESTINATION):
global FilesLeft
DST = os.path.join(DESTINATION, os.path.basename(SOURCE))
# <- the previous will copy the Source folder inside of the Destination folder. Result Target: path/to/Destination/SOURCE_NAME
# -> UNCOMMENT the next (# DST = DESTINATION) to copy the CONTENT of Source to the Destination. Result Target: path/to/Destination
DST = DESTINATION # UNCOMMENT this to specify the Destination as the target itself and not the root folder of the target
#
if DST.startswith(SOURCE):
print " "
print BOLD + UNDERLINE + 'Source folder can\'t be changed.' + CEND
print 'Please check your target path...'
print " "
print BOLD + ' CANCELED' + CEND
print " "
exit()
#count bytes to copy
Bytes2copy = 0
for root, dirs, files in os.walk(SOURCE): # USE for filename in os.listdir(SOURCE): # if you don't want RECURSION #
dstDIR = root.replace(SOURCE, DST, 1) # USE dstDIR = DST # if you don't want RECURSION #
for filename in files: # USE if not os.path.isdir(os.path.join(SOURCE, filename)): # if you don't want RECURSION #
dstFILE = os.path.join(dstDIR, filename)
if os.path.exists(dstFILE): continue # must match the main loop (after "threading.Thread")
# To overwrite delete dstFILE first here so the progress works properly: ex. change continue to os.unlink(dstFILE)
# To rename new files adding date and time, instead of deleating and overwriting,
# comment 'if os.path.exists(dstFILE): continue'
Bytes2copy += os.path.getsize(os.path.join(root, filename)) # USE os.path.getsize(os.path.join(SOURCE, filename)) # if you don't want RECURSION #
FilesLeft += 1
# <- count bytes to copy
#
# Treading to call the preogress
threading.Thread(name='progresso', target=getPERCECENTprogress, args=(SOURCE, DST, Bytes2copy)).start()
# main loop
for root, dirs, files in os.walk(SOURCE): # USE for filename in os.listdir(SOURCE): # if you don't want RECURSION #
dstDIR = root.replace(SOURCE, DST, 1) # USE dstDIR = DST # if you don't want RECURSION #
if not os.path.exists(dstDIR):
os.makedirs(dstDIR)
for filename in files: # USE if not os.path.isdir(os.path.join(SOURCE, filename)): # if you don't want RECURSION #
srcFILE = os.path.join(root, filename) # USE os.path.join(SOURCE, filename) # if you don't want RECURSION #
dstFILE = os.path.join(dstDIR, filename)
if os.path.exists(dstFILE): continue # MUST MATCH THE PREVIOUS count bytes loop
# <- <- this jumps to the next file without copying this file, if destination file exists.
# Comment to copy with rename or overwrite dstFILE
#
# RENAME part below
head, tail = os.path.splitext(filename)
count = -1
year = int(time.strftime("%Y"))
month = int(time.strftime("%m"))
day = int(time.strftime("%d"))
hour = int(time.strftime("%H"))
minute = int(time.strftime("%M"))
while os.path.exists(dstFILE):
count += 1
if count == 0:
dstFILE = os.path.join(dstDIR, '{:s}[{:d}.{:d}.{:d}]{:d}-{:d}{:s}'.format(head, year, month, day, hour, minute, tail))
else:
dstFILE = os.path.join(dstDIR, '{:s}[{:d}.{:d}.{:d}]{:d}-{:d}[{:d}]{:s}'.format(head, year, month, day, hour, minute, count, tail))
# END of RENAME part
shutil.copy2(srcFILE, dstFILE)
FilesLeft -= 1
#
'''
Ex.
CopyProgress('/path/to/SOURCE', '/path/to/DESTINATION')
'''
Alternatively, you can use ROBOCOPY with the os module. It won't give you a progress bar, but it'll give you a percentage indicator as well as a robust summary at the end.
import os
def robocopy(source, destination, extension=''):
os.system("robocopy {} {} {} /xx /njh".format(source, destination, extension))
# Usage example
robocopy(r'C:\Users\Example\Downloads', r'C:\Users\Example\Desktop', '*.mov')
The example above will copy all .mov files to the desktop
Leaving extension blank will copy all files in the source folder to the destination folder.
/xx removes extra files/directories from being listed
/njh removes job header
See documentation for more info:
https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/robocopy
That is simple PySide app can copy any file from source to destination
#!/usr/bin/python3
import os
import sys
from PySide2.QtWidgets import QProgressBar, QApplication, QDialog, QMainWindow, QPushButton
from PySide2.QtCore import QThread, Signal, Slot
class ProgressDialog(QDialog):
def __init__(self, parent, source, destination):
QDialog.__init__(self, parent)
self.resize(400, 50)
self.parent = parent
self.source = source
self.destination = destination
self.prog = QProgressBar(self)
self.prog.setMaximum(100)
self.prog.setMinimum(0)
self.prog.setFormat("%p%")
def start(self):
self.show()
self.copy()
def copy(self):
copy_thread = CopyThread(self, self.source, self.destination)
copy_thread.procPartDone.connect(self.update_progress)
copy_thread.procDone.connect(self.finished_copy)
copy_thread.start()
def update_progress(self, progress):
self.prog.setValue(progress)
def finished_copy(self, state):
self.close()
class CopyThread(QThread):
procDone = Signal(bool)
procPartDone = Signal(int)
def __init__(self, parent, source: str, destination: str):
QThread.__init__(self, parent)
self.source = source
self.destination = destination
def run(self):
self.copy()
self.procDone.emit(True)
def copy(self):
source_size = os.stat(self.source).st_size
copied = 0
with open(self.source, "rb") as source, open(self.destination, "wb") as target:
while True:
chunk = source.read(1024)
if not chunk:
break
target.write(chunk)
copied += len(chunk)
self.procPartDone.emit(copied * 100 / source_size)
class MainWindow(QMainWindow):
def __init__(self, parent: object = None) -> None:
super().__init__(parent)
self.src = "/path/to/file.ext"
self.dest = "/path/to/file.ext"
self.btn = QPushButton(self)
self.btn.setText("Start copy")
self.btn.clicked.connect(self.run)
self.setCentralWidget(self.btn)
def run(self):
self.prog = ProgressDialog(self, self.src, self.dest)
self.prog.start()
def main():
app = QApplication(sys.argv)
window = MainWindow()
window.show()
sys.exit(app.exec_())
if __name__ == "__main__":
main()
Save this script in the 'main.py' file and execute the command
python3 main.py
The following uses tqdm to generate a progress bar while copying a single file.
from tqdm import tqdm
def copy_with_progress(src, dst):
size = os.path.getsize(src)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
with tqdm(total=size, unit='B', unit_scale=True, desc=f'Copying {src} to {dst}') as pbar:
while True:
chunk = fsrc.read(4096)
if not chunk:
break
fdst.write(chunk)
pbar.update(len(chunk))