I am making file conversation between two formats. If it takes too long, I want to skip that file and continue. Here is the script that I am using:
import time
from threading import Thread
from os.path import basename
import os, glob
# Custom exception class
class TimeoutException(Exception):
pass
# Custom alarm handler
class Alarm(Thread):
def __init__(self, timeout):
Thread.__init__(self)
self.timeout = timeout
self.setDaemon(True)
def run(self):
time.sleep(self.timeout)
raise TimeoutException
for folder, subfolders, filenames in os.walk(directory):
if any([filename.endswith('.scad') for filename in filenames]):
if filename.endswith('.scad'):
os.chdir(folder)
of = filename.replace('.scad', '.stl') # name of the outfile .stl
# This try/except loop ensures that
# you'll catch TimeoutException when it's sent.
try:
alarm = Alarm(5) # Create the timer. Once 5 seconds are over, a alarm will go off.
alarm.start() # Start the timer
os.system("\"convert.exe\" -o {} {}".format(of, filename))
except TimeOutException:
print("Timed out for file name: {}".format("filename"))
else:
del alarm # Reset the alarm
How could I make the script work in Windows? I referred sigalarm in Linux, but couldn't make it work in windows
Related
I have a simple Watchdog and Queue process to monitor files in a directory.
Code taken from https://camcairns.github.io/python/2017/09/06/python_watchdog_jobs_queue.html
import time
from watchdog.events import PatternMatchingEventHandler
from watchdog.observers import Observer
from queue import Queue
from threading import Thread
dir_path = "/data"
def process_queue(q):
while True:
if not q.empty():
event = q.get()
print("New event %s" % event)
time.sleep(5)
class FileWatchdog(PatternMatchingEventHandler):
def __init__(self, queue, patterns):
PatternMatchingEventHandler.__init__(self, patterns=patterns)
self.queue = queue
def process(self, event):
self.queue.put(event)
def on_created(self, event):
self.process(event)
if __name__ == '__main__':
watchdog_queue = Queue()
worker = Thread(target=process_queue, args=(watchdog_queue,))
worker.setDaemon(True)
worker.start()
event_handler = FileWatchdog(watchdog_queue, patterns="*.ini")
observer = Observer()
observer.schedule(event_handler, path=dir_path)
observer.start()
try:
while True:
time.sleep(2)
except KeyboardInterrupt:
observer.stop()
observer.join()
Once the process is running new files are processed correctly.
However if I restart the process and a file already exists in the directory it is ignored.
I have tried to create a dict to add to the queue
for file in os.listdir(dir_path):
if file.endswith(".ini"):
file_path = os.path.join(dir_path, file)
event = {'event_type' : 'on_created', 'is_directory' : 'False', 'src_path' : file_path}
watchdog_queue.put(event)
but it's expecting an object of type (class 'watchdog.events.FileCreatedEvent') and I can't work out how to create this.
Alternatively I can see in the Watchdog documentation (class watchdog.utils.dirsnapshot.DirectorySnapshot) but I cannot work out how to run this and add it to the queue.
Any suggestions on how I can add existing files to the queue on startup ?
This code should do what you are trying to achieve.
from watchdog.events import FileCreatedEvent
# Loop to get all files; dir_path is your lookup folder.
for file in os.listdir(dir_path):
filename = os.path.join(dir_path, file)
event = FileCreatedEvent(filename)
watchdog_queue.put(event)
I stumbled over the same problem and maybe this solution is for you too. At least on linux this works like a charm.
Add the "on_modified" method
class FileWatchdog(PatternMatchingEventHandler):
def __init__(self, queue, patterns):
PatternMatchingEventHandler.__init__(self, patterns=patterns)
self.queue = queue
...
def on_modified(self, event):
self.process(event)
Now after starting the observer, loop through all files in directory and "touch" them, so they will be "modified".
# Loop to get all files; dir_path is your lookup folder.
for file in os.listdir(dir_path):
filename = os.path.join(dir_path, file)
os.popen(f'touch {filename}')
No need to add special filters as your FileHandler will handle that.
I have a decorator written as such:
import threading
from time import sleep
from functools import wraps
import sys
import os
def repeat_periodically(f):
""" Repeat wrapped function every second """
#wraps(f)
def wrap(self, *args, **kwargs):
def wrap_helper(*args, **kwargs):
try:
threading.Timer(1.0, wrap_helper).start()
f(self)
except KeyboardInterrupt:
try:
sys.exit(1)
except:
os._exit(1)
wrap_helper()
return wrap
I'm not sure if it continues to open a new thread every single time it calls itself, but regardless, I'm unable to kill the process when I hit CTRL + C. I've also added the same try-except block in the function that I've decorated:
#repeat_periodically
def get_stats(self):
try:
# log some state information
except KeyboardInterrupt:
try:
sys.exit(1)
except:
os._exit(1)
My program just continues to run and all I see in the terminal is
^C <the stuff that I am logging>
<the stuff that I am logging>
<the stuff that I am logging>
In other words, it just keeps logging, even though I'm trying to kill it with CTRL + C.
Update:
I should mention that the above process is spun up from another thread:
tasks = [
{'target': f, 'args': (arg1)},
{'target': g},
]
for task in tasks:
t = threading.Thread(**task)
t.start()
Specifically it is the second task that spins up the Timer. However, if I set t.daemon = True, the process just runs once and exits. The first task uses watchdog. I've essentially used the example code from the watchdog documentation:
def watch_for_event_file(Event):
path = sys.argv[1] if len(sys.argv) > 1 else '.'
event_handler = LoggingCreateHandler(Event)
observer = Observer()
observer.schedule(event_handler, path)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
(Sorry for all the updates)
From the Thread documentation:
The entire Python program exits when no alive non-daemon threads are left.
So making your Timer threads as daemon threads should solve your problem. So replace:
threading.Timer(1.0, wrap_helper).start()
with:
t = threading.Timer(1.0, wrap_helper)
t.daemon = True
t.start()
I'm using the following code to monitor file access from a running job.
When the job is stopped my code receive a SIGINT.
As this job is very intensive, there's buffered IO and I can't unbuffered those writes, and I want a precise log.
So I tried to catch SIGINT and flush the file before shutting down my script I end up with :
RuntimeError: reentrant call inside <_io.BufferedWriter name=
As I understand from several articles I read, it's impossible to consistently use write/print/flush command as they are not thread safe in a signal handler.
My question is how can I ensure that my file is written properly before shutting down the script ?
Here's a simpler version of my script:
import signal
import sys
import os
import time
from time import strftime
import inotify.adapters
separator = ';'
jump = '\n'
logfile_pointer = open("path/to/log/file", 'w')
#Try to close nicely everything
def signal_handler(signal, frame):
logfile_pointer.flush()
logfile_pointer.close()
sys.exit(0)
#Register signal handler
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
eventHandler = inotify.adapters.InotifyTrees(["/folder/one","/folder/two"])
for event in eventHandler.event_gen():
if event is not None:
(_, type_names, path, filename) = event
try:
timestamp = '%.2f'%(time.time())
filepath=path +'/'+ filename
logfile_pointer.write ("{}{}{}{}{}{}{}{}".format(timestamp, separator, filepath , separator , type_names[0] ,separator, os.path.getsize(filepath) , jump )
except os.error as e:
pass
The typical approach here is to have the signal handler set a flag, and return without exiting. The main loop checks the flag and when it’s set, cleans up and exits.
In this particular instance this means you need to have the event producer yield regularly; with PyInotify you can do this by setting a short timeout. This would end up looking like
[...]
exit_requested = False
def signal_handler(signal, frame):
# Perhaps check which signal was received...
exit_requested = True
[...]
for event in eventHandler.event_gen(timeout_s = 1):
if exit_requested:
# Clean up and exit
if event:
...
When event_gen returns None because it timed out, inotify events which occur before the next call to event_gen will be queued and not lost: inotify events are consumed when they are read from the inotify file descriptor, and the event handler here keeps this open.
I had several issue to solve one being the way to stop my script from running as Python have some strange thread conception, here's my solution :
define a thread that will be the inotify watcher:
import os
import sys
import time
import signal
import argparse
import inotify.adapters
from time import strftime
from threading import Thread
from argparse import RawTextHelpFormatter
class EventMonitor(Thread):
separator = ';'
jump = '\n'
def __init__(self, folders, logfile):
Thread.__init__(self)
check_message=''
self.eventHandler = None
self.stop = False
self.logfile = open(logfile,'w',buffering=bufferSize)
self.line_count = 0
self.alive=True
self.eventHandler = inotify.adapters.InotifyTrees(folders)
def run(self):
while not self.stop:
for event in self.eventHandler.event_gen( timeout_s = 3 ):
try:
if event is not None:
(_, type_names, path, filename) = event
timestamp = '%.2f'%(time.time())
filepath=path +'/'+ filename
self.logfile.write ("{}{}{}{}{}{}{}{}".format(timestamp, self.separator, filepath , self.separator , type_names[0] ,self.separator, os.path.getsize(filepath) , self.jump ))
except os.error as e:
pass
for event in self.eventHandler.event_gen( timeout_s = 1 ):
try:
if event is not None:
(_, type_names, path, filename) = event
timestamp = '%.2f'%(time.time())
filepath=path +'/'+ filename
self.logfile.write ("{}{}{}{}{}{}{}{}".format(timestamp, self.separator, filepath , self.separator , type_names[0] ,self.separator, os.path.getsize(filepath) , self.jump ))
except os.error as e:
pass
self.logfile.flush()
self.logfile.close()
self.alive=False
def stopped(self):
if not self.stop:
self.stop = True
else:
print("Event Monitoring is already disabled")
def isAlive(self):
return self.alive
Then in my main script :
import os
import sys
import time
import signal
import argparse
import traceback
from time import strftime
from CPUMonitor import CPUMonitor
from EventMonitor import EventMonitor
from argparse import RawTextHelpFormatter
#define argument
parser = argparse.ArgumentParser(description='attache spies on multiple folders in argument and generate a csv log file containing a list of event on files.File is formatted like this: \ntimestamp;fullpath;event;size\n123456897.25;/path/file;IN_OPEN;0\n/123456899.25;path/file;IN_CLOSE;1234\n.....\nFor more info about inotify events => `man inotify`',formatter_class=RawTextHelpFormatter)
parser.add_argument("-l", "--log-folder",type=str, help="Destination folder for the logs. If no value /tmp is used", default='/tmp')
parser.add_argument("-e", "--event", help="enable file event watch ",action="store_true")
parser.add_argument( 'folders', metavar='folderpath', type=str ,help='a list of folder path to spy on if -e is not set this will be ignore.', nargs = '*', default=[os.getcwd()])
args = parser.parse_args()
#Try to close nicely everything
def signal_handler(signal, frame):
if CPU_thread is not None:
CPU_thread.stopped()
if Event_thread is not None:
Event_thread.stopped()
print('Kill signal receive.{}CPU and Event monitoring stopped.{}'.format(jump,jump))
sys.exit(0)
#Register signal handler
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
try:
#define variable
separator = ';'
jump = '\n'
logDest = ''
go = True
Event_logfile = None
Event_logfile_debug = None
Event_thread = None
jobname = ''
check_message=''
if not os.path.isdir(args.log_folder):
go=False
check_message = check_message + "/!\ Log folder {} is not a directory. Monitoring won't start{}".format(args.log_folder,jump)
elif not os.access(args.log_folder, os.W_OK | os.X_OK) :
go=False
check_message = check_message + "/!\ Log folder {} is not writable. Monitoring won't start{}".format(args.log_folder,jump)
else:
check_message = check_message + "Log folder is a proper directory and can be RW. {}".format(jump)
if not go :
print(check_message)
sys.exit(-2)
if go :
event_logfile = args.log_folder + '/Event_'+os.environ['JOB_ID'] + '_' + strftime("%Y-%m-%d_%H:%M:%S") + '-log.txt'
print('Event logfile: {}{}'.format(event_logfile,jump) )
print( 'Start monitoring of the event on: {} {}'.format( args.folders, jump ))
Event_thread = EventMonitor(args.folders, event_logfile)
Event_thread.start()
else:
print(("Error detected, monitoring hasn't started{}".format(jump)))
sys.exit(-4)
while Event_thread is not None and Event_thread.isAlive() :
time.sleep(5)
if Event_thread is not None:
Event_thread.join()
except Exception as error:
traceback.print_exc()
print(str(error))
sys.exit(-5)
In the thread as long as the thread is not stopped it will look for event and write them inside the file.
When stopped() is called the loop will time out after 3 seconds without event then I start the event loop a last time with a shorter timeout of 1 seconds, once all events are treated, the thread stops and isAlive() return False.
In the main program when SIGINT or SIGHUP is received it ask the thread to stop, and the python script only stops once the thread stops properly.
This code work both in Python 2.7.15 and 3.6.7 and above; however, keep in mind that this is a simplified version of my code and it might not work as is and might need some adjustment.
PS: thanks to Stephen answer which helps me a lot.
I have a function that downloads videos from specific URLs and I launch this function through a thread to avoid GUI freezing, but I want a function to stop or pause the download. How to do this?
Here is the code:
def download_videos(self):
ydl1 = youtube_dl.YoutubeDL(self.get_opts())
if self.get_Urls().__len__() > 0:
ydl1.download(self.get_Urls())
def downloadVideoThrd(self):
self.t1 = threading.Thread(target=self.download_videos())
self.t1.start()
You can use these two options. Use multiprocessing library instead threading or raise SystemExit exception in the subthread
Note: When i tested youtube-dl, it is resume downloading from where it is left. That's why when we start to download same url, youtube-dl will resume, but downloaded file need to be in the filesystem
Here is the first option, in this solution, we not use thread, we use subprocess because we can send any signal to the subprocess and handle this signal whatever we want.
import multiprocessing
import time
import ctypes,signal,os
import youtube_dl
class Stop_Download(Exception): # This is our own special Exception class
pass
def usr1_handler(signum,frame): # When signal comes, run this func
raise Stop_Download
def download_videos(resume_event):
signal.signal(signal.SIGUSR1,usr1_handler)
def download(link):
ydl1 = youtube_dl.YoutubeDL()
ydl1.download([link])
try:
download(link)
except Stop_Download: #catch this special exception and do what ever you want
print("Stop_Download exception")
resume_event.wait() # wait for event to resume or kill this process by sys.exit()
print("resuming")
download(link)
def downloadVideoThrd():
resume_event=multiprocessing.Event()
p1 = multiprocessing.Process(target=download_videos,args=(resume_event,))
p1.start()
print("mp start")
time.sleep(5)
os.kill(p1.pid,signal.SIGUSR1) # pause the download
time.sleep(5)
down_event.set() #resume downlaod
time.sleep(5)
os.kill(p1.pid,signal.SIGUSR2) # stop the download
downloadVideoThrd()
Here is the second solution, you can also check this for more detail about killing the thread. We will raise a SystemExit exception in the subthread by main thread. We can both stop or pause thread. To pause thread you can use threading.Event() class. To stop the thread(not process) you can use sys.exit()
import ctypes,time, threading
import youtube_dl,
def download_videos(self):
try:
ydl1 = youtube_dl.YoutubeDL(self.get_opts())
if self.get_Urls().__len__() > 0:
ydl1.download(self.get_Urls())
except SystemExit:
print("stopped")
#do whatever here
def downloadVideoThrd(self):
self.t1 = threading.Thread(target=self.download_videos)
self.t1.start()
time.sleep(4)
"""
raise SystemExit exception in self.t1 thread
"""
ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(self.t1.ident),
ctypes.py_object(SystemExit))
I am trying to make a simple function to download file in python
The code is something like
def download(url , dest):
urllib.urlretrieve(url, dest)
My issue is that if I want to cancel the download process in the middle of downloading how do I approach???
This function runs in the background of app and is triggered by a button. Now I am trying to trigger it off with another button.
The platform is XBMC.
A simple class to do the same as your download function:
import urllib
import threading
class Downloader:
def __init__(self):
self.stop_down = False
self.thread = None
def download(self, url, destination):
self.thread = threading.Thread(target=self.__down, args=(url, destination))
self.thread.start()
def __down(self, url, dest):
_continue = True
handler = urllib.urlopen(url)
self.fp = open(dest, "w")
while not self.stop_down and _continue:
data = handler.read(4096)
self.fp.write(data)
_continue = data
handler.close()
self.fp.close()
def cancel(self):
self.stop_down = True
So, when someone clicks the "Cancel" button you have to call the cancel() method.
Please note that this will not remove the partially downloaded file if you cancel it, but that should not be hard to achieve using os.unlink(), for example.
The following example script shows how to use it, starting the download of a ~20Mb file and cancelling it after 5 seconds:
import time
if __name__ == "__main__":
url = "http://ftp.postgresql.org/pub/source/v9.2.3/postgresql-9.2.3.tar.gz"
down = Downloader()
down.download(url, "file")
print "Download started..."
time.sleep(5)
down.cancel()
print "Download canceled"
If you are canceling by pressing CTRL+C, then you can use this built in exception and proceed with what you think the best move should be.
In this case, if I cancel in the middle of a download, I simply want that partial file to be deleted:
def download(url , dest):
try:
urllib.urlretrieve(url, dest)
except KeyboardInterrupt:
if os.path.exists(dest):
os.remove(dest)
except Exception, e:
raise