I have tried to implement multithreading into a tool I've written, but I'm getting unexpected results.
Here is the working code:
#Import modules
import os
import time
import sys
#Params
print("___________Auto ShutApp___________\n")
print("Type the AppName then the time before app will shut")
apptokill = str(input("\nWhich App you'd like to Shut: "))
Time = int(input("\nHow long before App should stop(in minutes): "))
def timeS(arg): #arg = time in seconds
time.sleep(arg*60)
def killer(apptokill, Time):
timeS(Time)
os.system("pkill %s"%(apptokill))
killer(apptokill, Time)
This works fine, but I wanted to know when it would kill an app so I added a new function
def Timer():
time_start = time.time()
seconds = 0
minutes = 0
while True:
try:
sys.stdout.write("\rTimer have start since {minutes} Minutes {seconds} Seconds".format(minutes=minutes, seconds=seconds))
sys.stdout.flush()
time.sleep(1)
seconds = int(time.time() - time_start) - minutes * 60
if seconds >= 60:
minutes += 1
seconds = 0
except KeyboardInterrupt:
break
This one also works well, but if I place the function it will run either killer or Timer - not both.
I then tried some mutlithreading, adding the code below:
if __name__ == "__main__":
t1 = threading.Thread(target=killer, name='t1')
t2 = threading.Thread(target=Timer, name='t2')
t1.start()
t2.start()
t1.join()
t2.join()
but from this code I get this error:
Traceback (most recent call last):
File "/usr/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/usr/lib/python3.7/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
TypeError: killer() missing 2 required positional arguments: 'apptokill' and 'Time'
I assumed apptokill is saved in memory and so Time as well. I'm completely lost.
You have to provide arguments to killer
When calling by hand you are doing it like this
killer(apptokill, Time)
When starting killer in a thread it still needs to know what to use as arguments. The solution would look like this:
t1 = threading.Thread(target=killer, name='t1', args=(apptokill, Time))
Related
1)I am trying to do 8 parallel processes using concurrentFutures. The code is working as expected. But the results are printed only after all the processes are completed. I want the results printed as soon as they are available (not after when all the processes are done). How can I do that?
2)The "do_it" function is not printing anything in IDLE environment, only print commands in the main() function are printed on the IDLE window. But print commands in "do_it" are printing on online python compiler window (https://www.programiz.com/python-programming/online-compiler/). why?
I am using python3.9.5 on Windows OS.
import concurrent.futures
import time
def do_it(x):
print(f'sleeping for {x}')
start = time.perf_counter()
time.sleep(x)
end = time.perf_counter()
tTaken = round(end - start, 2)
return ['done sleeping', tTaken]
def main():
start = time.perf_counter()
with concurrent.futures.ProcessPoolExecutor() as executor:
delays = [1,2,3,4,5,6,7,8]
results = [executor.submit(do_it, x) for x in delays]
for f in concurrent.futures.as_completed(results):
[txt, duration] = f.result()
print(f'{txt} : time taken {duration}')
end = time.perf_counter()
tTaken = round(end - start, 2)
print(f'total time taken : {tTaken}')
if __name__ == '__main__':
main()
You only start as_completed after the executor has finished.
Try running while the executor still exists instead:
def main():
start = time.perf_counter()
print("Submitting jobs")
with concurrent.futures.ProcessPoolExecutor() as executor:
delays = [1, 2, 3, 4, 5, 6, 7, 8]
results = [executor.submit(do_it, x) for x in delays]
print("Running as_completed", flush=True)
for f in concurrent.futures.as_completed(results):
[txt, duration] = f.result()
print(f"{txt} : time taken {duration}", flush=True)
end = time.perf_counter()
tTaken = round(end - start, 2)
print(f"total time taken : {tTaken}")
That should result in something like:
Submitting jobs
sleeping for 1
sleeping for 2
sleeping for 3
Running as_completed
sleeping for 4
sleeping for 5
done sleeping : time taken 1.04
sleeping for 6
done sleeping : time taken 2.03
sleeping for 7
done sleeping : time taken 3.0
sleeping for 8
done sleeping : time taken 4.01
done sleeping : time taken 5.05
done sleeping : time taken 6.07
done sleeping : time taken 7.01
done sleeping : time taken 8.03
total time taken : 12.08
Edit
With regard to the missing output of the child processes, this has to do with how Python has to start new processes on ms-windows. As opposed to UNIX/POSIX systems, file handles are not inherited by child processes. So you won't see print output from a child process on ms-windows. At a guess, the online python is running on a UNIX/POSIX system.
I use python multiprocessing to compute some sort of scores on DNA sequences from a large file.
For that I write and use the script below.
I use a Linux machine with 48 cpu in python 3.8 environment.
Th code work fine, and terminate the work correctly and print the processing time at the end.
Problem: when I use the htop command, I find that all 48 processes are still alive.
I don't know why, and I don't know what to add to my script to avoid this.
import csv
import sys
import concurrent.futures
from itertools import combinations
import psutil
import time
nb_cpu = psutil.cpu_count(logical=False)
def fun_job(seq_1, seq_2): # seq_i : (id, string)
start = time.time()
score_dist = compute_score_dist(seq_1[1], seq_2[1])
end = time.time()
return seq_1[0], seq_2[0], score_dist, end - start # id seq1, id seq2, score, time
def help_fun_job(nested_pair):
return fun_job(nested_pair[0], nested_pair[1])
def compute_using_multi_processing(list_comb_ids, dict_ids_seqs):
start = time.perf_counter()
with concurrent.futures.ProcessPoolExecutor(max_workers=nb_cpu) as executor:
results = executor.map(help_fun_job,
[((pair_ids[0], dict_ids_seqs[pair_ids[0]]), (pair_ids[1], dict_ids_seqs[pair_ids[1]]))
for pair_ids in list_comb_ids])
save_results_to_csv(results)
finish = time.perf_counter()
proccessing_time = str(datetime.timedelta(seconds=round(finish - start, 2)))
print(f' Processing time Finished in {proccessing_time} hh:mm:ss')
def main():
print("nb_cpu in this machine : ", nb_cpu)
file_path = sys.argv[1]
dict_ids_seqs = get_dict_ids_seqs(file_path)
list_ids = list(dict_ids_seqs) # This will convert the dict_keys to a list
list_combined_ids = list(combinations(list_ids, 2))
compute_using_multi_processing(list_combined_ids, dict_ids_seqs)
if __name__ == '__main__':
main()
Thank you for your help.
Edit : add the complete code for fun_job (after #Booboo answer)
from Bio import Align
def fun_job(seq_1, seq_2): # seq_i : (id, string)
start = time.time()
aligner = Align.PairwiseAligner()
aligner.mode = 'global'
score_dist = aligner.score(seq_1[1],seq_2[1])
end = time.time()
return seq_1[0], seq_2[0], score_dist, end - start # id seq1, id seq2, score, time
When the with ... as executor: block exits, there is an implicit call to executor.shutdown(wait=True). This will wait for all pending futures to to be done executing "and the resources associated with the executor have been freed", which presumably includes terminating the processes in the pool (if possible?). Why your program terminates (or does it?) or at least you say all the futures have completed executing, while the processes have not terminated is a bit of a mystery. But you haven't provided the code for fun_job, so who can say why this is so?
One thing you might try is to switch to using the multiprocessing.pool.Pool class from the multiprocessing module. It supports a terminate method, which is implicitly called when its context manager with block exits, that explicitly attempts to terminate all processes in the pool:
#import concurrent.futures
import multiprocessing
... # etc.
def compute_using_multi_processing(list_comb_ids, dict_ids_seqs):
start = time.perf_counter()
with multiprocessing.Pool(processes=nb_cpu) as executor:
results = executor.map(help_fun_job,
[((pair_ids[0], dict_ids_seqs[pair_ids[0]]), (pair_ids[1], dict_ids_seqs[pair_ids[1]]))
for pair_ids in list_comb_ids])
save_results_to_csv(results)
finish = time.perf_counter()
proccessing_time = str(datetime.timedelta(seconds=round(finish - start, 2)))
print(f' Processing time Finished in {proccessing_time} hh:mm:ss')
I've written a basic wrapper around sched.scheduler, but I'm getting:
File "/usr/local/anaconda3/lib/python3.7/sched.py", line 149, in run
delayfunc(time - now)
OSError: [Errno 22] Invalid argument
What is the cause of this error? I'm following https://docs.python.org/3/library/sched.html
Code:
#!/usr/local/anaconda3/bin/python
from time import time, sleep
from datetime import datetime, timezone
from sched import scheduler
class Scheduler:
# specify one or t'other e.g. "2021-03-17 02:29:00"
def __init__(self, event_time_utc:str='', local_unix_timestamp_override:float=0):
self.event_local_timestamp = Scheduler.to_local_unix_timestamp(event_time_utc) \
if event_time_utc else local_unix_timestamp_override
# self.seconds_until_event = event_local_timestamp - time()
print(self.event_local_timestamp - time())
self.sch = scheduler()
def run(self, blocking=False):
self.sch.run(blocking)
#staticmethod
def to_local_unix_timestamp(time_utc: str):
dt_utc = datetime.strptime(time_utc, "%Y-%m-%d %H:%M:%S")
dt_local = dt_utc.replace(tzinfo=timezone.utc)
return dt_local.timestamp()
#staticmethod
def to_human_readable(unix_timestamp):
return datetime.fromtimestamp(unix_timestamp).strftime('%Y-%m-%d %H:%M:%S.%f')
def schedule(self, offset_from_event_time_s:float, callback):
print('Event scheduled for local unix timestamp:', self.event_local_timestamp + offset_from_event_time_s)
self.sch.enterabs(
time = self.event_local_timestamp + offset_from_event_time_s,
priority = 1,
action = callback
)
if __name__ == '__main__':
sch = Scheduler(local_unix_timestamp_override=time()+5)
print(f'Started at time: {Scheduler.to_human_readable(time())}')
def callback():
print(f'Callback at time: {Scheduler.to_human_readable(time())}')
sch.schedule(offset_from_event_time_s=1, callback=callback)
sch.run(blocking=True)
Output:
> ./scheduler.py
4.999997854232788
Started at time: 2021-03-17 11:00:03.424191
f 1615953609.424139
Traceback (most recent call last):
File "./scheduler.py", line 55, in <module>
sch.run(blocking=True)
File "./scheduler.py", line 20, in run
self.sch.run(blocking)
File "/usr/local/anaconda3/lib/python3.7/sched.py", line 149, in run
delayfunc(time - now)
OSError: [Errno 22] Invalid argument
The root of the problem is that YOU are assuming that the sched module is using time.time() for its timing function. That is a false assumption. It is using time.monotonic() by default, which returns the time since your program started. That is going to be a small number, like 6. When you try to delay until 1615644884.1231558, that makes it go crazy.
Change this:
self.sch = scheduler()
to this:
self.sch = scheduler(time)
I tried debugging the issue and found that the max length of floating point value allowed in time.sleep() is 8 (I couldn't find any resource on why only 8) but since you are using timestamp the value exceeds that. You can try following code on your machine and check that:
import time
# Will work fine
time.sleep(16156448)
# Will result in error
time.sleep(1615644884.1231558)
I'm looking to make a loop system (inspired by Marc Rebillet, check him out) and am struggling with getting 2 sounds to play over themselves.
I've started out using sounddevice, as it's the first audio module that I found, but I'm not sure if there's a different once that would work better.
import time
def record(duration): # records a few seconds of audio
print('recording in:')
print('3')
time.sleep(1)
print('2')
time.sleep(1)
print('1')
time.sleep(1)
print('recording')
record = sd.rec(int(duration * fs), samplerate=48000, channels=2)
sd.wait()
print('done\n\n')
return record
duration = 3.5 # seconds
recordOne = record(duration)
recordTwo = record(duration)
while True: # repeats both pieces of audio
sd.play(recordOne, fs)
sd.play(recordTwo, fs)
sd.wait()
This code ended up only playing the recordTwo, and doesn't layer them over eachother. Again, I'd like to be able to play multiple sounds at the same time. Thank you for any help!
Have you tried a multi-thread solution?
import time
import threading
def record(duration): # records a few seconds of audio
print('recording in:')
for t in range(3, 0, -1):
print(t)
time.sleep(1)
print('recording')
print("mock record process...")
start = time.time()
time.sleep(10)
print(f'done, {time.time() - start} seconds total.\n\n')
return record
duration = 3.5 # seconds
recordOne = record(duration)
recordTwo = record(duration)
def play(record_piece):
print("paly the sound")
time.sleep(duration)
while True: # repeats both pieces of audio
t1 = threading.Thread(target=play, args=(recordOne,))
t2 = threading.Thread(target=play, args=(recordTwo,))
t1.start()
t2.start()
t1.join()
t2.join()
or you can try to find some library that can combine the two audio tracks before you play them.
I read the doc of sounddevice, it provides a playback method, have you tried it?
playback
I have read about python sched (task scheduler), it works like a cron.
but I have a question :
lets say if I schedule a function to run after every 2 hours and in the mean time my system gets shut down, then I again restart the system...
Does the scheduler automatically start and run the function after 2 hours? Or do I have to start that again after shutting down the system?
Does sched work like a daemon?
Answer to all three questions is No.
sched is different from cron. It takes a generic timer or counter function and a delay function and lets you to schedule a function call after a particular time (an event as defined by your generic timer function).
It won't run after you close your program, unless you maintain state by writing to a file or db. This is complicated and using cron would be better.
sched works on events, but not on background. so, it not is not exactly a deamon, but you can deamonize it running the program in the background using OS facilities.
and if this is so :
will this also work even after system restart?
answer is : NO then how can turbogear scheduler can run using cronos in cron? scheduled events in turbogear will also be vanished after SYSTEM restart.
PLEASE CORRECT ME IF I AM WRONG.
import time
import sched
import datetime
import threading
import calendar
#from datetime import datetime
class test:
def __init__(self):
self.name = ''
def getSec(self):
now = datetime.datetime.now()
print "now - ", now
currentYear = now.year
currentMonth = now.month
currentDay = now.day
currentHour = now.hour
currentMinute = now.minute
currentSecond = now.second
currentMicroseconds = now.microsecond
command = "python runbackup.py"
print "command is - ", command
print "currentMinute - ", currentMinute
print "currentSecond - ", currentSecond
# current time
a = datetime.datetime(currentYear, currentMonth, currentDay, currentHour, currentMinute, currentSecond, currentMicroseconds)
last_date_of_current_month = calendar.monthrange(currentYear, currentMonth)[1]
print "last_date_of_current_month - ", last_date_of_current_month
b = datetime.datetime(currentYear, currentMonth, int(last_date_of_current_month), 23, 59, 59, 000000)
#b = datetime.datetime(currentYear, currentMonth, int(29), 18, 29, 00, 000000)
#print "date time of b is - %s %s " % (18, 29)
c = b-a
print "c is - ", c
time.sleep(1)
scheduler = sched.scheduler(time.time, time.sleep)
#scheduler.cancel(e1)
sec = c.seconds
print "second - ", sec
print "scheduler entered."
e1 = scheduler.enter(sec, 1, self.getSec, ())
t = threading.Thread(target=scheduler.run)
print "thread started."
print "======================================"
t.start()
#scheduler.cancel(e1)
#print "canceled."
return True
if __name__=='__main__' :
obj = test()
obj.getSec()