I need to run a python script at the time a sh file is called and during all the time this process is running.
basically it is a python spinner during installation
import sys
import time
do
def spinn():
print "processing...\\",
syms = ['\\', '|', '/', '-']
bs = '\b'
for _ in range(10):
for sym in syms:
sys.stdout.write("\b%s" % sym)
sys.stdout.flush()
time.sleep(.1)
spinn()
while
def installing():
import subprocess
subprocess.call(["sudo sh", "installer.sh"],shell=True)
installing()
is there a way to to this on python?
subprocess.call() waits for the subprocess to exit. Use subprocess.Popen instead. Then use .poll() periodically to check for when the process exits.
import itertools
import os
import subprocess
import sys
import time
def installing():
null = open(os.devnull, 'wb')
p = subprocess.Popen('echo blah && sleep 5', shell=True, stdout=null)
#p = subprocess.Popen('sudo sh installer.sh', shell=True, stdout=null)
return p, null
def spin(p_stdout):
p, stdout = p_stdout
syms = itertools.cycle(['\\', '|', '/', '-'])
sys.stdout.write('processing....')
sys.stdout.flush()
while p.poll() is None:
sys.stdout.write('\b'+next(syms))
sys.stdout.flush()
time.sleep(0.1)
p.wait()
stdout.close()
spin(installing())
Related
I want Python to kill this process if the output of the child process meets the criteria
For example this is a infinity loop while_file.py, it print 0 to 999 then looks like there's no response.
i = 0
while 1:
if i < 1000:
print i
i += 1
I want to check if the output of the child process is 999, then kill it.
import os
import signal
import subprocess
def run_cmd(cmd):
pro = subprocess.Popen(cmd, stdout=subprocess.PIPE,
shell=True, preexec_fn=os.setsid)
while True:
r = pro.stdout.read()
print r
if r == 999:
os.killpg(os.getpgid(pro.pid), signal.SIGTERM)
if __name__=='__main__':
print run_cmd('python while_file.py')
But it seems no response... why? Is it be blocked?
There are a couple of problems:
"read" function (from pro.stoud.read) reads the whole file (until EOF). Which in your case will not work because the stdout is never close, which means no EOF. You should use readline
when you are reading you are reading strings not numbers (so you should compare with "999" not with 999)
I would recommend making sure there is no buffering (can get nasty later if something stays in a buffer)
The changed codes:
import sys
i = 0
while True:
if i < 1000:
print i
sys.stdout.flush()
i += 1
and
import os
import signal
import subprocess
def run_cmd(cmd):
pro = subprocess.Popen(cmd, stdout=subprocess.PIPE,
shell=True, preexec_fn=os.setsid)
while True:
r = pro.stdout.readline()
r = r.strip()
print(r)
if r == "999":
os.killpg(os.getpgid(pro.pid), signal.SIGTERM)
print("Process killed")
break
if __name__=='__main__':
print run_cmd('PYTHONUNBUFFERED=1; python a.py')
I don't know if it is a good idea or not, but here is a way you can do it:
while_file.py
i = 0
while 1:
if i < 1000:
print(i)
i += 1
stackoverflow.py
import os
import signal
import subprocess
def run_cmd(cmd):
pro = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, preexec_fn=os.setsid)
for line in iter(pro.stdout.readline, ''):
r = line.rstrip()
print(r)
if r == b'999':
os.killpg(os.getpgid(pro.pid), signal.SIGTERM)
print("True !")
if __name__ == '__main__':
print(run_cmd('python while_file.py'))
From a Python script, I need to call a PL->EN translation service. The translation requires 3 steps: tokenization, translation, detoknization
From Linux, I can achieve this using 3 processes by the following commands executed in mentioned order:
/home/nlp/opt/moses/scripts/tokenizer/tokenizer.perl -l pl < path_to_input.txt > path_to_output.tok.txt
/home/nlp/opt/moses/bin/moses -f /home/nlp/Downloads/TED/tuning/moses.tuned.ini.1 -drop-unknown -input-file path_to_output.tok.txt -th 8 > path_to_output.trans.txt
/home/nlp/opt/moses/scripts/tokenizer/detokenizer.perl -l en < path_to_output.trans.txt > path_to_output.final.txt
which translates the file path_to_input.txt and outputs to path_to_output.final.txt
I have made the following script for combining the 3 processes:
import shlex
import subprocess
from subprocess import STDOUT,PIPE
import os
import socket
class Translator:
#staticmethod
def pl_to_en(input_file, output_file):
# Tokenize
print("Tokenization started")
with open("tokenized.txt", "w+") as tokenizer_output:
with open(input_file) as tokenizer_input:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/tokenizer.perl - l pl"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdin=tokenizer_input, stdout=tokenizer_output)
p.wait()
print("Tokenization finished")
#Translate
print("Translation started")
with open("translated.txt", "w+") as translator_output:
cmd = "/home/nlp/opt/moses/bin/moses -f /home/nlp/Downloads/TED/tuning/moses.tuned.ini.1 -drop-unknown -input-file tokenized.txt -th 8"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdout=translator_output)
p.wait()
print("Translation finished")
# Detokenize
print("Detokenization started")
with open("translated.txt") as detokenizer_input:
with open("detokenized.txt", "w+") as detokenizer_output:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/detokenizer.perl -l en"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdin=detokenizer_input, stdout=detokenizer_output)
p.wait()
print("Detokenization finished")
translator = Translator()
translator.pl_to_en("some_input_file.txt", "some_output_file.txt")
But only the tokenization part works.
The translator just outputs an empty file translated.txt. When looking at the output in the terminal, it looks like the translator loads the file tokenized.txt correctly, and does a translation. The problem is just how I collect the output from that process.
I would try something like the following - sending the output of the translator process to the pipe, and making the input of the detokenizer the pipe instead of using the files.
import shlex
import subprocess
from subprocess import STDOUT,PIPE
import os
import socket
class Translator:
#staticmethod
def pl_to_en(input_file, output_file):
# Tokenize
print("Tokenization started")
with open("tokenized.txt", "w+") as tokenizer_output:
with open(input_file) as tokenizer_input:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/tokenizer.perl - l pl"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdin=tokenizer_input, stdout=tokenizer_output)
p.wait()
print("Tokenization finished")
#Translate
print("Translation started")
cmd = "/home/nlp/opt/moses/bin/moses -f /home/nlp/Downloads/TED/tuning/moses.tuned.ini.1 -drop-unknown -input-file tokenized.txt -th 8"
args = shlex.split(cmd)
translate_p = subprocess.Popen(args, stdout=subprocess.PIPE)
translate_p.wait()
print("Translation finished")
# Detokenize
print("Detokenization started")
with open("detokenized.txt", "w+") as detokenizer_output:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/detokenizer.perl -l en"
args = shlex.split(cmd)
detokenizer_p = subprocess.Popen(args, stdin=translate_p.stdout, stdout=detokenizer_output)
detokenizer_p.wait()
print("Detokenization finished")
translator = Translator()
translator.pl_to_en("some_input_file.txt", "some_output_file.txt")
For example:
#!/usr/bin/env python3
# cmd.py
import time
for i in range(10):
print("Count %d" % i)
time.sleep(1)
#!/usr/bin/env python3
import subprocess
# useCmd.py
p = subprocess.Popen(['./cmd.py'], stdout=subprocess.PIPE)
out, err = p.communicate()
out = out.decode()
print(out)
In useCmd.py I can print out the output of cmd.py, but only after it's finished outputting. How can I print out it in realtime and still have it stored in a string? (sort of like tee in bash.)
If you don't have to deal with stdin, you could avoid using communicate that is blocking, and read directly from the process stdout until your stdout ends:
p = subprocess.Popen(['python', 'cmd.py'], stdout=subprocess.PIPE)
# out, err = p.communicate()
while True:
line = p.stdout.readline()
if line != '':
print line,
else:
break
related
At the end of a script, I would like to return the peak memory usage. After reading other questions, here is my script:
#!/usr/bin/env python
import sys, os, resource, platform
print platform.platform(), platform.python_version()
os.system("grep 'VmRSS' /proc/%s/status" % os.getpid())
print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
dat = [x for x in xrange(10000000)]
os.system("grep 'VmRSS' /proc/%s/status" % os.getpid())
print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
and here is what I get:
$ test.py
Linux-2.6.18-194.26.1.el5-x86_64-with-redhat-5.5-Final 2.7.2
VmRSS: 4472 kB
0
VmRSS: 322684 kB
0
Why is resource.getrusage always returning me 0?
The same thing happens interactively in a terminal. Can this be due to the way Python was specifically installed on my machine? (It's a computer cluster I'm using with others and managed by admins.)
Edit: same thing happen when I use subprocess; executing this script
#!/usr/bin/env python
import sys, os, resource, platform
from subprocess import Popen, PIPE
print platform.platform(), platform.python_version()
p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()], shell=False, stdout=PIPE)
print p.communicate()
print "resource:", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
dat = [x for x in xrange(10000000)]
p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()], shell=False, stdout=PIPE)
print p.communicate()
print "resource:", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
gives this:
$ test.py
Linux-2.6.18-194.26.1.el5-x86_64-with-redhat-5.5-Final 2.7.2
('VmRSS:\t 4940 kB\n', None)
resource: 0
('VmRSS:\t 323152 kB\n', None)
resource: 0
Here's a way to replace the ´os.system´ call
In [131]: from subprocess import Popen, PIPE
In [132]: p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()], shell=False, stdout=PIPE)
In [133]: p.communicate()
Out[133]: ('VmRSS:\t 340832 kB\n', None)
I also have no issue running the line you felt you have problems with:
In [134]: print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
340840
Edit
The rusage issue could well be a kernel dependent issue and simply not available on your red hat dist http://bytes.com/topic/python/answers/22489-getrusage
You could of course have a separate thread in your code looking at the current usage and storing throughout the execution of the code and store the highest value observed
Edit 2
Here's a full solution skipping resource and monitoring usages via Popen. The frequency of checking must of course be relevant but not frequent so that it eats all cpu.
#!/usr/bin/env python
import threading
import time
import re
import os
from subprocess import Popen, PIPE
maxUsage = 0
keepThreadRunning = True
def memWatch(freq=20):
global maxUsage
global keepThreadRunning
while keepThreadRunning:
p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()],
shell=False, stdout=PIPE)
curUsage = int(re.search(r'\d+', p.communicate()[0]).group())
if curUsage > maxUsage:
maxUsage = curUsage
time.sleep(1.0 / freq)
if __name__ == "__main__":
t = threading.Thread(target=memWatch)
t.start()
print maxUsage
[p for p in range(1000000)]
print maxUsage
[str(p) for p in range(1000000)]
print maxUsage
keepThreadRunning = False
t.join()
The memWatch function can be optimized by calculating the sleep time once, not reformatting the path to the process each loop and compiling the regular expression before entering the while loop. But in all I hope that was the functionality you sought.
I am trying to execute two commands in parallel for 10 seconds using the following piece of code, but the whole process takes more than 10 seconds as you can see in the output. Would you please help me to better understand the reason and the best solution for this question.
stime = datetime.datetime.now()
print stime
commands = ("sudo /usr/local/bin/snort -v -u snort -g snort -c /usr/local/snort/etc/snort.conf -i eth0 &", "sudo gedit test")
for p in commands:
p = subprocess.Popen(shlex.split(p), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
class Alarm(Exception):
pass
def alarm_handler(signum, frame):
raise Alarm
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(10) #in seconds
try:
stdoutdata, stderrdata = p.communicate()
signal.alarm(0) #reset the alarm
except Alarm:
print 'Ooops, taking too long!!!!'
etime = datetime.datetime.now()
print etime
And the output:
2013-01-08 03:30:00.836412
Ooops, taking too long!!!!
2013-01-08 03:30:16.548519
I feel like a threading.Timer might be more appropriate:
from threading import Timer
from subprocess import Popen,PIPE
import shlex
import datetime
import sys
jobs = ['sleep 100','sleep 200']
timers = []
processes = []
print datetime.datetime.now()
for job in jobs:
p = Popen(shlex.split(job),stdout = PIPE)
t = Timer(10,lambda p=p: p.terminate())
t.start()
timers.append(t)
processes.append(p)
for t in timers:
t.join()
stdout,stderr = processes[0].communicate()
stdout,stderr = processes[1].communicate()
print datetime.datetime.now()
import multiprocessing
import subprocess
import shlex
import time
commands = ("echo -n HI-FIRST ", "echo -n HI-SECOND ")
def parallel():
p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
stdoutdata, stderrdata = p.communicate()
print stdoutdata + "\t" + time.ctime()
for cmd in commands:
p = multiprocessing.Process(target=parallel)
p.start()
Output:
$ python stack.py
HI-FIRST Fri Jan 11 08:47:18 2013
HI-SECOND Fri Jan 11 08:47:18 2013