I'm struggled in trying to emulate this simple piece of bash:
$ cat /tmp/fifo.tub &
[1] 24027
$ gunzip -c /tmp/filedat.dat.gz > /tmp/fifo.tub
line 01
line 02
line 03
line 04
line 05
line 06
line 07
line 08
line 09
line 10
[1]+ Done cat /tmp/fifo.tub
Basically I tried this subprocess approach:
# -*- coding: utf-8 -*-
import os
import sys
import shlex
import pprint
import subprocess
def main():
fifo = '/tmp/fifo.tub'
filedat = '/tmp/filedat.dat.gz '
os.mkfifo(fifo,0777)
cat = "cat %s" % fifo
args_cat = shlex.split(cat)
pprint.pprint(args_cat)
cat = subprocess.Popen( args_cat,
close_fds=True,
preexec_fn=os.setsid)
print "PID cat: %s" % cat.pid
f = os.open(fifo ,os.O_WRONLY)
gunzip = 'gunzip -c %s' % (filedat)
args_gunzip = shlex.split(gunzip)
pprint.pprint(args_gunzip)
gunzip = subprocess.Popen( args_gunzip,
stdout = f,
close_fds=True,
preexec_fn=os.setsid)
print "PID gunzip: %s" % gunzip.pid
while not cat.poll():
# hangs for ever
pass
return True
if __name__=="__main__":
main()
cat process never ends.
Alternatively I tried to bypass the problem with threads but I get the same result.
import os
import sys
import shlex
import pprint
import subprocess
import threading
class Th(threading.Thread):
def __init__(self,cmd,stdout_h=None):
self.stdout = None
self.stderr = None
self.cmd = cmd
self.stdout_h = stdout_h
self.proceso = None
self.pid = None
threading.Thread.__init__(self)
def run(self):
if self.stdout_h:
self.proceso = subprocess.Popen(self.cmd,
shell=False,
close_fds=True,
stdout=self.stdout_h)
else:
self.proceso = subprocess.Popen( self.cmd,
close_fds=True,
shell=False)
print "PID: %d" % self.proceso.pid
def main():
fifo = '/tmp/fifo.tub'
filedat = '/tmp/filedat.dat.gz '
try:
os.unlink(fifo)
except:
pass
try:
os.mkfifo(fifo,0777)
except Exception , err:
print "Error '%s' tub %s." % (err,fifo)
sys.exit(5)
cat = "cat %s" % fifo
args_cat = shlex.split(cat)
pprint.pprint(args_cat)
cat = Th(cmd=args_cat)
cat.start()
try:
f = os.open(fifo ,os.O_WRONLY)
except Exception, err:
print "Error '%s' when open fifo %s " % (err,fifo)
sys.exit(5)
gunzip = 'gunzip -c %s ' % (filedat)
args_gunzip = shlex.split(gunzip)
pprint.pprint(args_gunzip)
gunzip = Th(cmd=args_gunzip,stdout_h=f)
gunzip.start()
gunzip.join()
cat.join()
while gunzip.proceso.poll() is None:
pass
if cat.proceso.poll() is None:
print "Why?"
cat.proceso.terminate()
return True
if __name__=="__main__":
main()
I'm clearly missing something, any help will be really welcome.
You are not closing the FIFO file descriptor so cat is just hanging there thinking there is more to come.
I think you can use the .wait() method as well to do the same thing as your while loop.
# -*- coding: utf-8 -*-
import os
import sys
import shlex
import pprint
import subprocess
def main():
fifo = '/tmp/fifo.tub'
filedat = '/tmp/filedat.dat.gz '
os.mkfifo(fifo,0777)
cat = "cat %s" % fifo
args_cat = shlex.split(cat)
pprint.pprint(args_cat)
cat = subprocess.Popen( args_cat,
close_fds=True,
preexec_fn=os.setsid)
print "PID cat: %s" % cat.pid
f = os.open(fifo ,os.O_WRONLY)
gunzip = 'gunzip -c %s' % (filedat)
args_gunzip = shlex.split(gunzip)
pprint.pprint(args_gunzip)
gunzip = subprocess.Popen( args_gunzip,
stdout = f,
close_fds=True,
preexec_fn=os.setsid)
print "PID gunzip: %s" % gunzip.pid
gunzip.wait()
print "gunzip finished"
os.close(f)
cat.wait()
print "cat finished"
return True
if __name__=="__main__":
main()
Related
The following script (which should take the output from p1 and pipe it to p2, and then output the result in the terminal) doesn't seem to work as expected.
Code as follows :
#!/binr/bin/python
# Script to lauch mosquitto_sub to monitor a mqtt topic -- SYNTAX : mosk topic
import sys
import subprocess
total = len(sys.argv)
cmdargs = str(sys.argv)
print ("The total numbers of args passed to the script: %d " % total)
print ("Args list: %s " % cmdargs)
# Pharsing args one by one
print ("Script name: %s" % str(sys.argv[0]))
print ("First argument: %s" % str(sys.argv[1]))
path = str(sys.argv[1])
print (path)
p1 = subprocess.Popen(['mosquitto_sub','-h','192.168.1.100','-t',path,'-v'], shell=False, stdout=subprocess.PIPE)
p2 = subprocess.Popen(['ts'], stdin=p1.stdout, shell=False, stdout=subprocess.PIPE)
for line in p2.stdout:
sys.stdout.write(line)
with an input as follows "./mosk2.py test/+" and whilst publishing MQTT via mosquitto on the relevant topics, I never get the expected output in the terminal
Solved - I ended up stepping neatly around the problem (cheating) as follows :
cmd = "mosquitto_sub -h 192.168.1.100 -v -t " + topic + " | ts"
print "The command which was executed is : " , cmd
def run_command(command):
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
while True:
output = process.stdout.readline()
if output == '' and process.poll() is not None:
break
if output:
print output.strip()
rc = process.poll()
return rc
run_command(cmd) #This is the lauch of the actual command
If I run "python /home/pi/temp/getTemp.py" from the terminal command line I get
"Error, serial port '' does not exist!" If I cd to the temp directory and run "python getTemp.py" it runs fine. Can anyone tell me why?
#!/usr/bin/env python
import os
import sys
import socket
import datetime
import subprocess
import signal
port = "/dev/ttyUSB0"
tlog = '-o%R,%.4C'
hlog = '-HID:%R,H:%h'
clog = '-OSensor %s C: %.2C'
def logStuff(data):
with open("/home/pi/temp/templog.txt", "a") as log_file:
log_file.write(data + '\n')
def main():
try:
output = subprocess.check_output(['/usr/bin/digitemp_DS9097U', '-q', '-a'])
for line in output.split('\n'):
if len(line) == 0:
logStuff("len line is 0")
continue
if 'Error' in line:
logStuff("error in output")
sys.exit()
line = line.replace('"','')
if line.count(',') == 1:
(romid, temp) = line.split(',')
poll = datetime.datetime.now().strftime("%I:%M:%S %p on %d-%B-%y")
content =(romid + "," + poll + "," + temp)
print content
return content
except subprocess.CalledProcessError, e:
print "digitemp error:\n", e.output
except Exception as e:
logStuff('main() error: %s' %e)
os.kill(os.getpid(), signal.SIGKILL)
if __name__ == "__main__":
main()
It probably cannot find the configuration file, which is normally stored in ~/.digitemprc when you run it with -i to initialize the network. If it was created in a different directory you need to always tell digitemp where to find it by passing -c
From a Python script, I need to call a PL->EN translation service. The translation requires 3 steps: tokenization, translation, detoknization
From Linux, I can achieve this using 3 processes by the following commands executed in mentioned order:
/home/nlp/opt/moses/scripts/tokenizer/tokenizer.perl -l pl < path_to_input.txt > path_to_output.tok.txt
/home/nlp/opt/moses/bin/moses -f /home/nlp/Downloads/TED/tuning/moses.tuned.ini.1 -drop-unknown -input-file path_to_output.tok.txt -th 8 > path_to_output.trans.txt
/home/nlp/opt/moses/scripts/tokenizer/detokenizer.perl -l en < path_to_output.trans.txt > path_to_output.final.txt
which translates the file path_to_input.txt and outputs to path_to_output.final.txt
I have made the following script for combining the 3 processes:
import shlex
import subprocess
from subprocess import STDOUT,PIPE
import os
import socket
class Translator:
#staticmethod
def pl_to_en(input_file, output_file):
# Tokenize
print("Tokenization started")
with open("tokenized.txt", "w+") as tokenizer_output:
with open(input_file) as tokenizer_input:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/tokenizer.perl - l pl"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdin=tokenizer_input, stdout=tokenizer_output)
p.wait()
print("Tokenization finished")
#Translate
print("Translation started")
with open("translated.txt", "w+") as translator_output:
cmd = "/home/nlp/opt/moses/bin/moses -f /home/nlp/Downloads/TED/tuning/moses.tuned.ini.1 -drop-unknown -input-file tokenized.txt -th 8"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdout=translator_output)
p.wait()
print("Translation finished")
# Detokenize
print("Detokenization started")
with open("translated.txt") as detokenizer_input:
with open("detokenized.txt", "w+") as detokenizer_output:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/detokenizer.perl -l en"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdin=detokenizer_input, stdout=detokenizer_output)
p.wait()
print("Detokenization finished")
translator = Translator()
translator.pl_to_en("some_input_file.txt", "some_output_file.txt")
But only the tokenization part works.
The translator just outputs an empty file translated.txt. When looking at the output in the terminal, it looks like the translator loads the file tokenized.txt correctly, and does a translation. The problem is just how I collect the output from that process.
I would try something like the following - sending the output of the translator process to the pipe, and making the input of the detokenizer the pipe instead of using the files.
import shlex
import subprocess
from subprocess import STDOUT,PIPE
import os
import socket
class Translator:
#staticmethod
def pl_to_en(input_file, output_file):
# Tokenize
print("Tokenization started")
with open("tokenized.txt", "w+") as tokenizer_output:
with open(input_file) as tokenizer_input:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/tokenizer.perl - l pl"
args = shlex.split(cmd)
p = subprocess.Popen(args, stdin=tokenizer_input, stdout=tokenizer_output)
p.wait()
print("Tokenization finished")
#Translate
print("Translation started")
cmd = "/home/nlp/opt/moses/bin/moses -f /home/nlp/Downloads/TED/tuning/moses.tuned.ini.1 -drop-unknown -input-file tokenized.txt -th 8"
args = shlex.split(cmd)
translate_p = subprocess.Popen(args, stdout=subprocess.PIPE)
translate_p.wait()
print("Translation finished")
# Detokenize
print("Detokenization started")
with open("detokenized.txt", "w+") as detokenizer_output:
cmd = "/home/nlp/opt/moses/scripts/tokenizer/detokenizer.perl -l en"
args = shlex.split(cmd)
detokenizer_p = subprocess.Popen(args, stdin=translate_p.stdout, stdout=detokenizer_output)
detokenizer_p.wait()
print("Detokenization finished")
translator = Translator()
translator.pl_to_en("some_input_file.txt", "some_output_file.txt")
For example:
#!/usr/bin/env python3
# cmd.py
import time
for i in range(10):
print("Count %d" % i)
time.sleep(1)
#!/usr/bin/env python3
import subprocess
# useCmd.py
p = subprocess.Popen(['./cmd.py'], stdout=subprocess.PIPE)
out, err = p.communicate()
out = out.decode()
print(out)
In useCmd.py I can print out the output of cmd.py, but only after it's finished outputting. How can I print out it in realtime and still have it stored in a string? (sort of like tee in bash.)
If you don't have to deal with stdin, you could avoid using communicate that is blocking, and read directly from the process stdout until your stdout ends:
p = subprocess.Popen(['python', 'cmd.py'], stdout=subprocess.PIPE)
# out, err = p.communicate()
while True:
line = p.stdout.readline()
if line != '':
print line,
else:
break
related
I am trying to execute two commands in parallel for 10 seconds using the following piece of code, but the whole process takes more than 10 seconds as you can see in the output. Would you please help me to better understand the reason and the best solution for this question.
stime = datetime.datetime.now()
print stime
commands = ("sudo /usr/local/bin/snort -v -u snort -g snort -c /usr/local/snort/etc/snort.conf -i eth0 &", "sudo gedit test")
for p in commands:
p = subprocess.Popen(shlex.split(p), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
class Alarm(Exception):
pass
def alarm_handler(signum, frame):
raise Alarm
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(10) #in seconds
try:
stdoutdata, stderrdata = p.communicate()
signal.alarm(0) #reset the alarm
except Alarm:
print 'Ooops, taking too long!!!!'
etime = datetime.datetime.now()
print etime
And the output:
2013-01-08 03:30:00.836412
Ooops, taking too long!!!!
2013-01-08 03:30:16.548519
I feel like a threading.Timer might be more appropriate:
from threading import Timer
from subprocess import Popen,PIPE
import shlex
import datetime
import sys
jobs = ['sleep 100','sleep 200']
timers = []
processes = []
print datetime.datetime.now()
for job in jobs:
p = Popen(shlex.split(job),stdout = PIPE)
t = Timer(10,lambda p=p: p.terminate())
t.start()
timers.append(t)
processes.append(p)
for t in timers:
t.join()
stdout,stderr = processes[0].communicate()
stdout,stderr = processes[1].communicate()
print datetime.datetime.now()
import multiprocessing
import subprocess
import shlex
import time
commands = ("echo -n HI-FIRST ", "echo -n HI-SECOND ")
def parallel():
p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
stdoutdata, stderrdata = p.communicate()
print stdoutdata + "\t" + time.ctime()
for cmd in commands:
p = multiprocessing.Process(target=parallel)
p.start()
Output:
$ python stack.py
HI-FIRST Fri Jan 11 08:47:18 2013
HI-SECOND Fri Jan 11 08:47:18 2013