I have the below command in my python script running a command line application which is currently running 'blind' in the background. If I run this command from a command line, mac or PC, I get a real-time readout of that application running, as it runs for 40 mins+ and reports various different info while it is running. Currently as I say it runs blind and gives me all the info at the end on a readout in python, but I want it to essentially open a command line and run so I can see all the info in real-time, but i can't find a way to do this. Here is my current code using a subprocess.Popen. Any other ways?
command1 = transporterLink + " -m verify -f " + indir1 + " -u " + username + " -p " + password + " -o " + indir1 + "\\VerifyLog.txt -s " + provider1 + " -v eXtreme"
process = subprocess.Popen(command1, stdout=subprocess.PIPE, shell=True)
I wrote about this a few years ago, but I don't think I titled the article very well:
http://www.blog.pythonlibrary.org/2010/06/05/python-running-ping-traceroute-and-more/
Basically you need to redirect stdout. I usually do something like this:
class RedirectText:
def __init__(self,aWxTextCtrl):
self.out=aWxTextCtrl
def write(self,string):
self.out.WriteText(string)
And then in my actual wx class, I do something like this in the init:
self.redir=RedirectText(log)
sys.stdout=self.redir
Then when you call subprocess, you would do something like this example:
def pingIP(self, ip):
proc = subprocess.Popen("ping %s" % ip, shell=True,
stdout=subprocess.PIPE)
print
while True:
line = proc.stdout.readline()
wx.Yield()
if line.strip() == "":
pass
else:
print line.strip()
if not line: break
proc.wait()
Note the wx.Yield() call. That allows wxPython to update when we print the line to stdout, which we have redirected to a text control.
Here is an example of sorts:
import subprocess
import sys
import wx
class RedirectText:
def __init__(self,aWxTextCtrl):
self.out=aWxTextCtrl
def write(self,string):
self.out.WriteText(string)
########################################################################
class MyFrame(wx.Frame):
""""""
#----------------------------------------------------------------------
def __init__(self):
"""Constructor"""
wx.Frame.__init__(self, None, title="Test")
panel = wx.Panel(self)
log = wx.TextCtrl(panel, wx.ID_ANY, size=(300,100),
style = wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL)
btn = wx.Button(panel, label="Run")
btn.Bind(wx.EVT_BUTTON, self.onRun)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(log, 1, wx.ALL|wx.EXPAND, 5)
sizer.Add(btn, 0, wx.ALL|wx.CENTER, 5)
panel.SetSizer(sizer)
self.redir=RedirectText(log)
sys.stdout=self.redir
#----------------------------------------------------------------------
def onRun(self, event):
""""""
command1 = transporterLink + " -m verify -f " + indir1 + " -u " + username + " -p " + password + " -o " + indir1 + "\\VerifyLog.txt -s " + provider1 + " -v eXtreme"
process = subprocess.Popen(command1, stdout=subprocess.PIPE, shell=True)
while True:
line = process.stdout.readline()
wx.Yield()
print line
if not line:
break
process.wait()
#----------------------------------------------------------------------
if __name__ == "__main__":
app = wx.App(False)
frame = MyFrame()
frame.Show()
app.MainLoop()
I go it to work by changing this section from Mikes answer, but only trouble is now when the application ends and the output finishes printing lines my python freezes. Any ideas why?
def onRun(self, event):
""""""
command1 = transporterLink + " -m verify -f " + indir1 + " -u " + username + " -p " + password + " -o " + indir1 + "\\VerifyLog.txt -s "
+ provider1 + " -v eXtreme"
master, slave = pty.openpty()
process = Popen(command1, shell=True, stdin=PIPE, stdout=slave, stderr=slave, close_fds=True)
stdout = os.fdopen(master)
while True:
line = stdout.readline()
wx.Yield()
print line.rstrip()
if not line:
break
process.wait()
Related
I'm trying to make python script (currently on windows) which will open some sub-processes (which will run infinitely) and script should periodically check do all of opened sub-processes still work correctly. So it should be done with while loop, I guess.
The sub-processes are about FFMPEG livestreaming.
The problem is when I do time.sleep(n) in my loop, because then every FFMPEG livestream stops, so I suppose time.sleep affect on all of child subprocesses.
I have no idea how to make it work.
Here is my python code:
import os, time, sys, datetime, smtplib, configparser, logging, subprocess, psutil
import subprocess
def forwardudpstream(channel_number, ip_input, ip_output):
try:
ffmpeg_command = 'ffmpeg -i udp://' + ip_input + ' -vcodec copy -acodec copy -f mpegts "udp://' + ip_output + '?pkt_size=1316"'
ffmpeg_output = subprocess.Popen(ffmpeg_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
return str(ffmpeg_output.pid)
except:
print ("Exception!")
return '0'
while True:
configuration = 'config.ini'
channel_list_file = 'CHANNEL_LIST.conf'
pid_folder = "D:\\Forward_UDP_Stream\\pids\\"
channel_list = [line.rstrip('\n') for line in open(channel_list_file)]
for line in channel_list:
if not line.startswith('#') and ('|' in line):
channel_number, ip_input, ip_output = line.split('|')
print('----------')
print("Channel number = ", channel_number)
print("IP Input = ", ip_input)
print("IP Output = ", ip_output)
pid_file_found = False
print("Checking if pid file exists...")
for pidfile in os.listdir(pid_folder):
if pidfile.startswith(channel_number + '-'):
print("Pid file is found for this channel.")
pid_file_found = True
pid = int(pidfile.split('-')[1].split('.')[0])
print("PID = ", str(pid))
print("Checking if corresponding process is active...")
if not psutil.pid_exists(pid):
print("Process is not active.")
print("Removing old pid file.")
os.remove(pid_folder + pidfile)
print("Starting a new process...")
pid_filename = channel_number + '-' + forwardudpstream(channel_number, ip_input, ip_output) + '.pid'
pid_file = open(pid_folder + pid_filename, "a")
pid_file.write("Process is running.")
pid_file.close()
else:
print("Process is active!")
break
if pid_file_found == False:
print("Pid file is not found. Starting a new process and creating pid file...")
pid_filename = channel_number + '-' + forwardudpstream(channel_number, ip_input, ip_output) + '.pid'
pid_file = open(pid_folder + pid_filename, "a")
pid_file.write("Process is running.")
pid_file.close()
time.sleep(10)
Here is my CHANNEL_LIST.conf file example:
1|239.1.1.1:10000|239.1.1.2:10000
2|239.1.1.3:10000|239.1.1.4:10000
Perhaps there is some other solution to put waiting and sub-processes to work together. Does anyone have an idea?
UPDATE:
I finally make it work when I removed stdout=subprocess.PIPE part from the subprocess command.
Now it looks like this:
ffmpeg_output = subprocess.Popen(ffmpeg_command, stderr=subprocess.STDOUT, shell=False)
So now I'm confused why previous command was making a problem...?
Any explanation?
The problem: I'm trying to dynamically build a python user-data script for amazon in a jenkins deploy script and pass it to an ASG to be executed at runtime. I pass my vars to the deploy script and then dynamically create the python script based on arguments.
I'm getting an unexpected string replacement error and I'm not entirely sure why handoff.sh is what passed the arguments from jenkins to the deploy script:
the error:
[deploy-and-configure-test] $ /bin/sh -xe /tmp/hudson8978997207867591628.sh
+ sh /var/lib/jenkins/workspace/deploy-and-configure-test/handoff.sh
Traceback (most recent call last):
File "/var/lib/jenkins/workspace/deploy-and-configure-test/asgBuilder.py", line 393, in <module>
''' % (str(repo), str(playbook),str(user_data_ins), str(in_user_data)))
TypeError: %u format: a number is required, not str
the dynamic portion of my deploy script:
in_user_data = args.in_user_data
playbook = args.playbook
repo = args.repo
user_data_ins = ('''export CLOUD_ENVIRONMENT=%s\n
export CLOUD_MONITOR_BUCKET=%s\n
export CLOUD_APP=%s\n
export CLOUD_STACK=%s\n
export CLOUD_CLUSTER=%s\n
export CLOUD_AUTO_SCALE_GROUP=%s\n
export CLOUD_LAUNCH_CONFIG=%s\n
export EC2_REGION=%s\n
export CLOUD_DEV_PHASE=%s\n
export CLOUD_REVISION=%s\n
export CLOUD_DOMAIN=%s\n
export SG_GROUP=%s\n''' % (cloud_environment,
cluster_monitor_bucket,
cluster_name,
cloud_stack,
cloud_cluster,
cloud_auto_scale_group,
cloud_launch_config,
provider_region,
cloud_dev_phase,
cloud_revision,
cloud_domain,
export_env_sg_name))
user_data_ins = ('''
#!/usr/bin/python
import os
import subprocess
import time
import uuid
def shell_command_execute(command):
p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
print output
return output
repo = "%s"
playbook = "%s"
echo_bash_profile = "echo %s >> ~/.bash_profile" % user_echo
shell_command_execute(echo_bash_profile)
var_user_data = "%s"
for varb in var_user_data.split('|'):
echo_bash_profile_passed = "echo " + varb + " >> ~/.bash_profile"
shell_command_execute(echo_bash_profile_passed)
command = 'git clone ' + repo
shell_command_execute(command)
folder = repo.split('/')[4].replace('.git','')
#https://github.com/test/test.git # replaced for security.
execute_playbook = ('ansible-playbook -i "localhost," -c local' + '/' + os.path.dirname(os.path.realpath(__file__)) + '/' + folder + '/' + playbook >> ansible.log')
print execute_playbook
shell_command_execute(execute_playbook)
''' % (str(repo), str(playbook),str(user_data_ins), str(in_user_data)))
text_file = open("user-data.py", "wa")
text_file.write(user_data_ins)
text_file.close()
lc_user_data = '${file("%s/user-data.py")}' %wd
updated still not working
user_data_ins = ('''export CLOUD_ENVIRONMENT=%s\n
export CLOUD_MONITOR_BUCKET=%s\n
export CLOUD_APP=%s\n
export CLOUD_STACK=%s\n
export CLOUD_CLUSTER=%s\n
export CLOUD_AUTO_SCALE_GROUP=%s\n
export CLOUD_LAUNCH_CONFIG=%s\n
export EC2_REGION=%s\n
export CLOUD_DEV_PHASE=%s\n
export CLOUD_REVISION=%s\n
export CLOUD_DOMAIN=%s\n
export SG_GROUP=%s\n''' % (cloud_environment,
cluster_monitor_bucket,
cluster_name,
cloud_stack,
cloud_cluster,
cloud_auto_scale_group,
cloud_launch_config,
provider_region,
cloud_dev_phase,
cloud_revision,
cloud_domain,
export_env_sg_name))
user_data_ins = ('''
#!/usr/bin/python
import os
import subprocess
import time
import uuid
def shell_command_execute(command):
p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
print output
return output
repo = "%s"
playbook = "%s"
echo_bash_profile = "echo %s >> ~/.bash_profile" % user_echo
shell_command_execute(echo_bash_profile)
var_user_data = "%s"
for varb in var_user_data.split('|'):
echo_bash_profile_passed = "echo " + varb + " >> ~/.bash_profile"
shell_command_execute(echo_bash_profile_passed)
command = 'git clone ' + repo
shell_command_execute(command)
folder = repo.split('/')[4].replace('.git','')
#https://github.com/zukeru/vision_provis.git
execute_playbook = ('ansible-playbook -i "localhost," -c local' + '/' + os.path.dirname(os.path.realpath(__file__)) + '/' + folder + '/' + playbook >> ansible.log')
print execute_playbook
shell_command_execute(execute_playbook)
''' % (str(repo), str(playbook),str(user_data_ins), str(in_user_data)))
text_file = open("user-data.py", "wa")
text_file.write(user_data_ins)
text_file.close()
lc_user_data = '${file("%s/user-data.py")}' %wd
#Grant Zukel
I would recommend doing the following.
In the last line change to
'''.format (str(repo), str(playbook),str(user_data_ins), str(in_user_data)))
And in your code change your first %s to {0} which would be str(repo) and every subsequent would be {1}... {2} etc
The problem is you have string replacement inside the string.
Whenever you have this, you need to have double percent:
echo_bash_profile = "echo %s >> ~/.bash_profile" %% user_echo
It is this line that is causing the error
'''bash_profile % user_echo'''
I would recommend using the string.format method if you are using python 2.6 or higher
Try this:
user_data_ins = ('''
#!/usr/bin/python
import os
import subprocess
import time
import uuid
def shell_command_execute(command):
p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
print output
return output
repo = "{0}"
playbook = "{1}"
echo_bash_profile = "echo {2} >> ~/.bash_profile" % user_echo
shell_command_execute(echo_bash_profile)
var_user_data = "{3}"
for varb in var_user_data.split('|'):
echo_bash_profile_passed = "echo " + varb + " >> ~/.bash_profile"
shell_command_execute(echo_bash_profile_passed)
command = 'git clone ' + repo
shell_command_execute(command)
folder = repo.split('/')[4].replace('.git','')
#https://github.com/zukeru/vision_provis.git
execute_playbook = ('ansible-playbook -i "localhost," -c local' + '/' + os.path.dirname(os.path.realpath(__file__)) + '/' + folder + '/' + playbook >> ansible.log')
print execute_playbook
shell_command_execute(execute_playbook)
'''.format(str(repo), str(playbook),str(user_data_ins), str(in_user_data)))
This line seems to be causing the issue:
echo_bash_profile = "echo %s >> ~/.bash_profile" % user_echo
Likely it sees the % user as %u.
Ok so #FirebladDan you were right i miseed one here is the working code:
user_data_ins = ('''
#!/usr/bin/python
import os
import subprocess
import time
import uuid
def shell_command_execute(command):
p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
print output
return output
repo = "%s"
playbook = "%s"
echo_bash_profile = "echo " + %s + " >> ~/.bash_profile"
shell_command_execute(echo_bash_profile)
var_user_data = "%s"
for varb in var_user_data.split('|'):
echo_bash_profile_passed = "echo " + varb + " >> ~/.bash_profile"
shell_command_execute(echo_bash_profile_passed)
command = 'git clone ' + repo
shell_command_execute(command)
folder = repo.split('/')[4].replace('.git','')
#https://github.com/zukeru/vision_provis.git
execute_playbook = ('ansible-playbook -i "localhost," -c local' + '/' + os.path.dirname(os.path.realpath(__file__)) + '/' + folder + '/' + playbook >> ansible.log')
print execute_playbook
shell_command_execute(execute_playbook)
''' % (str(repo), str(playbook),str(user_data_ins), str(in_user_data)))
text_file = open("user-data.py", "wa")
text_file.write(user_data_ins)
text_file.close()
lc_user_data = '${file("%s/user-data.py")}' %wd
I am running the below code which runs a command line application which runs for about 40 mins. While this is running my QUIT button is not accessible so I am unable to quit the running application. The below code and the button are both seated in their own def. Any ideas as to how I can get a working quit button while my application is running?
command1 = transporterLink + " -m verify -f " + indir1 + " -u " + username + " -p " + password + " -o " + logPath + " -s " + provider1 + " -v eXtreme"
master, slave = pty.openpty()
process = Popen(command1, shell=True, stdin=PIPE, stdout=slave, stderr=slave, close_fds=True)
stdout = os.fdopen(master)
global subject
subject = "Test"
while True:
wx.Yield()
line = stdout.readline()
line = line.rstrip()
print line
if "Returning 1" in line:
result1 = "Verify FAILED!"
subject = "FAILED! - "
self.sendEmail(self)
break
if "Returning 0" in line:
result1 = "Verify PASSED!"
subject = "PASSED! - "
self.sendEmail(self)
break
stdout.readline is blocking until there is something in stdout. You could use select module's poll
command1 = transporterLink + " -m verify -f " + indir1 + " -u " + username + " -p " + password + " -o " + logPath + " -s " + provider1 + " -v eXtreme"
master, slave = pty.openpty()
process = Popen(command1, shell=True, stdin=PIPE, stdout=master, stderr=slave, close_fds=True)
stdout = os.fdopen(master)
import select
q = select.poll()
q.register(stdout,select.POLLIN)
global subject
subject = "Test"
while True:
wx.Yield()
events = q.poll(0)
for fd, flags in events:
assert(fd == stdout.fileno())
line = stdout.readline()
print line
if "Returning 1" in line:
result1 = "Verify FAILED!"
subject = "FAILED! - "
self.sendEmail(self)
sys.exit(0)
if "Returning 0" in line:
result1 = "Verify PASSED!"
subject = "PASSED! - "
self.sendEmail(self)
sys.exit(0)
I am stuck in a while True loop which I can't seem to break, any suggestions please:
command1 = transporterLink + " -m verify -f " + indir1 + " -u " + username + " -p " + password + " -o " + indir1 + "/VerifyLog.txt -s " + provider1 + " -v eXtreme"
master, slave = pty.openpty()
process = Popen(command1, shell=True, stdin=PIPE, stdout=slave, stderr=slave, close_fds=True)
stdout = os.fdopen(master)
while True:
wx.Yield()
line = stdout.readline()
print line.rstrip()
if not line:
break
process.wait()
The simplest explanation is that you never get an empty line from stdout. Note that print line.rstrip() does not modify line; for example, if the last line ended with a newline, the loop would continue.
Sorted. I know that at the end of the last line it will return one of two strings so just needed to search for either of these two:
process = Popen(command1, shell=True, stdin=PIPE, stdout=slave, stderr=slave, close_fds=True)
stdout = os.fdopen(master)
while True:
wx.Yield()
line = stdout.readline()
line = line.rstrip()
print line
if "Returning 1" in line:
break
if "Returning 0" in line:
break
I have an app I've scraped together to try and spawn 3 threads and ssh into a server simultaneously.
I wrote an obviously offensively coded application which I know is wrong which I am looking for some guidance for, to accomplish my initial end goal as mentioned above.
For the argument passing, I know I need to finesse it with something like cmd or cmd2 later on but for now that's not my primary concern.
I know that right now I'm spawning a subprocess and doing things serially. I look forward to your replies.
#!/usr/bin/python
import sys, os
import subprocess
if (len(sys.argv) > 1):
if( sys.argv[1] == 'start' ):
print "starting jboss"
arg = str(sys.argv[1])
elif( sys.argv[1] == 'stop' ):
print "stopping"
elif( sys.argv[1] == 'status' ):
print "getting status"
arg = str(sys.argv[1])
print arg
else:
print "start or stop?"
exit(1)
else:
print "unexpected error"
host = "10.24.14.10 "
command = "sudo /etc/init.d/jbossas " + arg
fullcommand = "ssh " + " " + host + " " + " " + command + " "+ arg
print "full command: ", fullcommand
process = subprocess.Popen(fullcommand, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output,stderr = process.communicate()
status = process.poll()
print output
host = "10.24.14.20 "
fullcommand = "ssh " + " " + host + " " + " " + command + " "+ arg
process = subprocess.Popen(fullcommand, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output,stderr = process.communicate()
status = process.poll()
print output
host = "10.30.1.1 "
fullcommand = "ssh " + " " + host + " " + " " + command + " "+ arg
process = subprocess.Popen(fullcommand, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output,stderr = process.communicate()
status = process.poll()
print output