We Keep Coding

Difficulties in pipes between Python subprocesses in Linux

The following script (which should take the output from p1 and pipe it to p2, and then output the result in the terminal) doesn't seem to work as expected.
Code as follows :
#!/binr/bin/python
# Script to lauch mosquitto_sub to monitor a mqtt topic -- SYNTAX : mosk topic
import sys
import subprocess
total = len(sys.argv)
cmdargs = str(sys.argv)
print ("The total numbers of args passed to the script: %d " % total)
print ("Args list: %s " % cmdargs)
# Pharsing args one by one
print ("Script name: %s" % str(sys.argv[0]))
print ("First argument: %s" % str(sys.argv[1]))
path = str(sys.argv[1])
print (path)
p1 = subprocess.Popen(['mosquitto_sub','-h','192.168.1.100','-t',path,'-v'], shell=False, stdout=subprocess.PIPE)
p2 = subprocess.Popen(['ts'], stdin=p1.stdout, shell=False, stdout=subprocess.PIPE)
for line in p2.stdout:
sys.stdout.write(line)
with an input as follows "./mosk2.py test/+" and whilst publishing MQTT via mosquitto on the relevant topics, I never get the expected output in the terminal

Solved - I ended up stepping neatly around the problem (cheating) as follows :
cmd = "mosquitto_sub -h 192.168.1.100 -v -t " + topic + " | ts"
print "The command which was executed is : " , cmd
def run_command(command):
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
while True:
output = process.stdout.readline()
if output == '' and process.poll() is not None:
break
if output:
print output.strip()
rc = process.poll()
return rc
run_command(cmd) #This is the lauch of the actual command

Python - check subprocess activity every n seconds

I'm trying to make python script (currently on windows) which will open some sub-processes (which will run infinitely) and script should periodically check do all of opened sub-processes still work correctly. So it should be done with while loop, I guess.
The sub-processes are about FFMPEG livestreaming.
The problem is when I do time.sleep(n) in my loop, because then every FFMPEG livestream stops, so I suppose time.sleep affect on all of child subprocesses.
I have no idea how to make it work.
Here is my python code:
import os, time, sys, datetime, smtplib, configparser, logging, subprocess, psutil
import subprocess
def forwardudpstream(channel_number, ip_input, ip_output):
try:
ffmpeg_command = 'ffmpeg -i udp://' + ip_input + ' -vcodec copy -acodec copy -f mpegts "udp://' + ip_output + '?pkt_size=1316"'
ffmpeg_output = subprocess.Popen(ffmpeg_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=False)
return str(ffmpeg_output.pid)
except:
print ("Exception!")
return '0'
while True:
configuration = 'config.ini'
channel_list_file = 'CHANNEL_LIST.conf'
pid_folder = "D:\\Forward_UDP_Stream\\pids\\"
channel_list = [line.rstrip('\n') for line in open(channel_list_file)]
for line in channel_list:
if not line.startswith('#') and ('|' in line):
channel_number, ip_input, ip_output = line.split('|')
print('----------')
print("Channel number = ", channel_number)
print("IP Input = ", ip_input)
print("IP Output = ", ip_output)
pid_file_found = False
print("Checking if pid file exists...")
for pidfile in os.listdir(pid_folder):
if pidfile.startswith(channel_number + '-'):
print("Pid file is found for this channel.")
pid_file_found = True
pid = int(pidfile.split('-')[1].split('.')[0])
print("PID = ", str(pid))
print("Checking if corresponding process is active...")
if not psutil.pid_exists(pid):
print("Process is not active.")
print("Removing old pid file.")
os.remove(pid_folder + pidfile)
print("Starting a new process...")
pid_filename = channel_number + '-' + forwardudpstream(channel_number, ip_input, ip_output) + '.pid'
pid_file = open(pid_folder + pid_filename, "a")
pid_file.write("Process is running.")
pid_file.close()
else:
print("Process is active!")
break
if pid_file_found == False:
print("Pid file is not found. Starting a new process and creating pid file...")
pid_filename = channel_number + '-' + forwardudpstream(channel_number, ip_input, ip_output) + '.pid'
pid_file = open(pid_folder + pid_filename, "a")
pid_file.write("Process is running.")
pid_file.close()
time.sleep(10)
Here is my CHANNEL_LIST.conf file example:
1|239.1.1.1:10000|239.1.1.2:10000
2|239.1.1.3:10000|239.1.1.4:10000
Perhaps there is some other solution to put waiting and sub-processes to work together. Does anyone have an idea?
UPDATE:
I finally make it work when I removed stdout=subprocess.PIPE part from the subprocess command.
Now it looks like this:
ffmpeg_output = subprocess.Popen(ffmpeg_command, stderr=subprocess.STDOUT, shell=False)
So now I'm confused why previous command was making a problem...?
Any explanation?

Python program doesn`t run in the background even after using & .. why?

I have a python program which curls to a machine and tries to run that file and if the file runs in 300secs its fine else it kills it.
import threading,datetime,signal,os
from threading import Thread
import logging,time
from subprocess import Popen,PIPE
import subprocess,os,errno
from xmlrpclib import ServerProxy
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
id = 1
p=Popen('hostname',stdout=PIPE,stderr=PIPE)
out,err=p.communicate()
global hostname
hostname=out.replace('\n','')
dir_path='/export/home/host/'+str(hostname)
curl_path='http://1.1.1.1:8080/host_exec/'+str(hostname)
def run_file(new_id,data):
data=data.split(',')
if len(data)>1:
session=Popen(str(data[0])+' '+str(data[1]),shell=True,stdout=PIPE,stderr=PIPE,stdin=PIPE)
else:
session=Popen(str(data[0]),shell=True,stdout=PIPE,stderr=PIPE,stdin=PIPE)
start = datetime.datetime.now()
print 'pid is ',session.pid
stdout1=''
stderr1=''
while session.poll() is None:
print 'sleeping'
time.sleep(10)
now = datetime.datetime.now()
#Kill the process if its still running and timeout period is over ( max 20 secs )
if (now - start).seconds > 10:
os.kill(session.pid, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
killed_flag=1
break;
#This condition is checked to see if the process finished executing and doesn`t need to be killed - success
elif session.poll() is not None:
print 'Process has finished'
stdout1=session.stdout.read()
stderr1=session.stdout.read()
break;
#If timeout is still remaining - wait for it to finish or get killed
else:
print 'still executing'
if killed_flag==1:
stdout1=' THE SCRIPT COULDN`T COMPLETE EXECUTION ON HOSTNAME '+str(hostname)+' AS IT TOOK MORE THAN 5 MINUTES TO FINISH AND HENCE GOT KILLED : TRY RERUNNING IT OR RUN IT MANUALLY ON THE BOX'
print stdout1
ops=stdout1 + stderr1
s = ServerProxy('http://paste/',allow_none=True)
page_url=s.pastes.newPaste("text",ops,None)
#print page_url
#print stdout
#Connect to db and update the table with the clob generated
session = Popen(['sqlplus','-S','abc/abc:1500'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
if flag==1:
#print flag
sql='update abc set temp_op = temp_op || \''+str(page_url)+',\', temp_db = temp_db || \''+str(hostname)+',\' where job_id=\''+str(new_id)+'\' ;'
if flag==2:
#print flag
sql='update abc config set output=\''+str(page_url)+'\', job_status=\'Completed\' where job_id=\''+str(new_id)+'\' ;'
session.stdin.write(sql)
session.stdin.flush()
stdout, stderr = session.communicate()
#print stdout
#print stderr
def do_this():
print "Running with Current Id : ", id
session=Popen('/export/home/curl '+str(curl_path)+'/filenames.txt',shell=True,stdout=PIPE,stderr=PIPE,stdin=PIPE)
stdout,stderr= session.communicate()
files_in_dir=stdout.split(' ')
if(len(files_in_dir)>1):
print files_in_dir
for file_name in files_in_dir:
if file_name:
file_list=file_name.split('_')
new_id=file_list[1]
if new_id>id:
session=Popen('/export/home/curl '+str(curl_path)+'/'+str(file_name),shell=True,stdout=PIPE,stderr=PIPE,stdin=PIPE)
file_content,stderr= session.communicate()
t = Thread(target=run_file,args=(new_id,file_content,))
t.start()
global id
id = new_id
else:
print 'No new file to process'
else:
print "EMPTY FOLDER"
while True:
do_this()
time.sleep(10)
But when I run it like
python abc.py &
It doesn`t run in the background. Why?
Also, When I do a CTRL+C or COMMAND+C to kill it .. it still keeps on running.

python multithreaded ssh app

I have an app I've scraped together to try and spawn 3 threads and ssh into a server simultaneously.
I wrote an obviously offensively coded application which I know is wrong which I am looking for some guidance for, to accomplish my initial end goal as mentioned above.
For the argument passing, I know I need to finesse it with something like cmd or cmd2 later on but for now that's not my primary concern.
I know that right now I'm spawning a subprocess and doing things serially. I look forward to your replies.
#!/usr/bin/python
import sys, os
import subprocess
if (len(sys.argv) > 1):
if( sys.argv[1] == 'start' ):
print "starting jboss"
arg = str(sys.argv[1])
elif( sys.argv[1] == 'stop' ):
print "stopping"
elif( sys.argv[1] == 'status' ):
print "getting status"
arg = str(sys.argv[1])
print arg
else:
print "start or stop?"
exit(1)
else:
print "unexpected error"
host = "10.24.14.10 "
command = "sudo /etc/init.d/jbossas " + arg
fullcommand = "ssh " + " " + host + " " + " " + command + " "+ arg
print "full command: ", fullcommand
process = subprocess.Popen(fullcommand, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output,stderr = process.communicate()
status = process.poll()
print output
host = "10.24.14.20 "
fullcommand = "ssh " + " " + host + " " + " " + command + " "+ arg
process = subprocess.Popen(fullcommand, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output,stderr = process.communicate()
status = process.poll()
print output
host = "10.30.1.1 "
fullcommand = "ssh " + " " + host + " " + " " + command + " "+ arg
process = subprocess.Popen(fullcommand, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output,stderr = process.communicate()
status = process.poll()
print output

How to count number of strings in a list

My needs check windows process number.
import sys
import os
import commands
ip = sys.argv[5]
val = sys.argv[1]
oid = "HOST-RESOURCES-MIB::hrSWRunName"
cmd = "snmpwalk -v 2c -c public %s %s" % (ip,oid)
(r_c,r_e) = commands.getstatusoutput(cmd)
if r_c != 0:
print "C - snmpwalk is Error."
else:
for i in r_e.split('\n'):
a = i.split(':')[-1].strip(' "')
print a
Result：
conhost.exe
conhost.exe
conhost.exe
conhost.exe
fdhost.exe
cmd.exe
fdhost.exe
I hope the result is. I do not know how to achieve it.
if sys.argv[1] <5:#count(conhost.exe)
print "critical -"
else:
print "OK - "
How the statistics of my results? conhost.exe 4 conhost.exe 1 conhost.exe 1

# replace here
else:
processes = r_e.split('\n')
programs = 0
for program in processes:
programFile = program.split(':')[-1].strip(' "')
# the first argument you pass to the program should be conhost.exe
if programFile == sys.argv[1]:
programs = programs + 1
if programs < 5 :#count(conhost.exe)
print "critical: running less than 5 times:", sys.argv[1]
else:
print "OK"
second version
# replace here
else:
processes = r_e.split('\n')
processes = map(lambda program: program.split(':')[-1].strip(' "'), processes)
if processes.count(sys.argv[1]) < 5 :#count(conhost.exe)
print "critical: running less than 5 times:", sys.argv[1]
else:
print "OK"