We Keep Coding

Custom Popen.communicate method gives wrong output

Let's start by considering this code:
proc_stdin.py
import sys
if __name__ == '__main__':
for i, line in enumerate(sys.stdin):
sys.stdout.write(line)
test.py
import subprocess
def run_bad(target, input=None):
proc = subprocess.Popen(
target,
universal_newlines=True,
shell=True,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE if input else subprocess.DEVNULL,
stdout=subprocess.PIPE,
)
if input:
proc.stdin.write(input)
proc.stdin.flush()
proc.stdin.close()
lines = []
for line in iter(proc.stdout.readline, ""):
line = line.rstrip("\n")
lines.append(line)
proc.stdout.close()
ret_code = proc.wait()
return "\n".join(lines)
def run_good(target, input):
return subprocess.Popen(
target,
universal_newlines=True,
shell=True,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
).communicate(input=input)[0]
if __name__ == '__main__':
lst = [
"",
"token1",
"token1\n",
"token1\r\n",
"token1\n\n",
"token1\r\n\ntoken2",
"token1 token2",
"token1\ntoken2",
"token1\r\ntoken2",
"token1\n\ntoken2",
"token1\r\n\ntoken2",
"token1 \ntoken2\ntoken2\n"
]
cmd = "python proc_stdin.py"
for inp in lst:
a, b = run_bad(cmd, inp), run_good(cmd, inp)
if a != b:
print("Error: {} vs {}".format(repr(a), repr(b)))
else:
print("ok: {}".format(repr(a)))
Output:
ok: ''
ok: 'token1'
Error: 'token1' vs 'token1\n'
Error: 'token1\n' vs 'token1\n\n'
Error: 'token1\n' vs 'token1\n\n'
ok: 'token1\n\n\ntoken2'
ok: 'token1 token2'
ok: 'token1\ntoken2'
ok: 'token1\n\ntoken2'
ok: 'token1\n\ntoken2'
ok: 'token1\n\n\ntoken2'
Error: 'token1 \ntoken2\ntoken2' vs 'token1 \ntoken2\ntoken2\n'
My question is, why is the output of both run_bad & run_good not equal in all cases? How would you change the run_bad function so the output becomes equal than run_good?
You also may wonder, why are you not using directly Popen.communicate for this particular case or other helpers from subprocess module? Well, in the real world case I'm creating a plugin for SublimeText3 which is forcing me to stick to python3.3 (can't use many of the modern subprocess goodies) plus I'd like to inject some callbacks while reading the lines from stdout and that's something I can't do by using the Popen.communicate method (as far as I know).
Thanks in advance.

If you strip newlines from every line and then add them back between the lines, what happens to the last newline (if any)? (There’s no final, empty line after a final newline because your iter discards it.) This is why Python’s readline (or line iteration) function includes the newlines: they’re necessary to represent the end of the file accurately.

TypeError: Can't convert 'bytes' object to str implicitly using Python3.5

I am using Python 3.5 on the following code.
def raxml(DIR,cleaned,num_cores,seqtype):
assert cleaned.endswith(".aln-cln"),\
"raxml infile "+cleaned+" not ends with .aln-cln"
assert seqtype == "aa" or seqtype == "dna","Input data type: dna or aa"
assert len(read_fasta_file(DIR+cleaned)) >= 4,\
"less than 4 sequences in "+DIR+cleaned
clusterID = cleaned.split(".")[0]
tree = DIR+clusterID+".raxml.tre"
raw_tree = "RAxML_bestTree."+cleaned
model = "PROTCATWAG" if seqtype == "aa" else "GTRCAT"
if not os.path.exists(tree) and not os.path.exists(raw_tree):
# raxml crashes if input file starts with .
infasta = cleaned if DIR == "./" else DIR+cleaned
cmd = ["raxml","-T",str(num_cores),"-p","12345","-s",\
infasta,"-n",cleaned,"-m",model]
print (" ".join(cmd))
p = subprocess.Popen(cmd,stdout=subprocess.PIPE)
out = p.communicate()
assert p.returncode == 0,"Error raxml"+out[0]
try:
os.rename(raw_tree,tree)
os.remove("RAxML_info."+cleaned)
os.remove("RAxML_log."+cleaned)
os.remove("RAxML_parsimonyTree."+cleaned)
os.remove("RAxML_result."+cleaned)
os.remove(DIR+cleaned+".reduced")
except: pass # no need to worry about extra intermediate files
return tree
It runs and returns the following code:
"raxml_wrapper.py", line 30, in raxml
assert p.returncode == 0,"Error raxml"+out[0]
TypeError: Can't convert 'bytes' object to str implicitly
Initially, I tried the following:
p = subprocess.Popen(cmd,stdout=subprocess.PIPE)
p = p.decode('utf-8')
out = p.communicate()
assert p.returncode == 0,"Error raxml"+out[0]
That didn't fix the issue at all. I have looked at similar questions, but I cannot come up with a solution to this. I would appreciate some help on this.
Thanks!

p, a Popen object, doesn't have a .decode(...) member.
You need to actually decode the output
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
out, _ = p.communicate()
out = out.decode('utf-8')
assert p.returncode == 0, 'Error raxml' + out[0]
That said, this code can be improved to use subprocess.check_output:
# does roughly the same thing, you'll get `subprocess.CalledProcessError` instead of `AssertionError`
out = subprocess.check_output(cmd).decode('UTF-8')
Or if you happen to be using python3.6+
out = subprocess.check_output(cmd, encoding='UTF-8')

I do not know exactly what your p.communicate() method does, but it seems that it returns a byte object as a result. And this piece of code, cannot add this byte object to "Error raxml" str object:
assert p.returncode == 0,"Error raxml"+out[0]
Maybe you should try converting it to str as this:
assert p.returncode == 0,"Error raxml"+str(out[0])

trimming strings in python 3

I have this function to get a file type :
def get_file_type():
try:
cmd = ['/usr/bin/file', '/home/user']
p = Popen(cmd, stdout=PIPE).communicate()[0]
p = str(p).split(':')[1:]
if len(p) > 1:
' : '.join(p).strip().replace('\\n', '')
else:
p = p[0].strip().replace('\\n', '')
print(p)
except CalledProcessError:
print('unknown')
But it returns this : directory'
The ending apostrophe is not a typo, it is what bothers me. And I don't understand why (not that it bothers me.. ;) )
Thank you

The problem is that you're treating bytes as a string and you're using Python3. So what you're getting when you call str(p) looks like this:
"b'/home/user: directory\\n'"
You could fix this by doing p.decode().split instead of str(p).split

None in redirected stdout

I have this part of following code:
p = subprocess.Popen(['C:/Python27/python.exe', '-m', 'robot', '-d', logs_directory, input_file], stdout=subprocess.PIPE)
for line in iter(p.stdout.readline, ''):
output = sys.stdout.write(line)
#sys.stdout.write(line)
print "\n\n"
print "************************************************"
print output
print "\n\n"
print "************************************************"
print "\n\n"
But 'output' variable in console shows 'none'
************************************************
None
************************************************
What i am doing wrong here?

sys.stdout.write(line) returns the number of characters written on stdout.
You could write your loop like this:
output = ""
for line in iter(p.stdout.readline, ''):
output += line
sys.stdout.write(line)
imho you can also remove the sys.stdout.write(line) part. But I don't know what you are meant to do with it so...

Try output += line instead of output = sys.stdout.write(line)

You can also try function check_output.
Here is an example:
>>> subprocess.check_output(["echo", "Hello World!"])
'Hello World!\n'
You can try:
cmd = ['C:/Python27/python.exe', '-m', 'robot', '-d', logs_directory, input_file]
output = subprocess.check_output(cmd)
print output
In most of the cases, it can do the job simply.
For details, you may want to reference to subprocess.check_output

How to append column onto csv on the fly

I am trying to read the first column of my CSV, run a web-service using this column, take the output from this and append it to my CSV. I'd like to do this on a line-by-line basis.
Here is what I have come up with so far :
loadData = lambda f: np.genfromtxt(open(f,'r'), delimiter='\n')
with open('FinalCSV.csv','rb') as tsvin, open('FinalCSV.csv', 'a+b') as csvout:
tsvin = list(np.array(p.read_table('train.tsv'))[:,0])
writer = csv.writer(csvout)
count = 0
for row in csvout:
sep = '|'
row = row.split(sep, 1)[0]
cmd = subprocess.Popen("python GetJustAlexaRanking.py " + row ,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(output, err) = cmd.communicate()
exit_code = cmd.wait()
outlist = output.split('\r\n')
try:
outrank1 = outlist[1][outlist[1].index(':')+1:]
except ValueError:
outrank1 = "?"
row.append(str(outrank1).rstrip()) #writing,error here
print [str(outlist[0]).rstrip(), str(outrank1).rstrip()]
count+=1
However this is giving me the error that
Traceback (most recent call last):
File "File.py", line 28, in <module>
row.append(str(outrank1).rstrip()) #writing,error here
AttributeError: 'str' object has no attribute 'append'
How can I accomplish what I wish to do?
Edit :
loadData = lambda f: np.genfromtxt(open(f,'r'), delimiter='\n')
with open('FinalCSV.csv','rb') as tsvread, open('FinalCSVFin.csv', 'wb') as csvout:
tsvin = list(np.array(p.read_table('train.tsv'))[:,0])
writer = csv.writer(csvout)
count = 0
for row in tsvread:
sep = '|'
row = row.split(sep, 1)[0]
cmd = subprocess.Popen("python GetJustAlexaRanking.py " + row ,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(output, err) = cmd.communicate()
exit_code = cmd.wait()
outlist = output.split('\r\n')
try:
outrank1 = outlist[1][outlist[1].index(':')+1:]
except ValueError:
outrank1 = "?"
row = [row, outrank1.rstrip()]
writer.writerow(row)
print [str(outlist[0]).rstrip(), str(outrank1).rstrip()]
count+=1

Your row is not a list, but a string:
row = row.split(sep, 1)[0]
You then use that string in a subprocess command.
You'll need to make it a list again; instead of append, use:
row = [row, outrank1.rstrip()]
where outrank1 is always a string anyway, no need to call str() on it.
Note that if you are trying to both read from and write to the csvout file handle, you'll have to be very careful about your read-write position. You cannot just write to a file handle and hope to replace existing data for example. Best to use a separate, new file to write to and have that replace the old file location by moving one over the other.