Is there any way to write both commands and their output to an external file?
Let's say I have a script outtest.py :
import random
from statistics import median, mean
d = [random.random()**2 for _ in range(1000)]
d_mean = round(mean(d), 2)
print(f'mean: {d_mean}')
d_median = round(median(d), 2)
print(f'median: {d_median}')
Now if I want to capture its output only I know I can just do:
python3 outtest.py > outtest.txt
However, this will only give me an outtest.txt file with e.g.:
mean: 0.34
median: 0.27
What I'm looking for is a way to get an output file like:
import random
from statistics import median, mean
d = [random.random()**2 for _ in range(1000)]
d_mean = round(mean(d), 2)
print(f'mean: {d_mean}')
>> mean: 0.34
d_median = round(median(d), 2)
print(f'median: {d_median}')
>> median: 0.27
Or some other format (markdown, whatever). Essentially, something like jupyter notebook or Rmarkdown but with using standard .py files.
Is there any easy way to achieve this?
Here's a script I just wrote which quite comprehensively captures printed output and prints it alongside the code, no matter how it's printed or how much is printed in one go. It uses the ast module to parse the Python source, executes the program one statement at a time (kind of as if it was fed to the REPL), then prints the output from each statement. Python 3.6+ (but easily modified for e.g. Python 2.x):
import ast
import sys
if len(sys.argv) < 2:
print(f"Usage: {sys.argv[0]} <script.py> [args...]")
exit(1)
# Replace stdout so we can mix program output and source code cleanly
real_stdout = sys.stdout
class FakeStdout:
''' A replacement for stdout that prefixes # to every line of output, so it can be mixed with code. '''
def __init__(self, file):
self.file = file
self.curline = ''
def _writerow(self, row):
self.file.write('# ')
self.file.write(row)
self.file.write('\n')
def write(self, text):
if not text:
return
rows = text.split('\n')
self.curline += rows.pop(0)
if not rows:
return
for row in rows:
self._writerow(self.curline)
self.curline = row
def flush(self):
if self.curline:
self._writerow(self.curline)
self.curline = ''
sys.stdout = FakeStdout(real_stdout)
class EndLineFinder(ast.NodeVisitor):
''' This class functions as a replacement for the somewhat unreliable end_lineno attribute.
It simply finds the largest line number among all child nodes. '''
def __init__(self):
self.max_lineno = 0
def generic_visit(self, node):
if hasattr(node, 'lineno'):
self.max_lineno = max(self.max_lineno, node.lineno)
ast.NodeVisitor.generic_visit(self, node)
# Pretend the script was called directly
del sys.argv[0]
# We'll walk each statement of the file and execute it separately.
# This way, we can place the output for each statement right after the statement itself.
filename = sys.argv[0]
source = open(filename, 'r').read()
lines = source.split('\n')
module = ast.parse(source, filename)
env = {'__name__': '__main__'}
prevline = 0
endfinder = EndLineFinder()
for stmt in module.body:
# note: end_lineno will be 1-indexed (but it's always used as an endpoint, so no off-by-one errors here)
endfinder.visit(stmt)
end_lineno = endfinder.max_lineno
for line in range(prevline, end_lineno):
print(lines[line], file=real_stdout)
prevline = end_lineno
# run a one-line "module" containing only this statement
exec(compile(ast.Module([stmt]), filename, 'exec'), env)
# flush any incomplete output (FakeStdout is "line-buffered")
sys.stdout.flush()
Here's a test script:
print(3); print(4)
print(5)
if 1:
print(6)
x = 3
for i in range(6):
print(x + i)
import sys
sys.stdout.write('I love Python')
import pprint
pprint.pprint({'a': 'b', 'c': 'd'}, width=5)
and the result:
print(3); print(4)
# 3
# 4
print(5)
# 5
if 1:
print(6)
# 6
x = 3
for i in range(6):
print(x + i)
# 3
# 4
# 5
# 6
# 7
# 8
import sys
sys.stdout.write('I love Python')
# I love Python
import pprint
pprint.pprint({'a': 'b', 'c': 'd'}, width=5)
# {'a': 'b',
# 'c': 'd'}
You can call the script and inspect its output. You'll have to make some assumptions though. The output is only from stdout, only from lines containing the string "print", and each print produces only one line of output. That being the case, an example command to run it:
> python writer.py script.py
And the script would look like this:
from sys import argv
from subprocess import run, PIPE
script = argv[1]
r = run('python ' + script, stdout=PIPE)
out = r.stdout.decode().split('\n')
with open(script, 'r') as f:
lines = f.readlines()
with open('out.txt', 'w') as f:
i = 0
for line in lines:
f.write(line)
if 'print' in line:
f.write('>>> ' + out[i])
i += 1
And the output:
import random
from statistics import median, mean
d = [random.random()**2 for _ in range(1000)]
d_mean = round(mean(d), 2)
print(f'mean: {d_mean}')
>>> mean: 0.33
d_median = round(median(d), 2)
print(f'median: {d_median}')
>>> median: 0.24
More complex cases with multiline output or other statements producing output, this won't work. I guess it would require some in depth introspection.
My answer might be similar to #Felix answer, but I'm trying to do it in a pythonic way.
Just grab the source code (mycode) and then execute it and capture the results, finally write them back to the output file.
Remember that I've used exec to the demonstration for the solution and you shouldn't use it in a production environment.
I also used this answer to capture stdout from exec.
import mycode
import inspect
import sys
from io import StringIO
import contextlib
source_code = inspect.getsource(mycode)
output_file = "output.py"
#contextlib.contextmanager
def stdoutIO(stdout=None):
old = sys.stdout
if stdout is None:
stdout = StringIO()
sys.stdout = stdout
yield stdout
sys.stdout = old
# execute code
with stdoutIO() as s:
exec(source_code)
# capture stdout
stdout = s.getvalue().splitlines()[::-1]
# write to file
with open(output_file, "w") as f:
for line in source_code.splitlines():
f.write(line)
f.write('\n')
if 'print' in line:
f.write(">> {}".format(stdout.pop()))
f.write('\n')
You can assign result to variable before print out and later on write those values to file
Here is example
import random
from statistics import median, mean
d = [random.random()**2 for _ in range(1000)]
d_mean = round(mean(d), 2)
foo = f'mean: {d_mean}'
print(foo)
d_median = round(median(d), 2)
bar = f'median: {d_median}'
print(bar)
# save to file
with open("outtest.txt", "a+") as file:
file.write(foo + "\n")
file.write(bar + "\n")
file.close()
argument "a+" mean open file and append content to file instead of overwrite if file was exist.
Related
Can I configure python to have matlab like print, so that when I just have a function
returnObject()
that it simply prints that object without me having to type print around it? I assume this is not easy, but something like if an object does not get bound by some other var it should get printed, so that this would work.
a = 5 #prints nothing
b = getObject() #prints nothing
a #prints 5
b #prints getObject()
getObject() #prints the object
If you use an ipython notebook individual cells work like this. But you can only view one object per cell by typing the objects name. To see multiple objects you'd need to call print, or use lots of cells.
You could write a script to modify the original script based on a set of rules that define what to print, then run the modified script.
A basic script to do this would be:
f = open('main.py', 'r')
p = open('modified.py', 'w')
p.write('def main(): \n')
for line in f:
temp = line
if len(temp) == 1:
temp = 'print(' + line + ')'
p.write('\t' + temp)
p.close()
from modified import main
main()
The script main.py would then look like this:
x = 236
x
output:
236
Idea is as follows: parse AST of Python code, replace every expression with call to print and content of expression as argument and then run the modified version. I'm not sure whether it works with every code, but you might try. Save it as matlab.py and run your code as python3 -m matlab file.py.
#!/usr/bin/env python3
import ast
import os
import sys
class PrintAdder(ast.NodeTransformer):
def add_print(self, node):
print_func = ast.Name("print", ast.Load())
print_call = ast.Call(print_func, [node.value], [])
print_statement = ast.Expr(print_call)
return print_statement
def visit_Expr(self, node):
if isinstance(node.value, ast.Call) and node.value.func.id == 'print':
return node
return self.add_print(node)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-')
args = parser.parse_args()
with args.infile as infile:
code = infile.read()
file_name = args.infile.name
tree = ast.parse(code, file_name, 'exec')
tree = PrintAdder().visit(tree)
tree = ast.fix_missing_locations(tree)
bytecode = compile(tree, file_name, 'exec')
exec(bytecode)
if __name__ == '__main__':
main()
Is it possible to make a method, which prints a next line of a code?
def print_next_line():
sth
import fxx
print 'XXX'
print_next_line()
file.split('/')
....
>>> 'XXX'
>>> 'file.split('/')'
I was thinking that It could be somewhere in the stack, but I'm not sure because it is next, not previous line.
Straight approach. I use inspect module to determine file and line where print_next_line was called. Later I read the file to find next string. You might want to add some error handling here (what if there is no next line in a file? and so on)
def print_next_line():
def get_line(f, lineno):
with open(f) as fp:
lines = fp.readlines()
return lines[lineno-1]
import inspect
callerframerecord = inspect.stack()[1]
frame = callerframerecord[0]
info = inspect.getframeinfo(frame)
line_ = info.lineno
file_ = info.filename
print get_line(file_, line_ + 1)
print 'XXX'
a = 1
print_next_line()
b = a*2
All you need is a profiling tool or just a debugger.
Use Python's inspect module:
import inspect
def print_next_line():
lineno = inspect.currentframe().f_back.f_lineno
with open(__file__) as f:
print(f.readlines()[lineno].rstrip())
Well you could open() your .py file and iterate to find specific line, then print it.
I have a following script in python.
for i in range(10000):
print i
Above piece of python code prints the value of i from 0 to 9999 on the console.
Now I would like to route the output of the script directly to an external file.
On linux, I can get it done using the following command
$ python python_script.py > python_out.txt
What is the equivalent command on windows 7, IDLE Python Shell and under PyLab ?
Also, Above script prints numbers from 0 to 9999. I would like to take a snapshot of the output,
i.e. I want to route the first 85 records / numbers to out1.txt OR
I want to route the numbers that are divisible by 5 to out2.txt
without changing the actual script.
Also provide me the Python documentation to find out more.
file1, file2 = "out1.txt", "out2.txt"
with open(file1,'w') as f1,open(file2,"w") as f2:
for i in range(10000):
if i < 85:
f1.write("{0}\n".format(i)) # write to out1.txt
if i%5==0:
f2.write("{0}\n".format(i)) #write to out2.txt
print i #write to stdout or python_out.txt in your case
and run this program as:
$python python_script.py > python_out.txt
It is a bit ugly code, but you won't have to change your script.
class A:
def __init__(self, filename, predicate=(lambda ln, v:True)):
self.filename = filename
self.lines = 0
def write(self, text):
if predicate(self.lines, text):
with open(self.filename, 'a') as f:
f.write(text)
self.lines += text.count('\n')
Usage:
import sys
def predicate(linenumber, text):
if text.isdigit():
if int(text) % 5:
return False
return True
sys.stdout = A('out1.txt', predicate)
for i in range(10000):
print i
You have to open a file and write in it, something like this.
f = open('write_to_this_file.txt', 'w')
for i in xrange(10000):
f.write(i + "\n")
Here is more info http://docs.python.org/2/tutorial/inputoutput.html#reading-and-writing-files
#!/usr/bin/env python`
import sys`
import binascii`
import string
sample = "foo.apples"
data_file = open("file1.py","r")
dat_file = open("file2.txt", "w")
for line in data_file:
if sample in line:
dat_file.writelines(line)
dat_file.close()`
When I do this I am able to find the string foo.apples. The problem is foo.apples is present in various lines in the python file. I want those lines which are inside a particular function. I need the lines within this def function.
Example:
def start():
foo.apples(a,b)
foo.apples(c,d) ... so on.
The following program finds defs and will append the sample string to the output file if the indentation remains within the def.
import re
sample = 'foo.apples'
data_file = open("file1.py", "r")
out_file = open("file2.txt", "w")
within_def = False
def_indent = 0
for line in data_file:
def_match = re.match(r'(\s*)def\s+start\s*\(', line) # EDIT: fixed regex
if def_match and not within_def:
within_def = True
def_indent = len(def_match.group(1))
elif within_def and re.match(r'\s{%s}\S' % def_indent, line):
within_def = False
if within_def and sample in line:
out_file.writelines(line)
out_file.close()
data_file.close()
Tested working on an example file1.py.
One, slightly off the beaten path approach to this would be to use the getsource method of the inspect module. Consider the following (theoretical) test1.py file:
class foo(object):
apples = 'granny_smith'
#classmethod
def new_apples(cls):
cls.apples = 'macintosh'
def start():
"""This is a pretty meaningless python function.
Attempts to run it will definitely result in an exception being thrown"""
print foo.apples
foo.apples = 3
[x for x in range(10)]
import bar as foo
Now you want to know about the start code:
import inspect
import test1 #assume it is somewhere that can be imported
print inspect.getsource(test1.start)
Ok, now we have only the source of that function. We can now parse through that:
for line in inspect.getsource(test1.start).splitlines():
if 'foo.apples' in line:
print line
There are some advantages here -- python does all the work of parsing out the function blocks when it imports the file. The downside though is that the file actually needs to be imported. Depending on where your files are coming from, this could introduce a HUGE security hole in your program -- You'll be running (potentially) "untrusted" code.
Here's a very non pythonic way, untested, but it should work.
sample = "foo.apples"
infile = open("file1.py", "r")
outfile = open("file2.txt", "w")
in_function = False
for line in infile.readlines():
if in_function:
if line[0] in(" ", "\t"):
if sample in line:
outfile.write(line)
else:
in_function = False
elif line.strip() == "def start():":
in_function = True
infile.close()
outfile.close()
I would suggest doing a function of this, which takes sample, the input file, and the function which we're supposed to search from as it's parameters. It would then return a list or tuple of all the lines that had the text in them.
def findFromFile(file, word, function):
in_function = False
matches = []
infile = open(file, "r")
for line in infile.readlines():
if in_function:
if line[0] in(" ", "\t"):
if word in line:
matches.append(line)
else:
in_function = False
elif line.strip() == "def %s():"%function:
in_function = True
infile.close()
return matches
I have a piece of code to read two files, convert them to sets, and then subtract one set from the other. I would like to use a string variable (installedPackages) for "a" instead of a file. I would also like to write to a variable for "c".
a = open("/home/user/packages1.txt")
b = open("/home/user/packages.txt")
c = open("/home/user/unique.txt", "w")
for line in set(a) - set(b):
c.write(line)
a.close()
b.close()
c.close()
I have tried the following and it does not work:
for line in set(installedPackages) - set(b):
I have tried to use StringIO, but I think I am using it improperly.
Here, finally, is how I have created installedPackages:
stdout, stderr = p.communicate()
installedPackages = re.sub('\n$', '', re.sub('install$', '', re.sub('\t', '', stdout), 0,re.MULTILINE))
Sample of packages.txt:
humanity-icon-theme
hunspell-en-us
hwdata
hyphen-en-us
ibus
ibus-gtk
ibus-gtk3
ibus-pinyin
ibus-pinyin-db-android
ibus-table
If you want to write to a string buffer file-like use StringIO
>>> from StringIO import StringIO
>>> installed_packages = StringIO()
>>> installed_packages.write('test')
>>> installed_packages.getvalue()
'test'
Something like the following?
Edit: after several iterations:
from subprocess import Popen, PIPE
DEBUG = True
if DEBUG:
def log(msg, data):
print(msg)
print(repr(data))
else:
def log(msg, data):
pass
def setFromFile(fname):
with open(fname) as inf:
return set(ln.strip() for ln in inf)
def setFromString(s):
return set(ln.strip() for ln in s.split("\n"))
def main():
# get list of installed packages
p = Popen(['dpkg', '--get-selections'], stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
installed_packages = setFromString(stdout)
# get list of expected packages
known_packages = setFromFile('/home/john/packages.txt')
# calculate the difference
unknown_packages = installed_packages - known_packages
unknown_packages_string = "\n".join(unknown_packages)
log("Installed packages:", installed_packages)
log("Known packages:", known_packages)
log("Unknown packages:", unknown_packages)
if __name__=="__main__":
main()
The set data type takes an iterable as a parameter, therefore if installedPackages a string with multiple items you need to split it by the delimiter. For example, the following code would split the string by all commas:
for line in set(installedPackages.split(',')) - set(b):
c.write(line)