We Keep Coding

Python coding for opening and saving data to a file

I am having an issue getting the train function to work correctly in python. I can not modify the def function. I am at the point where I need to get the second file to read lines one at a time for PosList and i need to match the value of movieWordCount[z] in OpenPos. If the file is there, then I am good to incrment column 2 by one of t hat line (segmented by a space). If it is not, then I need the else to append it to the file end. It does not work. It does not append the values if it is missing and I am not sure if it will find the value if it is there. I have been stuck getting thsi to work for two days.
Here is my code segment I am working with:
with open("PosList") as OpenPos:
lines = OpenPos.readlines()
print lines
if movieWordCount[z] in lines:
print "found"
#Now use tokenize to split it apart by space and set to new array for me to call column2
else:
print "not found"
lines.append(movieWordCount[z] + " 1" + "\n")
Here is my full code:
#!/usr/bin/python
#Import Counter
import collections
from collections import Counter
#Was already here but pickle is used for data input and export
import math, os, pickle, re
class Bayes_Classifier:
def __init__(self, trainDirectory = "movie_reviews/"):
#If file listing exists skip to train
if os.path.isfile('iFileList'):
print "file found"
self.train()
#self.classify()
#If file listing does not exist skip to train
if not os.path.isfile('iFileList'):
print "no file"
newfile = 'iFileList'
tempList = set()
subDir = './movie_reviews'
for filenames in os.listdir(subDir):
my_sub_path = os.path.join(os.sep,subDir,filenames)
tempList.add(filenames)
self.save("filenames", "try3")
f = []
for fFileObj in os.walk("movie_reviews/"):
f.extend(fFileObj)
break
pickle.dump(f, open( "save.p", "wb" ))
self.save(f, "try4")
with open(newfile, 'wb') as fi:
pickle.dump(tempList, fi)
#print tempList
self.train()
#self.classify()
def train(self):
'''Trains the Naive Bayes Sentiment Classifier.'''
print "File ready for training"
#Open iFileList to use as input for opening movie files
x = 0
OpenIFileList = open('iFileList','r')
print "iFileList now Open"
#Loop through the file
for line in OpenIFileList:
#print "Ready to read lines"
#print "reading line " + line
if x > 4:
if x % 2 == 0:
#print line
s = line
if '-' in s:
comp = s.split("'")
#print comp[2]
print comp[1] #This is What you need for t he movie file
compValue1 = comp[1]
#Determine Positive/Negative.
#compType is the variable I am storing it to.
compType = compValue1.split("-",2)[1]
#print compType #Prints that middle value like 5 or 1
# This will do the work based on the value.
if compType == '5':
# print "you have a five" #Confirms the loop I am in.
#If file does not exists create it
if not os.path.exists('PosList'):
print "no file"
file('PosList', 'w').close()
#Open file that needs to be reviewed for word count
compValue2 = "movie_reviews/" + compValue1
print compValue2 #Prints the directory and file path
OpenMovieList = open(compValue2,'r')
for commentLine in OpenMovieList:
commentPositive = commentLine.split(" ")
commentPositiveCounter = Counter(commentPositive)
#print commentPositiveCounter # " Comment Pos goes here"
#if commentLine != '' or commentLine != ' ':
#Get first word, second word, ....
if commentLine and (not commentLine.isspace()):
movieWordCount = self.tokenize(commentLine)
y = len(movieWordCount) #determines length of string
print y
z = 0
#print movieWordCount[0] # Shows the zero position in the file.
while z < y:
print "position " + str(z) + " word is " + movieWordCount[z] # Shows the word we are at and position id
with open("PosList") as OpenPos:
lines = OpenPos.readlines()
print lines
if movieWordCount[z] in lines:
print "found"
else:
print "not found"
lines.append(movieWordCount)
z = z + 1
#Close the files
OpenMovieList.close()
OpenPos.close()
x += 1
#for line2 in OpenIFileList.readlines():
#for line in open('myfile','r').readlines():
#do_something(line)
#Save results
#Close the File List
OpenIFileList.close()
def loadFile(self, sFilename):
'''Given a file name, return the contents of the file as a string.'''
f = open(sFilename, "r")
sTxt = f.read()
f.close()
return sTxt
def save(self, dObj, sFilename):
'''Given an object and a file name, write the object to the file using pickle.'''
f = open(sFilename, "w")
p = pickle.Pickler(f)
p.dump(dObj)
f.close()
def load(self, sFilename):
'''Given a file name, load and return the object stored in the file.'''
f = open(sFilename, "r")
u = pickle.Unpickler(f)
dObj = u.load()
f.close()
return dObj
def tokenize(self, sText):
'''Given a string of text sText, returns a list of the individual tokens that
occur in that string (in order).'''
lTokens = []
sToken = ""
for c in sText:
if re.match("[a-zA-Z0-9]", str(c)) != None or c == "\'" or c == "_" or c == '-':
sToken += c
else:
if sToken != "":
lTokens.append(sToken)
sToken = ""
if c.strip() != "":
lTokens.append(str(c.strip()))
if sToken != "":
lTokens.append(sToken)
return lTokens

To open a file for writing, you can use
with open('PosList', 'w') as Open_Pos
As you are using the with form, you do not need to close the file; Python will do that for you at the end of the with-block.
So assuming that the way you add data to the lines variable is correct, you could remove the superfluous code OpenMovieList.close() and OpenPos.close(), and append 2 lines to your code:
with open("PosList") as OpenPos:
lines = OpenPos.readlines()
print lines
if movieWordCount[z] in lines:
print "found"
else:
print "not found"
lines.append(movieWordCount)
with open("PosList", "w") as OpenPos:
OpenPos.write(lines)

python: read file and split the data

I have a file config and the contents are separated by space " "
cat config
/home/user1 *.log,*.txt 30
/home/user2 *.trm,*.doc,*.jpeg 10
I want to read this file,parse each line and print each field from the each line.
Ex:-
Dir = /home/user1
Fileext = *.log,*.txt
days=30
I couldn't go further than the below..
def dir():
file = open('config','r+')
cont = file.readlines()
print "file contents are %s" % cont
for i in range(len(cont)):
j = cont[i].split(' ')
dir()
Any pointers how to move further?

Your code is fine, you are just missing the last step processing each element of the splitted string, try this:
def dir():
file = open('config','r+')
cont = file.readlines()
print "file contents are %s" % cont + '\n'
elements = []
for i in range(len(cont)):
rowElems = cont[i].split(' ')
elements.append({ 'dir' : rowElems[0], 'ext' : rowElems[1], 'days' : rowElems[2] })
for e in elements:
print "Dir = " + e['dir']
print "Fileext = " + e['ext']
print "days = " + e['days']
dir()
At the end of this code, you will have all the rows processed and stored in an array of dictionaries you can easily access later.

You can write a custom function to parse each line, and then use the map function to apply that function against each line in file.readlines():
def parseLine(line):
# function to split and parse each line,
# and return the formatted string
Dir, FileExt, Days = line.split(' ')[:3]
return 'Dir = {}\nFileext = {}\nDays = {}'.format(Dir, FileExt, Days)
def dir():
with open('config','r+') as file:
print 'file contents are\n' + '\n'.join(map(parseLine, file.readlines()))
Results:
>>> dir()
file contents are
Dir = /home/user1
Fileext = *.log,*.txt
Days = 30
Dir = /home/user2
Fileext = *.trm,*.doc,*.jpeg
Days = 10

Trying to get output of ffprobe into variable

I am trying to grab the ffprobe values from the video file into a variable that I can compare against others or move the value into a database. The question I have; Is there a better way of doing it than below?
I don't like the multiple if/elif/line.startswith statements and I am not sure of split is the best way of getting the ffprobe values?
#!/usr/bin/python
import os, sys, subprocess, shlex, re, fnmatch
from subprocess import call
videoDrop_dir="/mnt/VoigtKampff/Temp/_Jonatha/test_drop"
for r,d,f in os.walk(videoDrop_dir):
for files in f:
print "Files: %s" % files
if files.startswith(('._', '.')):
print "This file: %s is not valid" % files
elif files.endswith(('.mov', '.mpg', '.mp4', '.wmv', '.mxf')):
fpath = os.path.join(r, files)
def probe_file(fpath):
cmnd = ['ffprobe', '-show_format', '-show_streams', '-pretty', '-loglevel', 'quiet', fpath]
p = subprocess.Popen(cmnd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print files
out, err = p.communicate()
print "===============================OUTPUT START: %s ===============================" % files
print out
for line in out.split('\n'):
line = line.strip()
if line.startswith('codec_name='):
s = line
codec_name = s.split('codec_name=', 1)
print "Codec is: %s" % codec_name[1]
codec_1 = codec_name[1]
elif line.startswith('codec_type='):
s = line
codec_type = s.split('codec_type=', 1)
print "Codec type is: %s" % codec_type[1]
codec_type1 = codec_type[1]
elif line.startswith('codec_long_name=', 1):
s = line
codec_long_name = s.split('codec_long_name=', 1)
print "Codec long name: %s" % codec_long_name[1]
codec_long_name = codec_long_name[1]
elif line.startswith('format_long_name=', 1):
s = line
format_long_name = s.split('format_long_name=', 1)
print "Format long name: %s" % format_long_name[1]
format_long_name = format_long_name[1]
elif line.startswith('width='):
s = line
width = s.split('width=', 1)
print "Video pixel width is: %s" % width[1]
p_width = width[1]
elif line.startswith('height='):
s = line
height = s.split('height=', 1)
print "Video pixel height is: %s" % height[1]
p_height = height[1]
elif line.startswith('bit_rate='):
s = line
bit_rate = s.split('bit_rate=', 1)
print "Bit rate is: %s" % bit_rate[1]
bit_rate1 = bit_rate[1]
elif line.startswith('display_aspect_ratio=', 1):
s = line
display_aspect_ratio = s.split('display_aspect_ratio=', 1)
print "Display aspect ratio: %s" % display_aspect_ratio[1]
display_aspect_ratio1 = display_aspect_ratio[1]
elif line.startswith('avg_frame_rate=', 1):
s = line
avg_frame_rate = s.split('avg_frame_rate=', 1)
print "Average Frame Rate: %s" % avg_frame_rate[1]
avg_frame_rate1 = avg_frame_rate[1]
print "===============================OUTPUT FINISH: %s ===============================" % files
if err:
print "===============================ERROR: %s ===============================" % files
print err
probe_file(fpath)
else:
if not files.startswith(('.mov', '.mpg', '.mp4', '.wmv', '.mxf')):
print "This file: %s is not a valid video file" % files

This is a bit late, but hopefully it helps others searching for a similar answer.
import json, subprocess
# grab info about video_file
ffprobe_cmd = '/home/ubuntu/bin/ffprobe -v quiet -print_format json -show_format -show_streams - i ' + v + ' 2>&1'
# print ffprobe_cmd
s = subprocess.Popen(ffprobe_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ffprobe_out, err = s.communicate()
ffprobe_dict = json.loads(ffprobe_out)
From here, I re-use a common method, search_dict, which can be used like:
search_dict(ffprobe_dict, 'height')
def search_dict(my_dict, field):
"""Takes a dict with nested lists and dicts,
and searches all dicts for a key of the field
provided.
"""
fields_found = []
for key, value in my_dict.iteritems():
if key == field:
fields_found.append(value)
elif isinstance(value, dict):
results = search_dict(value, field)
for result in results:
fields_found.append(result)
elif isinstance(value, list):
for item in value:
if isinstance(item, dict):
more_results = search_dict(item, field)
for another_result in more_results:
fields_found.append(another_result)
return fields_found

You should ask this question on https://codereview.stackexchange.com/

Python - file management and processing multiple zip files

I have some zip files in a folder. I have a script to process them. The data that is to be written to a database is in a different file and its structure is as follows:
some_text;database;file_name
some_text2;database2;file_name2
....
What is the best way to process this file? Also, an error message should be reported if there is no matching zip file name in that file.
My current code:
filelist = glob.glob(os.path.join(rootdir, '*.zip'))
if filelist:
for file in filelist:
print "Working on file ", file
#get only file name without .zip for compare
aa = file.split(sl)
bb = aa[len(aa) -1]
cc = bb.split(".")
ime_sole = cc[0]
fle = codecs.open(rootdir + sl + 'portal_schools.txt',
'r',encoding="cp1250")
line = fle.readline()
# Read lines
for line in iter(fle):
#print line,
a,b,c = line.split(";")
if c == ime_sole:
print c
database = str(b)
#distdir = str(c)
else:
print "some text"
return
fle.close()
But this fails because it is being read line by line. If in the first line there is no match, the code stops. I need it to continue trough the file and then, after all is done, start with a new zip file.

I know my code is far from perfect. The problem was with else. I moved it to the end of the whole code. It was a novice mistake. I also inserted try-catch so if it fails on one zip file, the next one is still processed. Now, it looks something like this:
filelist = glob.glob(os.path.join(rootdir, '*.zip'))
if filelist:
for file in filelist:
try:
aa = file.split(sl)
#print "aa ",aa
bb = aa[len(aa) -1]
#print "bb ", bb
cc = bb.split(".")
#print "cc ", cc
ime_sole = cc[0]
#print "imesole ", ime_sole
fle = codecs.open(rootdir + sl + 'portal_schools.txt','r',encoding="cp1250")
#line = fle.readline()
data = []
for line in iter(fle):
line = line.replace("\r\n", "")
x = line.split(";")
data.append(x)
result = [element for element in data if element[2] == ime_sole]
fle.close()
#print result
if result:
database = result[0][1]
vnos_data = "Podatki za %s , se vpisujejo v bazo %s " % (ime_sole, database)
host ="####"
user="####"
password = "####"
iUrnik_tables = iUrnik_tables_fromzip.Tables(defdir,file,sl,host,database,user,password)
id_skripte =iUrnik_tables[0]
date_begin = iUrnik_tables[1]
date_end = iUrnik_tables[2]
iUrnik_all_fromzip.FileWork(defdir,file,sl,host,database,user,password)
iUrnik_itt_zip.Proces(defdir,file,sl,host,database,user,password,id_skripte,date_begin,date_end)
trenutek = datetime.datetime.now()
trenutek = trenutek.strftime("%Y%m%d%H%M")
newfilename = os.path.splitext(file)[0]
newfilename = newfilename +"_" + str(trenutek) + os.path.splitext(file)[1]
folder = defdir + sl + ime_sole + sl + "archive"
destination = folder + sl
novoimezipa= destination + newfilename.split(sl)[-1]
if not os.path.exists(folder):
os.makedirs(folder)
os.chdir(folder)
shutil.copy(file,destination)
old = destination + file.split(sl)[-1]
os.rename(old , novoimezipa )
os.remove(file)
else:
nothing :)
#return
except:
print sys.exc_info()
else:
vnos_nodata= u"V mapi %s ni podatkov za prenos" % (rootdir)
Logging(defdir, sl, vnos_nodata)
I know it is not perfect but it works :)

I'm trying to understand how to call a function from another function in the same class

I'm attempting to login to an Ubuntu server and search logs at several different paths with a function that already works locally (Python 2.7 - win7 machine). Below is the function of how I login and select the logs (also, the basis of my program is Python's cmd module):
def do_siteserver(self, line):
import paramiko
paramiko.util.log_to_file('c:\Python27\paramiko-wininst.log')
host = '10.5.48.65'
portNum = raw_input("\nEnter a port number: ")
while True:
try:
port = int(portNum)
break
except:
print "\nPort is not valid!!!"
break
transport = paramiko.Transport((host,port))
while True:
try:
passW = raw_input("\nEnter the SiteServer weekly password: ")
password = passW
username = 'gilbert'
nport = str(port)
print '\nEstablishing SFTP connection to: {}:{} ...'.format(host,port)
transport.connect(username = username, password = password)
sftp = paramiko.SFTPClient.from_transport(transport)
print 'Authorization Successful!!!'
log_names = ("/var/log/apache2/access.log",
"/var/log/apache2/error.log",
"/var/opt/smartmerch/log/merch_error.log",
"/var/opt/smartmerch/log/merch_event.log",
"/var/opt/smartmerch/log/server_sync.log")
#call search function here?
#for log_file, local_name in log_names.iteritems():
# sftp.get(log_file, local_name)
#sftp.close()
#transport.close()
break
except:
print "\nAuthorization Failed!!!"
Here is the function (in the same class) that I want to call:
def do_search(self, line):
print '\nCurrent dir: '+ os.getcwd()
userpath = raw_input("\nPlease enter a path to search (only enter folder name, eg. SQA\log): ")
directory = os.path.join("c:\\",userpath)
os.chdir(directory)
print "\n SEARCHES ARE CASE SENSITIVE"
print " "
line = "[1]Single File [2]Multiple Files [3]STATIC HEX"
col1 = line[0:14]
col2 = line[15:32]
col3 = line[33:46]
print " " + col1 + " " + col2 + " " + col3
print "\nCurrent Dir: " + os.getcwd()
searchType = raw_input("\nSelect type of search: ")
if searchType == '1':
logfile = raw_input("\nEnter filename to search (eg. name.log): ")
fiLe = open(logfile, "r")
userString = raw_input("\nEnter a string name to search: ")
for i,line in enumerate(fiLe.readlines()):
if userString in line:
print "String: " + userString
print "File: " + os.join(directory,logfile)
print "Line: " + str(i)
break
else:
print "%s NOT in %s" % (userString, logfile)
fiLe.close()
elif searchType =='2':
print "\nDirectory to be searched: " + directory
#directory = os.path.join("c:\\","SQA_log")
userstring = raw_input("Enter a string name to search: ")
userStrHEX = userstring.encode('hex')
userStrASCII = ''.join(str(ord(char)) for char in userstring)
regex = re.compile(r"(%s|%s|%s)" % ( re.escape( userstring ), re.escape( userStrHEX ), re.escape( userStrASCII )))
choice = raw_input("1: search with respect to whitespace. 2: search ignoring whitespace: ")
if choice == '1':
for root,dirname, files in os.walk(directory):
for file in files:
if file.endswith(".log") or file.endswith(".txt"):
f=open(os.path.join(root, file))
for i,line in enumerate(f.readlines()):
result = regex.search(line)
if result:
print "\nLine: " + str(i)
print "File: " + os.path.join(root,file)
print "String Type: " + result.group() + '\n'
f.close()
re.purge()
if choice == '2':
regex2 = re.compile(r'\s+')
for root,dirname, files in os.walk(directory):
for file in files:
if file.endswith(".log") or file.endswith(".txt"):
f=open(os.path.join(root, file))
for i,line in enumerate(f.readlines()):
result2 = regex.search(re.sub(regex2, '',line))
if result2:
print "\nLine: " + str(i)
print "File: " + os.path.join(root,file)
print "String Type: " + result2.group() + '\n'
f.close()
re.purge()
elif searchType =='3':
print "\nDirectory to be searched: " + directory
print " "
#directory = os.path.join("c:\\","SQA_log")
regex = re.compile(r'(?:3\d){6}')
for root,dirname, files in os.walk(directory):
for file in files:
if file.endswith(".log") or file.endswith(".txt"):
f=open(os.path.join(root,file))
for i, line in enumerate(f.readlines()):
searchedstr = regex.findall(line)
ln = str(i)
for word in searchedstr:
print "\nString found: " + word
print "Line: " + ln
print "File: " + os.path.join(root,file)
print " "
logfile = open('result3.log', 'w')
f.close()
re.purge()

self.do_search(linenumber)
That is all.

Methods are invoked via the object that has them.
self.do_search(...)