We Keep Coding

Syntax error when running my classes on terminal

I am trying to create a file using luigi that will take a csv and create output the splits the data up into different dataframes based on whether or not a particular column in the csv contains a particular string
This is the file I created:
import luigi
import pandas as pd
import os
class read_file(luigi.Task):
fileName = luigi.Parameter()
def run(self):
full_file = pd.read_csv(self.fileName)
return full_file[['anonymous_id','channel','context_campaign_content',
'context_campaign_medium','context_campaign_name',
'context_campaign_source','context_campaign_term',
'timestamp','user_id','context_page_url',
'properties_url','properties_search','context_page_title',
'properties_path','context_user_agent','properties_referrer','rank']]
def output(self):
return full_file
class blog_readers(luigi.Task):
def run(self):
read_blog = read_file.full_file[read_file.full_file['properties_url'].str.contains('blog',regex=False)]
return read_blog
def requires(self):
return read_file
def output(self):
return read_blog
class logged_in(luigi.Task):
def run(self):
logged_in = read_file.full_file[read_file.full_file['properties_url'].str.contains('login',regex=False]
return logged_in
def requires(self):
return read_file
def output(self):
return logged_in
if __name__ == '__main__':
luigi.run()
However when I run this file on terminal-
python cleanup.py --local-scheduler read_blog --fileName '/Users/**/Desktop/file.csv'
I encounter this error message
File "cleancopy.py", line 64
logged_in = read_file.full_file[read_file.full_file['properties_url'].str.contains('login',regex=False]
^
SyntaxError: invalid syntax
I am not sure what the syntax error is being caused by

python watchdog invoke class method when event happend

I am using watchdog to get notified when a certain file is changed,
evertything worked as long as i am using no classes like so
class MyEventHandler(PatternMatchingEventHandler):
def on_modified(self, event):
super(MyEventHandler, self).on_modified(event)
logging.info("File %s was just modified" % event.src_path)
readConfigFile()
def readConfigFile():
# set certain values here
def main():
while True:
# using values from configFile here
if __name__ == '__main__':
main()
# do something else
but if i put some classes in place i don't know how to call the method then setup is like this now
class MyEventHandler(PatternMatchingEventHandler):
def on_modified(self, event):
super(MyEventHandler, self).on_modified(event)
logging.info("File %s was just modified" % event.src_path)
readConfigFile() #<- i want to call this method in class Configuration
class Configuration:
def readConfigFile(self):
# set certain values here
def provideValues(self):
# get certain values here
class MainClass:
def __init__(self):
self.conf = Configuration()
event_handler = MyEventHandler(patterns=patterns)
def main(self):
while True:
# using values from configFile here
bla = self.conf.provideValues()
if __name__ == '__main__':
mc = MainClass()
mc.main()
# do something else

There are a log of ways to do it; here's one:
class MyEventHandler(PatternMatchingEventHandler):
def hook(self, thing):
self.thing = thing
def on_modified(self, event):
super(MyEventHandler, self).on_modified(event)
logging.info("File %s was just modified" % event.src_path)
self.thing.readConfigFile() #<- i want to call this method in class Configuration
class MainClass:
def __init__(self):
self.conf = Configuration()
event_handler = MyEventHandler(patterns=patterns)
event_handler.hook(self.conf)

python3 and pywin32 closing excel

I am having an issue closing excel after using Dispatch.
import openpyxl
import os
from win32com import client
class CTAutomation:
def __init__(self, file):
self.invoice = xl.load_workbook(os.getcwd() + "\Templates\ctrates.xlsx")
self.xlTemplate = xl.load_workbook(os.getcwd() + "\Templates\invoiceTemplate.xlsx")
self.vpc = xl.load_workbook(os.getcwd() + "\Templates\Vpc.xlsx")
self.file = file
def invoice_make(self):
self.xlApp = client.Dispatch("Excel.Application")
self.xlbook = self.xlApp.Workbooks.Open(os.getcwd() + '\TestFiles\\' + self.file)
self.ws = self.xlbook.Worksheets[0]
self.ws.Visible = 1
self.ws.ExportAsFixedFormat(0, os.getcwd() + "\complitedpdf\\" + self.file + ".pdf")
self.quit()
def quit(self):
self.xlbook.Close()
self.xlApp.Quit()
def xlformater(self):
return None
def main():
pwd = os.listdir(os.getcwd() + "\TestFiles")
for file in pwd:
CTAutomation(file.strip(".xlsx")).invoice_make()
if __name__ == "__main__":
main()
all works well till this part. i have found a few posts about this topic in the forum but i feel that im still missing something to close the app,
.xlsx and xls(Latest Versions) to pdf using python in example
some advice would be much appreciated .

Essentially it is your class object persisting in memory. Consider wrapping the process in a context manager using with(). And call the invoice_make() within the context.
Additionally, you had an incorrect Excel method by indexing workbook by zero with square brackets.
Finally, consider using os.path.join() to aviod back or forward slashes and use a try/except block to catch COM exceptions and properly release objects from memory.
import openpyxl as xl
import os
from win32com import client
cwd = os.getcwd()
class CTAutomation:
def __init__(self):
self.invoice = xl.load_workbook(os.path.join(cwd, "Templates", "ctrates.xlsx"))
self.xlTemplate = xl.load_workbook(os.path.join(cwd, "Templates", "invoiceTemplate.xlsx"))
self.vpc = xl.load_workbook(os.path.join(cwd, "Templates", "Vpc.xlsx"))
def invoice_make(self, file):
try:
self.xlApp = client.Dispatch("Excel.Application")
self.xlbook = self.xlApp.Workbooks.Open(os.path.join(cwd, "TestFiles", file))
self.ws = self.xlbook.Worksheets(1) # USE PARENTHESES (NOT BRACKETS AND NON-ZERO INDEX)
#self.ws.Visible = 1 # KEEP PROCESS IN BACKGROUND
self.ws.ExportAsFixedFormat(0, os.path.join(cwd, "complitedpdf", file.replace(".xlsx",".pdf")))
self.xlbook.Close(False)
self.xlApp.Quit()
except Exception as e:
print(e)
finally:
self.ws = None # RELEASE EXCEL OBJS FROM MEMORY
self.xlbook = None
self.xlApp = None
def xlformater(self):
return None
def __enter__(self):
return self # BOUND TO as IN with()
def __exit__(self, *err):
return None
def main():
pwd = os.listdir(os.path.join(cwd, "TestFiles"))
with CTAutomation() as obj: # CONTEXT MANAGER
for file in pwd:
print(file)
obj.invoice_make(file)
if __name__ == "__main__":
main()

How to check if a file is already opened (in the same process)

And I'd like to specifically achieve that with the try catch construct.
This related question suggests that I can do:
try:
open(fileName, 'wb+')
except:
print("File already opened!")
raise
However, it doesn't work me. I can open the same file multiple times without any problem:
fileObj1 = open(fileName, 'wb+')
fileObj2 = open(fileName, 'wb+')
Is it because I have Python 3.5? Or because I'm using Raspbian?
Thanks for the help!

You should open the same file but assign them to different variables, like so:
file_obj = open(filename, "wb+")
if not file_obj.closed:
print("File is already opened")
The .closed only checks if the file has been opened by the same Python process.

I would suggest using something like this
# Only works on Windows
def is_open(file_name):
if os.path.exists(file_name):
try:
os.rename(file_name, file_name) #can't rename an open file so an error will be thrown
return False
except:
return True
raise NameError
Edited to fit the OP's specific issues
class FileObject(object):
def __init__(self, file_name):
self.file_name = file_name
self.__file = None
self.__locked = False
#property
def file(self):
return self.__file
#property
def locked(self):
return self.__locked
def open(self, mode, lock=True):#any testing on file should go before the if statement such as os.path.exists()
#replace mode with *args if you want to pass multiple modes
if not self.locked:
self.__locked = lock
self.__file = open(self.file_name, mode)
return self.file
else:
print 'Cannot open file because it has an exclusive lock placed on it'
return None #do whatever you want to do if the file is already open here
def close(self):
if self.file != None:
self.__file.close()
self.__file = None
self.__locked = False
def unlock(self):
if self.file != None:
self.__locked = False

re-using the variable in a function in a class in python

I will try to explain the problem I am facing with a small piece of code:
class MyHandler(PatternMatchingEventHandler):
patterns = ["*.csv","*.processing", "*.transforming","*.loading"]
def process(self, event):
eventFileName = event.src_path
eventType = event.event_type
if eventType == 'moved':
eventFileName = event.dest_path
fileNameWithPath, fileExtension = os.path.splitext(eventFileName)
if fileExtension == '.processing':
# Here some function is called to do something, and then appends ".loading" to the file name
testVariable = 75.3
if fileExtension == '.loading':
print testVariable
def on_moved(self, event):
self.process(event)
def on_created(self, event):
self.process(event)
if __name__ == '__main__':
observer = Observer()
observer.schedule(MyHandler(), path='.')
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
When I try to do the above I am getting this error: global name 'testVariable' is not defined which kinda makes sense but how do I make the above work? I tried to define "testVariable" globally and initiated to 0 and tried using it the way I showed in above code but it did not work as well.
I also tried initiating testVariable as testVariable=0 inside the class (right after "patterns = ....." line but I got this error: local variable "testVariable" referenced before assignment pointing towards print testVariable. So that didnot work as well.

"(...) how do I make the above work?"
By defining testVariable outside your conditional statements. E.g. here:
def process(self, event):
eventFileName = event.src_path
testVariable = 0
...
This will make it available within the process function. If you want it to be available throughout the class, you can define it here:
class MyHandler(PatternMatchingEventHandler):
patterns = ["*.csv","*.processing", "*.transforming","*.loading"]
testVariable = 0
But then you have to access it via the self object within functions like so:
def process(self, event):
...
if fileExtension == '.processing':
# Here some function is called to do something, and then appends ".loading" to the file name
self.testVariable = 75.3

testVariable only exists if you have the extension ".processing". If it's ".loading", the program tries to print a variable that hasn't been made to exist.
If statements do not create a garbage collecting scope in Python, so you don't have to "declare" it outside, so long as somewhere in your if-tree, tesVariable gets a value.

def process(self, event):
def extension():
eventFileName = event.src_path
eventType = event.event_type
if eventType == 'moved':
eventFileName = event.dest_path
return os.path.splitext(eventFileName)[1]
if extension() == '.processing':
...
if extension() == '.loading':
...