I'm attempting to use Map to reference a class function, but am having difficulty with formatting/ordering. I have heard that using map is sort of obsolete so I am definitely open to alternative solutions (for loops?) Thanks in advance.
lognames = [ "C:\Users\makker1\Desktop\logs\loga.txt",
"C:\Users\makker1\Desktop\logs\logb.txt",
"C:\Users\makker1\Desktop\logs\logc.txt" ]
class LogFile:
def __init__(self,filepath):
self.logfile = open(filepath, "r")
self.head = None
def __str__(self):
return "x=" + str(self.x) + "y="+str(self.y)
def readline (self):
if self.head != None:
self.head = self.logfile.readline()
def previewline (self):
if self.head == None:
self.head = self.logfile.readline()
def close (self):
self.logfile.close()
logs = map(LogFile(self,filepath).__init__(), lognames)
heads = map(lambda log: None, logs)
>>>
Traceback (most recent call last):
File "C:\Users\makker1\Desktop\mergesort-final.py", line 30, in <module>
logs = map(LogFile(self,filepath).__init__, lognames)
NameError: name 'self' is not defined
>>>
If any more info is needed, please let me know. I realize that there are tons of posts about this very problem and have sorted through many of them with no avail.
Here is a list comprehension answer. I like this better than map().
logs = [LogFile(fname) for fname in lognames]
You don't have to call __init__ explicitly. Try:
logs = map(LogFile, lognames)
Sometimes it helps to think of a class as being callable. You can think of a class as something like the following:
def LogFile(filepath):
class _LogFile:
def __init__(self, path):
...
return _LogFile(filepath)
Basically, a class can be thought of as something that you call to create an object instance. This isn't really true, but in many cases it will appear to be.
Related
How can I implement a static property or method-pair in Python using a sharable and accessible static attribute?
class StaticClass:
__static_attr: str
#classmethod
def set(cls, input__):
cls.__static_class = input__
#classmethod
def get(cls):
return cls.__static_attr
StaticClass.set("Hello!")
print(StaticClass.get())
Output:
C:\Users\pc\AppData\Local\Microsoft\WindowsApps\python3.7.exe C:/Users/pc/source/repos/main.py
Traceback (most recent call last):
File "C:/Users/pc/source/repos/main.py", line 15, in <module>
print(StaticClass.get())
File "C:/Users/pc/source/repos/main.py", line 11, in get
return cls.__static_attr
AttributeError: type object 'StaticClass' has no attribute '_StaticClass__static_attr'
Process finished with exit code 1
Edit: The above source code has a typo in set(). If that typo is fixed, the code runs perfectly.
However, in the case of my original source code, that attribute was a List[str]. In that case, the program only runs only if the attribute is initialized through a pair of square brackets [].
import os
from typing import List
class SearchFilesInTheDirectory:
__file_list: List[str] = []
#classmethod
def do_indexing_of_files(cls, path, file_extension):
for root, dirs, files in os.walk(path):
for file in files:
if file.endswith(file_extension):
cls.__file_list.append(os.path.join(root, file))
#classmethod
def get_files_list(cls):
return cls.__file_list
#classmethod
def printf(cls):
if cls.__file_list is not None:
for file in cls.__file_list:
print(file)
else:
print("directory is empty")
#classmethod
def write_to_file(cls, file_name):
if cls.__file_list is not None:
with open(file_name, 'w') as f:
for f_name in cls.__file_list:
f.write("%s\t%s\n" % f_name)
else:
print("directory is empty")
It would be a better idea is to initialize the class static variable, not just declare its type. This way you would be able to call get even before set, and have it return a default value:
class StaticClass:
__static_attr = ""
#classmethod
def set(cls, input__):
cls.__static_attr = input__
#classmethod
def get(cls):
return cls.__static_attr
StaticClass.set("Hello!")
print(StaticClass.get())
Also, it seems like your original code had a typo, as a result of which set method was assigning to a variable other than the one you were declaring (__static_class instead of __static_attr). Because of that, and because the static variable was not initialized, your get method failed.
I have the following base class:
class ClientRepo(Repository):
def __init__(self) -> None:
self.__clientList = []
def hasClientWithId(self, clientId):
for client in self.__clientList:
if client.getId() == clientId:
return True
return False
def addClient(self, client):
if type(client).__name__ == 'ClientDAO':
if not self.hasClientWithId(client.getId()):
client.setClientId(self.__maximumIndexInClientList() + 1)
self.__clientList.append(client)
else:
raise ObjectAlreadyInCollectionException
else:
raise TypeError
which basically only holds a list and can add a ClientDAO to it.
And the following, which derives from it:
class ClientFileRepository(ClientRepo):
def __init__(self, fileName) -> None:
super().__init__()
self.__fileName = fileName
self.__file = None
def hasClientWithId(self, clientId):
self.__loadRepo()
hasClientWithId = super().hasClientWithId(clientId)
super().clean()
return hasClientWithId
def addClient(self, client):
self.__loadRepo()
super().addClient(client)
self.__storeRepo()
super().clean()
def __loadFileReadMode(self):
self.__file = open(self.__fileName, "r")
def __loadFileWriteMode(self):
self.__file = open(self.__fileName, "w")
def __closeFile(self):
self.__file.close()
def __loadRepo(self):
self.__loadFileReadMode()
for line in self.__file:
splitLine = line.split()
clientToAdd = ClientDAO(splitLine[1])
clientToAdd.setClientId(int(splitLine[0]))
super().addClientWithId(clientToAdd)
self.__closeFile()
def __storeRepo(self):
self.__loadFileWriteMode()
self.__file.write("")
for client in super().getList():
self.__file.write(self.clientToString(client))
self.__closeFile()
def clientToString(self, clientDAO):
return str(clientDAO.getId()) + " " + clientDAO.getName() + "\n"
a class which should load the list from a file, call addClient from parent, and store the updated list in the file. The problem is that after child class loads the file in addClient, it calls the method in the parent, which calls hasClientWithId, from the child, again. But I want it to call hasClientWithId, from the parent, that is, the context it is in. Can I achieve that?
I can think of several ways to achieve your goal. I ranked them from worst to best
1. Exactly what you asked for
You wanted that ClientRepo.addClient calls ClientRepo.hasClientWithId instead of ClientFileRepository.hasClientWithId. It is possible to enforce that:
class ClientRepo(Repository):
def addClient(self, client):
if type(client).__name__ == 'ClientDAO':
if not ClientRepo.hasClientWithId(self, client.getId()):
client.setClientId(self.__maximumIndexInClientList() + 1)
self.__clientList.append(client)
else:
raise ObjectAlreadyInCollectionException
else:
raise TypeError
This is not a good approach, because it's unintuitive and breaks the principles of OOP. Any other programmer writing a subclass of ClientRepo that overrides hasClientWithId would expect that this will have an effect for every call to hasClientWithId even inside of addClient
2. Let ClientFileRepository decide which function to use
Add a variable
self.__isFileOpen = False
in ClientFileRepository.__init__, set it to True when you open the file and to False when you close the file. Then change the hasClientWithId within ClientFileRepository to
def hasClientWithId(self, clientId):
if not self.__isFileOpen:
self.__loadRepo()
result = super().hasClientWithId(clientId)
super().clean()
return result
else:
return super().hasClientWithId(clientId)
to avoid opening the same file again. This works, but it is pretty difficult to write new functions for this class, because you always need to be aware if the function call is a call from within your class or from somewhere else. Also this seems pretty inefficient, because you read and write the entire file, even when you only add one client.
3. Read the file only once and modify the underlying ClientRepo
class ClientFileRepository(ClientRepo):
def __init__(self, fileName) -> None:
super().__init__()
self.__fileName = fileName
self.__loadRepo()
# No hasClientWithId needed
def addClient(self, client):
super().addClient(client)
self.__storeRepo()
def __loadRepo(self):
with open(self.__filename) as file:
for line in file:
splitLine = line.split()
clientToAdd = ClientDAO(splitLine[1])
clientToAdd.setClientId(int(splitLine[0]))
super().addClientWithId(clientToAdd)
def __storeRepo(self):
with open(self.__filename, "w") as file:
file.write("")
for client in super().getList():
file.write(self.clientToString(client))
This obviously assumes that the file is not changed by someone else between calls to addClient and the program still overwrites the entire file for every addClient. If this is a problem for you it is best to be explicit and make loadRepo and storeRepo public. Then the programmer using this class can decide when loading and saving are necessary and useful. You can use context managers for this.
Extra: Read and save the file for every method
You can use function decorators to use solution 2 without writing the same code for every function:
import functools
def loadAndStore(function):
#functoools.wraps(function)
def wrappedFunction(self, *args, **kwargs):
if self.__isFileOpen:
return function(self, *args, **kwargs)
else:
self.__isFileOpen = True
self.__loadRepo()
try:
return function(self, *args, **kwargs)
except Exception as e: # Only catch expected exceptions
raise
finally:
self.__storeRepo()
self.clear() # some cleanup
self.__isFileOpen = False
return wrappedFunction
class ClientFileRepository(ClientRepo):
def __init__(self, fileName) -> None:
super().__init__()
self.__fileName = fileName
self.__isFileOpen = False
#loadAndStore
def hasClientWithId(self, clientId):
return super().hasClientWithId(clientId)
#loadAndStore
def addClient(self, client):
super().addClient(client)
def __loadRepo(self):
with open(self.__filename) as file:
for line in file:
splitLine = line.split()
clientToAdd = ClientDAO(splitLine[1])
clientToAdd.setClientId(int(splitLine[0]))
super().addClientWithId(clientToAdd)
def __storeRepo(self):
with open(self.__filename, "w") as file:
file.write("")
for client in super().getList():
file.write(self.clientToString(client))
Be careful here, using this is not very intuitive. For example self.__isFileOpen is defined in __init__, but none of the methods below directly use it. Instead its use is hidden in the loadAndStore decorator.
Some quick hints at the end:
type(client).__name__ == 'ClientDAO' is bad practice. Use isinstance(client, ClientDAO) to fully adopt OOP
If this is not part of a bigger project with given naming conventions use the python style guide
Using private variables like __fileName is generally considered unnecessary, just prefix the variable with one underscore to indicate "internal use". The same is true for functions.
I have the following simple class definition:
def apmSimUp(i):
return APMSim(i)
def simDown(sim):
sim.close()
class APMSimFixture(TestCase):
def setUp(self):
self.pool = multiprocessing.Pool()
self.sims = self.pool.map(
apmSimUp,
range(numCores)
)
def tearDown(self):
self.pool.map(
simDown,
self.sims
)
Where class APMSim is defined purely by plain simple python primitive types (string, list etc.) the only exception is a static member, which is a multiprocessing manager.list
However, when I try to execute this class, I got the following error information:
Error
Traceback (most recent call last):
File "/home/peng/git/datapassport/spookystuff/mav/pyspookystuff_test/mav/__init__.py", line 77, in setUp
range(numCores)
File "/usr/lib/python2.7/multiprocessing/pool.py", line 251, in map
return self.map_async(func, iterable, chunksize).get()
File "/usr/lib/python2.7/multiprocessing/pool.py", line 567, in get
raise self._value
MaybeEncodingError: Error sending result: '[<pyspookystuff.mav.sim.APMSim object at 0x7f643c4ca8d0>]'. Reason: 'TypeError("can't pickle thread.lock objects",)'
Which is strange as thread.lock cannot be found anywhere, I strictly avoid any multithreading component (as you can see, only multiprocessing component is used). And none of these component exist in my class, or only as static member, what should I do to make this class picklable?
BTW, is there a way to exclude a black sheep member from pickling? Like Java's #transient annotation?
Thanks a lot for any help!
UPDATE: The following is my full APMSim class, please see if you find anything that violates it picklability:
usedINums = mav.manager.list()
class APMSim(object):
global usedINums
#staticmethod
def nextINum():
port = mav.nextUnused(usedINums, range(0, 254))
return port
def __init__(self, iNum):
# type: (int) -> None
self.iNum = iNum
self.args = sitl_args + ['-I' + str(iNum)]
#staticmethod
def create():
index = APMSim.nextINum()
try:
result = APMSim(index)
return result
except Exception as ee:
usedINums.remove(index)
raise
#lazy
def _sitl(self):
sitl = SITL()
sitl.download('copter', '3.3')
sitl.launch(self.args, await_ready=True, restart=True)
print("launching .... ", sitl.p.pid)
return sitl
#lazy
def sitl(self):
self.setParamAndRelaunch('SYSID_THISMAV', self.iNum + 1)
return self._sitl
def _getConnStr(self):
return tcp_master(self.iNum)
#lazy
def connStr(self):
self.sitl
return self._getConnStr()
def setParamAndRelaunch(self, key, value):
wd = self._sitl.wd
print("relaunching .... ", self._sitl.p.pid)
v = connect(self._getConnStr(), wait_ready=True) # if use connStr will trigger cyclic invocation
v.parameters.set(key, value, wait_ready=True)
v.close()
self._sitl.stop()
self._sitl.launch(self.args, await_ready=True, restart=True, wd=wd, use_saved_data=True)
v = connect(self._getConnStr(), wait_ready=True)
# This fn actually rate limits itself to every 2s.
# Just retry with persistence to get our first param stream.
v._master.param_fetch_all()
v.wait_ready()
actualValue = v._params_map[key]
assert actualValue == value
v.close()
def close(self):
self._sitl.stop()
usedINums.remove(self.iNum)
lazy decorator is from this library:
https://docs.python.org/2/tutorial/classes.html#generator-expressions
It would help to see how your class looks, but if it has methods from multiprocessing you may have issues just pickling it by default. Multiprocessing objects can use locks as well, and these are (obviously) unpickle-able.
You can customize pickling with the __getstate__ method, or __reduce__ (documented in the same place).
I'm writing a test runner. I have an object that can catch and store exceptions, which will be formatted as a string later as part of the test failure report. I'm trying to unit-test the procedure that formats the exception.
In my test setup, I don't want to actually throw an exception for my object to catch, mainly because it means that the traceback won't be predictable. (If the file changes length, the line numbers in the traceback will change.)
How can I attach a fake traceback to an exception, so that I can make assertions about the way it's formatted? Is this even possible? I'm using Python 3.3.
Simplified example:
class ExceptionCatcher(object):
def __init__(self, function_to_try):
self.f = function_to_try
self.exception = None
def try_run(self):
try:
self.f()
except Exception as e:
self.exception = e
def format_exception_catcher(catcher):
pass
# No implementation yet - I'm doing TDD.
# This'll probably use the 'traceback' module to stringify catcher.exception
class TestFormattingExceptions(unittest.TestCase):
def test_formatting(self):
catcher = ExceptionCatcher(None)
catcher.exception = ValueError("Oh no")
# do something to catcher.exception so that it has a traceback?
output_str = format_exception_catcher(catcher)
self.assertEquals(output_str,
"""Traceback (most recent call last):
File "nonexistent_file.py", line 100, in nonexistent_function
raise ValueError("Oh no")
ValueError: Oh no
""")
Reading the source of traceback.py pointed me in the right direction. Here's my hacky solution, which involves faking the frame and code objects which the traceback would normally hold references to.
import traceback
class FakeCode(object):
def __init__(self, co_filename, co_name):
self.co_filename = co_filename
self.co_name = co_name
class FakeFrame(object):
def __init__(self, f_code, f_globals):
self.f_code = f_code
self.f_globals = f_globals
class FakeTraceback(object):
def __init__(self, frames, line_nums):
if len(frames) != len(line_nums):
raise ValueError("Ya messed up!")
self._frames = frames
self._line_nums = line_nums
self.tb_frame = frames[0]
self.tb_lineno = line_nums[0]
#property
def tb_next(self):
if len(self._frames) > 1:
return FakeTraceback(self._frames[1:], self._line_nums[1:])
class FakeException(Exception):
def __init__(self, *args, **kwargs):
self._tb = None
super().__init__(*args, **kwargs)
#property
def __traceback__(self):
return self._tb
#__traceback__.setter
def __traceback__(self, value):
self._tb = value
def with_traceback(self, value):
self._tb = value
return self
code1 = FakeCode("made_up_filename.py", "non_existent_function")
code2 = FakeCode("another_non_existent_file.py", "another_non_existent_method")
frame1 = FakeFrame(code1, {})
frame2 = FakeFrame(code2, {})
tb = FakeTraceback([frame1, frame2], [1,3])
exc = FakeException("yo").with_traceback(tb)
print(''.join(traceback.format_exception(FakeException, exc, tb)))
# Traceback (most recent call last):
# File "made_up_filename.py", line 1, in non_existent_function
# File "another_non_existent_file.py", line 3, in another_non_existent_method
# FakeException: yo
Thanks to #User for providing FakeException, which is necessary because real exceptions type-check the argument to with_traceback().
This version does have a few limitations:
It doesn't print the lines of code for each stack frame, as a real
traceback would, because format_exception goes off to look for the
real file that the code came from (which doesn't exist in our case).
If you want to make this work, you need to insert fake data into
linecache's
cache (because traceback uses linecache to get hold of the source
code), per #User's answer
below.
You also can't actually raise exc and expect the fake traceback
to survive.
More generally, if you have client code that traverses tracebacks in
a different manner than traceback does (such as much of the inspect
module), these fakes probably won't work. You'd need to add whatever
extra attributes the client code expects.
These limitations are fine for my purposes - I'm just using it as a test double for code that calls traceback - but if you want to do more involved traceback manipulation, it looks like you might have to go down to the C level.
EDIT2:
That is the code of linecache.. I will comment on it.
def updatecache(filename, module_globals=None): # module_globals is a dict
# ...
if module_globals and '__loader__' in module_globals:
name = module_globals.get('__name__')
loader = module_globals['__loader__']
# module_globals = dict(__name__ = 'somename', __loader__ = loader)
get_source = getattr(loader, 'get_source', None)
# loader must have a 'get_source' function that returns the source
if name and get_source:
try:
data = get_source(name)
except (ImportError, IOError):
pass
else:
if data is None:
# No luck, the PEP302 loader cannot find the source
# for this module.
return []
cache[filename] = (
len(data), None,
[line+'\n' for line in data.splitlines()], fullname
)
return cache[filename][2]
That means before you testrun just do:
class Loader:
def get_source(self):
return 'source of the module'
import linecache
linecache.updatecache(filename, dict(__name__ = 'modulename without <> around',
__loader__ = Loader()))
and 'source of the module' is the source of the module you test.
EDIT1:
My solution so far:
class MyExeption(Exception):
_traceback = None
#property
def __traceback__(self):
return self._traceback
#__traceback__.setter
def __traceback__(self, value):
self._traceback = value
def with_traceback(self, tb_or_none):
self.__traceback__ = tb_or_none
return self
Now you can set the custom tracebacks of the exception:
e = MyExeption().with_traceback(1)
What you usually do if you reraise an exception:
raise e.with_traceback(fake_tb)
All exception prints walk through this function:
import traceback
traceback.print_exception(_type, _error, _traceback)
Hope it helps somehow.
You should be able to simply raise whatever fake exception you want where you want it in your test runs. The python exception docs suggest you create a class and raise that as your exception. It's section 8.5 of the docs.
http://docs.python.org/2/tutorial/errors.html
Should be pretty straightforward once you've got the class created.
I wrote a simple program to read through a log and to parse through and obtain the lowest beginning number (the head) and to print it. I am now editing that program and combining it with a class I wrote to parse an actual logfile. Essentially, as opposed to sorting based off of the simple number from the log from my previous program, I now need to reference the parsed information from one class into another class. I was wondering what the most convenient way to do this. I am a beginner programmer in python and don't know if I can explicitly reference the class.
Here are the classes.
Parser
class LogLine:
SEVERITIES = ['EMERG','ALERT','CRIT','ERR','WARNING','NOTICE','INFO','DEBUG']
severity = 1
def __init__(self, line):
try:
m = re.match(r"^(\d{4}-\d{2}-\d{2}\s*\d{2}:\d{2}:\d{2}),?(\d{3}),?(\s+\[(?:[^\]]+)\])+\s+[A-Z]+\s+(\s?[a-zA-Z0-9\.])+\s?(\((?:\s?\w)+\))\s?(\s?.)+", line)
timestr, msstr, sevstr, self.filename, linestr, self.message = m.groups()
self.line = int(linestr)
self.sev = self.SEVERITIES.index(sevstr)
self.time = float(calendar.timegm(time.strptime(timestr, "%Y-%m-%d %H:%M:%S,%f"))) + float(msstr)/1000.0
dt = datetime.strptime(t, "%Y-%m-%d %H:%M:%S,%f")
except Exception:
print 'error',self.filename
def get_time(self):
return self.time
def get_severity(self):
return self.sev
def get_message(self):
return self.message
def get_filename(self):
return self.filename
def get_line(self):
return self.line
Sorter
class LogFile:
def __init__(self,filepath):
self.logfile = open(filepath, "r")
self.head = None
def __str__(self):
return "x=" + str(self.x) + "y="+str(self.y)
def readline(self):
if self.head != None:
h = self.head
self.head = None
return h
else:
return self.logfile.readline().rstrip(' ')
def get_line(self):
if self.head == None:
self.head = self.readline().rstrip(' ')
return self.head.get.line()
else:
return self.head.get.line()
def close (self):
self.logfile.close()
I have begun to edit my second class by adding the get_line function. Don't know if I'm on the right track.
In simpler terms, I need the head to become "LogLine"
It is okay to use one class from another class. You have one class that parses a single line from a log file and builds an object that represents the line; and you have another class that reads lines from a log file. It would be very natural for the second class to call the first class.
Here is a very simple class that reads all lines from a log file and builds a list:
class LogFile(object):
def __init__(self,filepath):
with open(filepath, "r") as f:
self.lst = [LogLine(line) for line in f]
You can see that self.lst is being set to a list of lines from the input log file, but not just the text of the line; the code is calling LogLine(line) to store instances of LogLine. If you want, you can sort the list after you build it:
self.lst.sort(key=LogLine.get_line)
If the log files are very large, it might not be practical to build the list. You have a .get_line() method function, and we can use that:
class LogFile(object):
def __init__(self,filepath):
self.logfile = open(filepath, "r")
def get_line(self):
try:
line = next(self.logfile) # get next line from open file object
return LogLine(line)
except StopIteration: # next() raises this when you reach the end of the file
return None # return
def close(self):
self.logfile.close()
An open file object (returned by the open() function) can be iterated. We can call next() on this object and it will give us the next input line. When the end of file is reached, Python will raise StopIteration to signal the end of the file.
Here the code will catch the StopIteration exception and return None when the end of the log file is reached. But I think this isn't the best way to handle this problem. Let's make the LogFile class work in for loops and such:
class LogFile(object):
def __init__(self,filepath):
self.f = open(filepath)
def __next__(self): # Python 3.x needs this to be named "__next__"
try:
line = next(self.f)
return LogLine(line)
except StopIteration:
# when we reach the end of input, close the file object
self.f.close()
# re-raise the exception
raise
next = __next__ # Python 2.x needs this to be named "next"
A for loop in Python will repeatedly call the .__next__() method function (Python 3.x) or else the .next() method function (Python 2.x) until the StopIteration exception is raised. Here we have defined both method function names so this code should work in Python 2.x or in Python 3.x.
Now you can do this:
for ll in LogFile("some_log_file"):
... # do something with ll, which will always be a LogLine instance