I am writing a Python module in which two threads access one list. One thread adds 500 items to the list per second, and the other thread reads the list at an irregular interval. I want to make a thread-safe "list" class to avoid having to use locks every time I read or write to the list (suggested by this answer to a previous question on SO).
Here is my first go at a thread-safe list class (with help from these previous SO answers: 1 and 2). Are there any methods that should be locked that are not currently locked, or any methods that do not require a lock that are currently locked?
import collections
import threading
class ThreadSafeList(collections.MutableSequence):
"""Thread-safe list class."""
def __init__(self, iterable=None):
if iterable is None:
self._list = list()
else:
self._list = list(iterable)
self.rlock = threading.RLock()
def __len__(self): return len(self._list)
def __str__(self): return self.__repr__()
def __repr__(self): return "{}".format(self._list)
def __getitem__(self, i): return self._list[i]
def __setitem__(self, index, value):
with self.rlock:
self._list[index] = value
def __delitem__(self, i):
with self.rlock:
del self._list[i]
def __iter__(self):
with self.rlock:
for elem in self._list:
yield elem
def insert(self, index, value):
with self.rlock:
self._list.insert(index, value)
Related
Actually my sort algorithm works, but there is a problem.
I have a class namely SortedItem which includes
def __init__(self, point, cost):
self.coordinate = point
self.cost = cost
and I have also priority queue which sorts the this SortedItem by its cost:
class PriorityQueue:
def __init__(self):
self.items = []
def isEmpty(self):
return self.items == []
def sortComparatorByCost(self, item):
return item.cost
def enqueue(self, item):
self.items.append(item)
self.items.sort(key=self.sortComparatorByCost, reverse=True)
def dequeue(self):
return self.items.pop()
def returnQueueAsString(self):
queue_str = ""
for eachItem in self.items:
queue_str += str(eachItem) + " "
return queue_str
def isQueueContainsElement(self, element):
for eachElement in self.items:
if eachElement[0] == element:
return True
return False
The problem occurs here:
- I have defined some order to add queue. Let's say I am adding this objects to the queue:
obj1 = SortedItem((1,0), 10))
queue.enqueue(obj1)
obj2 = SortedItem((2,0), 15))
queue.enqueue(obj2)
obj3 = SortedItem((2,1), 15))
queue.enqueue(obj3)
Now I have to get objects from queue in this order (obj1, obj2, obj3).
However python built-in sort function sort these objects like this: (obj1, obj3, obj2) (because obj2 and obj3 has the same cost)
How can i solve this issue. I mean If 2 objects cost is the same, I should get the first added one.
Note that: I have just created a simple example of my problem. If you try this code you may get the objects in this order: obj1, obj2, obj3
Instead of sorting the items in reverse order and removing them from the right,
def enqueue(self, item):
self.items.append(item)
self.items.sort(key=self.sortComparatorByCost, reverse=True)
def dequeue(self):
return self.items.pop()
you could remove them from the left. That would avoid reversing the order of insertion of the items with the same cost.
def enqueue(self, item):
self.items.append(item)
self.items.sort(key=self.sortComparatorByCost)
def dequeue(self):
return self.items.pop(0)
Removing items from the beginning of a list is not efficient, however, so you could better use a deque (replacing pop(0) by popleft()) to fix that. A deque on the other hand, has no in-place sort() method, so would need to replace self.items.sort() by self.items = deque(sorted(self.items)) as well.
I am interested in counting the number of accesses to a dictionary's values. I am unsure how to include dictionary unpacking in the counter. Any tips?
from collections import defaultdict
class LDict(dict):
def __init__(self, *args, **kwargs):
'''
This is a read-counting dictionary
'''
super().__init__(*args, **kwargs)
self._lookup = defaultdict(lambda : 0)
def __getitem__(self, key):
retval = super().__getitem__(key)
self._lookup[key] += 1
return retval
def __setitem__(self, key, value):
super().__setitem__(key, value)
self._lookup[key] = self._lookup.default_factory()
def __delitem__(self, key):
super().__delitem__(self, key)
_ = self._lookup[key]
del self._lookup[key]
def list_unused(self):
return [key for key in self if self._lookup[key] == 0]
l = LDict(a='apple', b='bugger')
print({**l, **l})
print(l.list_unused())
_ = l['a']
print(l.list_unused())
You need to override more methods. Access is not centralized through __getitem__(): other methods like copy(), items(), etc. access the keys without going through __getitem()__. I would assume the ** operator uses items(), but you will need to handle ALL of the methods to keep track of EVERY access. In many cases you will have to make a judgement call. For example, does __repr__() count as an access? The returned string contains every key and value formatted, so I think it does.
I would recommend overriding all of these methods, because you have to do bookkeeping on assignment too.
def __repr__(self):
def __len__(self):
def __iter__(self):
def clear(self):
def copy(self):
def has_key(self, k):
def update(self, *args, **kwargs):
def keys(self):
def values(self):
def items(self):
EDIT: So apparently there's an important caveat here that directly relates to your implementation. if LDict extends dict, then none of these methods are invoked during the dictionary unpacking { **l, **l}.
Apparently you can follow the advice here though, and implement LDict without extending dict. This worked for me:
from collections import MutableMapping
class LDict(MutableMapping):
def __init__(self, *args, **kwargs):
'''
This is a read-counting dictionary
'''
self._lookup = defaultdict(lambda : 0)
self.data = {}
if kwargs:
self.data.update(kwargs)
def __getitem__(self, key):
retval = self.data[key]
self._lookup[key] += 1
return retval
def __setitem__(self, key, value):
self.data[key] = value
self._lookup[key] = self._lookup.default_factory()
def __delitem__(self, key):
del self.data[key]
_ = self._lookup[key]
del self._lookup[key]
def items(self):
print('items is being called!')
yield from self.data.items()
def __iter__(self):
print('__iter__ is being called!')
yield from self.data
def __len__(self):
return len(self.data)
def list_unused(self):
return [key for key in self if self._lookup[key] == 0]
l = LDict(a='apple', b='bugger')
print({**l, **l})
print(l.list_unused())
_ = l['a']
print(l.list_unused())
which produces the output:
__iter__ is being called!
__iter__ is being called!
{'b': 'bugger', 'a': 'apple'}
__iter__ is being called!
[]
__iter__ is being called!
[]
(I only implemented the bare minimum to get example to work, I still recommend implementing the set of methods I listed about if you want your counts to be correct!)
So I guess the answer to your question is you have to
Implement the __iter__(self) method
DO NOT inherit from dict().
Is there a way to give a comparator to set() so when adding items it checks an attribute of that item for likeness rather than if the item is the same? For example, I want to use objects in a set that can contain the same value for one attribute.
class TestObj(object):
def __init__(self, value, *args, **kwargs):
self.value = value
super().__init__(*args, **kwargs)
values = set()
a = TestObj('a')
b = TestObj('b')
a2 = TestObj('a')
values.add(a) # Ok
values.add(b) # Ok
values.add(a2) # Not ok but still gets added
# Hypothetical code
values = set(lambda x, y: x.value != y.value)
values.add(a) # Ok
values.add(b) # Ok
values.add(a2) # Not added
I have implemented my own sorta thing that does similar functionality but wanted to know if there was a builtin way.
from Queue import Queue
class UniqueByAttrQueue(Queue):
def __init__(self, attr, *args, **kwargs):
Queue.__init__(self, *args, **kwargs)
self.attr = attr
def _init(self, maxsize):
self.queue = set()
def _put(self, item):
# Potential race condition, worst case message gets put in twice
if hasattr(item, self.attr) and item not in self:
self.queue.add(item)
def __contains__(self, item):
item_attr = getattr(item, self.attr)
for x in self.queue:
x_attr = getattr(x, self.attr)
if x_attr == item_attr:
return True
return False
def _get(self):
return self.queue.pop()
Just define __hash__ and __eq__ on the object in terms of the attribute in question and it will work with sets. For example:
class TestObj(object):
def __init__(self, value, *args, **kwargs):
self.value = value
super().__init__(*args, **kwargs)
def __eq__(self, other):
if not instance(other, TestObj):
return NotImplemented
return self.value == other.value
def __hash__(self):
return hash(self.value)
If you can't change the object (or don't want to, say, because other things are important to equality), then use a dict instead. You can either do:
mydict[obj.value] = obj
so new objects replace old, or
mydict.setdefault(obj.value, obj)
so old objects are maintained if the value in question is already in the keys. Just make sure to iterate using .viewvalues() (Python 2) or .values() (Python 3) instead of iterating directly (which would get the keys, not the values). You could actually use this approach to make a custom set-like object with a key as you describe (though you'd need to implement many more methods than I show to make it efficient, the default methods are usually fairly slow):
from collections.abc import MutableSet # On Py2, collections without .abc
class keyedset(MutableSet):
def __init__(self, it=(), key=lambda x: x):
self.key = key
self.contents = {}
for x in it:
self.add(x)
def __contains__(self, x):
# Use anonymous object() as default so all arguments handled properly
sentinel = object()
getval = self.contents.get(self.key(x), sentinel)
return getval is not sentinel and getval == x
def __iter__(self):
return iter(self.contents.values()) # itervalues or viewvalues on Py2
def __len__(self):
return len(self.contents)
def add(self, x):
self.contents.setdefault(self.key(x), x)
def discard(self, x):
self.contents.pop(self.key(x), None)
OK so my actual code is somewhat elaborate but I am illustrating the problem that I am having with the following example code:
I have a class that has a list as one of its instance variable. I want the class to be an iterable and return the next element in the list when next is called in the for loop.
So I have as follows:
class SimplaWannaBeIteratable(object):
def __init__(self, list_to_iter, **kwargs)
self._list = list_to_iter
self._item = None
#... other code to initialize
def __iter__(self):
return self
def next(self):
self._item= next(self._list)
return self._item
def current(self):
#So that other uses cases have the access to the current member
return self._current
However if I do the following:
iter_item = SimplaWannaBeIteratable([1,2,3,4,5])
for item in iter_item:
return item
I get:
list object is not an iterator.
If I change the next as follows:
def next(self):
self._item= next(iter((self._list)))
return self._item
I get infinite output.
Can anyone tell me what I need to do to accomplish the task I want to do and why the code above is not working?
From what I understand every time next is called the iterator object associated with the list is called and its next is return. so why can't my list find its iterator?
You need an iterator to iterator over a list. A list itself is not an iterator so you cannot call next() on it.
class SimplaWannaBeIteratable(object):
def __init__(self, list_to_iter, **kwargs):
self._list = list_to_iter
self._item = None
def __iter__(self):
self._iter = iter(self._list) # create/initialize the iterator
return self
def __next__(self): # using the Python 3.x name
self._item = next(self._iter) # use the iterator
return self._item
# ...
You are calling next on self._list, which is a list, not an iterator. next only advances iterators, it does not set up an iterator from an iterable.
def __init__(self, ...):
# ...
self._iterator = iter(self._list)
def next(self):
self._item = next(self._iterator)
return self._item
Regarding your edit, you are getting an infinite recursion because you are calling next on a fresh iterator each time, rather than the same iterator. So you are losing the state of the iterator. Again, see my example above, which sets up the iterator once.
The __next__ special method that you are trying to implement is used to control iteration over a container-like class at each progressive step. If you do not need this functionality and simply want to make your class iterable, omit the method and return iter(self._list) from __iter__:
class SimplaWannaBeIteratable(object):
def __init__(self, list_to_iter, **kwargs):
self._list = list_to_iter
self._item = None
def __iter__(self):
return iter(self._list)
def current(self):
return self._current
Demo:
>>> iter_item = SimplaWannaBeIteratable([1,2,3,4,5])
>>> for item in iter_item:
... item
...
1
2
3
4
5
>>>
Ok so im trying to input a word in a stack and I want to print all of them after I input a string. So I can only print them one at a time. I tried using a for loop outside but Stacks are apparently not iterable. So I iterating it inside the stack. It still is not working.
class Stack:
def __init__(self):
self.items = []
def push(self,items):
self.items.insert(0,items)
def pop(self):
for x in self.items:
print( self.items.pop(0))
def show(self):
print (self.items)
s = Stack()
s.show()
placed = input("enter")
item = s.pop()
print(item, "is on top", s)
Give your Stack class a __len__ method, this will make testing if the stack is empty easier:
class Stack:
def __init__(self):
self.items = []
def push(self,item):
self.items.append(item)
def pop(self):
return self.items.pop()
def show(self):
print (self.items)
def __len__(self):
return len(self.items)
stack = Stack()
stack.push('World!')
stack.push('Hello')
while stack: # tests the length through __len__
print(stack.pop())
Note that I simply .append() to the end of the .items list, then later on .pop() (no arguments) again, removing from the end of the list.
To make your class an iterable type, you'd need to add at least an __iter__ method, optionally together with a .__next__() method:
class Stack:
# rest elided
def __iter__(self):
return self
def next(self):
try:
return self.items.pop()
except IndexError: # empty
raise StopIteration # signal iterator is done