We Keep Coding

Pandas custom ExtensionDtype ordering issues with reindex

I am implementing a custom ExtensionDtype and seeing that the order of the results becomes out of sync with the original index when using reindex operation or appending/concat on dataframes. I understand the result can be reordered but I expect the index/column names to be aligned with the data.
Here is my implementation.
#register_extension_dtype
class CategoryListWithHashFeatureDtype(ExtensionDtype):
name = 'CategoryListWithHashFeature'
type = CategoryListWithHashFeature
kind = 'O'
na_value = DEAL_NAN
#classmethod
def construct_array_type(cls):
return CategoryListWithHashFeatureArray
#classmethod
def construct_from_string(cls, string):
if string == cls.name:
return cls()
else:
raise TypeError(f'Cannot construct a {cls} from {string}')
class CategoryListWithHashFeatureArray(ExtensionArray):
def __init__(self, values: List[CategoryListWithHashFeature] = None):
self._dtype = CategoryListWithHashFeatureDtype()
self.raw_names: List[str] = []
self.raw_values: List[Union[List[str], List[bytes]]] = []
self.hash_names: List[str] = []
self.hash_values: List[List[int]] = []
if values:
for feature in values:
if isinstance(feature, CategoryListWithHashFeature):
self.raw_names.append(feature.name)
self.raw_values.append(feature.values)
self.hash_names.append(feature.hash_name)
self.hash_values.append(feature.hash_values)
else:
self.raw_names.append(NAN_FEATURE_NAME)
self.raw_values.append([])
self.hash_names.append(NAN_FEATURE_NAME)
self.hash_values.append([])
#classmethod
def _from_values(cls,
raw_names: List[str],
raw_values: List[Union[List[str], List[bytes]]],
hash_names: List[str],
hash_values: List[List[int]]) -> CategoryListWithHashFeatureArray:
inst = CategoryListWithHashFeatureArray()
inst.raw_names = raw_names
inst.raw_values = raw_values
inst.hash_names = hash_names
inst.hash_values = hash_values
return inst
#classmethod
def _from_sequence(cls, scalars, dtype=None, copy=False):
if isinstance(scalars, ndarray):
scalars = list(scalars)
return cls(scalars)
#classmethod
def _from_factorized(cls, values, original):
return cls(values)
def __getitem__(self, index):
try:
if isinstance(index, int) or isinstance(index, np.int64):
if len(self) == 0:
return CategoryListWithHashFeatureDtype.na_value
return CategoryListWithHashFeature(
raw_name=self.raw_names[index],
raw_values=self.raw_values[index],
hash_name=self.hash_names[index],
hash_values=self.hash_values[index]
)
elif isinstance(index, slice):
raw_names: List[str] = self.raw_names[index]
return CategoryListWithHashFeatureArray([
CategoryListWithHashFeature(
raw_name=self.raw_names[idx],
raw_values=self.raw_values[idx],
hash_name=self.hash_names[idx],
hash_values=self.hash_values[idx])
for idx in range(len(raw_names))
])
elif isinstance(index, ndarray):
return CategoryListWithHashFeatureArray([
CategoryListWithHashFeature(
raw_name=self.raw_names[idx],
raw_values=self.raw_values[idx],
hash_name=self.hash_names[idx],
hash_values=self.hash_values[idx])
for idx in list(index)
])
else:
raise RuntimeError(f'Unsupported index type {str(type(index))}')
except Exception:
logging.exception(f'index: {index}. Val: {self.raw_names}')
return self._dtype.na_value
def __len__(self):
return len(self.raw_names)
#property
def dtype(self):
return self._dtype
def __eq__(self, other) -> bool:
if not isinstance(other, CategoryListWithHashFeatureArray):
return False
if len(self) != len(other):
return False
return all(self.__getitem__(idx) == other.__getitem__(idx) for idx in range(len(self)))
def __hash__(self):
h = 0
for idx in range(len(self)):
h += self.__getitem__(idx).__hash__()
return h
#property
def nbytes(self):
return sys.getsizeof(self.raw_names) + sys.getsizeof(self.raw_values) \
+ sys.getsizeof(self.hash_names) + sys.getsizeof(self.hash_values)
def isna(self):
missing_values = [val == CategoryListWithHashFeatureDtype.na_value or len(val) == 0 for val in self.raw_values]
return np.array(missing_values)
def take(self, indices, allow_fill=False, fill_value=None):
# During reindex we get a -1 index if the item was not found in the original data
data = [self.__getitem__(idx) for idx in indices if idx >= 0]
if allow_fill and fill_value is None:
fill_value = self._dtype.na_value
result = pd.api.extensions.take(data, indices, fill_value=fill_value, allow_fill=allow_fill)
return self._from_sequence(result)
def copy(self, deep=False):
copy = [self._copy_feature(idx) for idx in range(len(self.raw_names))]
return type(self)(copy)
def _copy_feature(self, idx: int) -> CategoryListWithHashFeature:
return CategoryListWithHashFeature(
raw_name=self.raw_names[idx],
raw_values=self.raw_values[idx].copy(),
hash_name=self.hash_names[idx],
hash_values=self.hash_values[idx].copy()
)
#classmethod
def _concat_same_type(cls, to_concat):
raw_names: List[str] = []
raw_values: List[Union[List[str], List[bytes]]] = []
hash_names: List[str] = []
hash_values: List[List[int]] = []
for feature_array in to_concat: # type: CategoryListWithHashFeatureArray
raw_names.extend(feature_array.raw_names)
raw_values.extend(feature_array.raw_values)
hash_names.extend(feature_array.hash_names)
hash_values.extend(feature_array.hash_values)
return cls._from_values(raw_names=raw_names,
raw_values=raw_values,
hash_names=hash_names,
hash_values=hash_values)
Tests:
category_list_features = [
CategoryListWithHashFeature('size2', ['s', 'm'], 'size2_hash', [2]),
CategoryListWithHashFeature('color', ['R'], 'color_hash', [1])
]
s1 = Series({f.name: f for f in category_list_features}, dtype='CategoryListWithHashFeature')
s2 = s1.reindex(index=['color', 'size', 'size2'])
print(s2)
color size2: ['s', 'm'] / Hashed size2: [2]
size :NAN_F: [] / Hashed :NAN_F: []
size2 color: ['R'] / Hashed color_hash: [1]
dtype: CategoryListWithHashFeature
My initial guess was there was something incorrect about my handling of the negative indices in the take method so I added the negative index check. Index values I get in my take method: [1, -1, 0] ==> the -1 corresponds to the missing size index/value in the series. I know this incorrect because the take function can actually get legit negative index and i need to support it. If I remove the negative index check I get the color value duplicated:
color color: ['R'] / Hashed color_hash: [1]
size :NAN_F: [] / Hashed :NAN_F: []
size2 color: ['R'] / Hashed color_hash: [1]
dtype: CategoryListWithHashFeature
I did some debugging and this seems to happen in the pandas.algorithms.take_nd function:
Relevant snippet:
func = _get_take_nd_function(
arr.ndim, arr.dtype, out.dtype, axis=axis, mask_info=mask_info
)
func(arr, indexer, out, fill_value)
I am using pandas version 1.2.1
I would appreciate some help on this. Did I miss something?
Thanks!

This was probably a silly question. If I have my own take method, I don't need to call pd.api.extensions.take again. I can just return my calculated results.
I would still like to know the reason for the internal take returning the unexpected result.

'int' object not callable heap sort function

'int' Object Not Callable Heap Sort Function
Im trying to create a function that return a sorted list but im getting the following error: "new_value = sort_heap.deleteMin() 'int' object is not callable"
This is the code:
class MinHeap:
def __init__(self):
self.heap=[]
def __str__(self):
return f'{self.heap}'
__repr__=__str__
def parent(self,index):
# -- YOUR CODE STARTS HERE
if index>len(self) or index<=1:
return None
else:
return self.heap[index//2-1]
def leftChild(self,index):
# -- YOUR CODE STARTS HERE
if index<1 or 2*index>len(self):
return None
else:
return self.heap[2*index-1]
def rightChild(self,index):
if index<1 or 2*index>len(self):
return None
else:
return self.heap[2*index-1]
def __len__(self):
return len(self.heap)
def insert(self,x):
self.heap.append(x)
current = len(self)
while self.parent(current) is not None and self.parent(current)>x:
self.heap[current-1], self.heap[current//2-1] = self.heap[current//2-1], self.heap[current-1]
current = current//2
#property
def deleteMin(self):
if len(self)==0:
return None
elif len(self)==1:
out=self.heap[0]
self.heap=[]
return out
deleted = self.heap[0]
current = 1
self.heap[0] = self.heap[len(self)-1]
x = self.heap.pop()
moved_value = self.heap[0]
while self.leftChild(current) is not None:
left=self.leftChild(current)
right=self.rightChild(current)
if right is not None:
if left<=right and left<moved_value:
self.heap[current-1], self.heap[current*2] = self.heap[current*2], self.heap[current-1]
current = current *2
elif left>right and right<moved_value:
self.heap[current-1], self.heap[current*2] = self.heap[current*2], self.heap[current-1]
current = (current*2) + 1
else:
break
else:
if left<moved_value:
self.heap[current-1], self.heap[(current*2)-1] = self.heap[(current*2)-1], self.heap[current-1]
current = current*2
else:
break
return deleted
def heapSort(numList):
'''
>>> heapSort([9,7,4,1,2,4,8,7,0,-1])
[-1, 0, 1, 2, 4, 4, 7, 7, 8, 9]
'''
sort_heap = MinHeap()
for i in range (len(numList)):
sort_heap.insert(numList[i])
sortedList= []
lenght=len(numList)
while lenght >0:
new_value = sort_heap.deleteMin()
sortedList.append(new_value)
lenght -= 1
return sortedList
The MinHeap class is a given but I'm allow to modify it. Could Someone please help? Thanks

Seeing your (btw wrong formatted code, please redo the formatting) code I can see, that deleteMin is a #property and not a class method. Therefore you shuld write:
new_value = sort_heap.deleteMin
# ^ You see, no brackets here

Class polynomial

I am learning classes in Python, I created a class called polynomial, and am trying to add two polynomials, but always get the following error message
soma.termos[i] = self[i] + other[i] TypeError: 'Polinomio' object
does not support indexing to fix
to fix I created an attribute that is the size of the vector and creates a vector of zeros of size n, but still, the error persists, what is wrong?
class Polinomio:
def __init__ (self, termos = [], n = 0):
self.termos = termos
self.n = [0] * n
def __len__ (self):
return len(self.termos)
def __setitem__ (self, i, x):
self.termos[i] = x
def __add__ (self, other):
soma = Polinomio(n = len(self.termos))
for i in range(len(self.termos)):
soma.termos[i] = self[i] + other[i]
def print (self):
print(self.termos)
p1 = Polinomio([1, 2, 3])
p2 = Polinomio([1, 2, 3])
p2.print()
p3 = Polinomio()
p3 = p1 + p2

You're not using your internal termos property when adding, instead you're trying to index your whole instance which, unsurprisingly, raises an error. Try changing your __add__ method to:
def __add__ (self, other):
soma = Polinomio(n = len(self.termos))
for i in range(len(self.termos)):
soma.termos[i] = self.termos[i] + other[i]
return soma
Or even better:
def __add__ (self, other):
soma = Polinomio(n = len(self.termos))
for i, v in enumerate(self.termos):
soma.termos[i] = v + other[i]
return soma
Also, do not initialize your termos list in your __init__ signature as it will always refer to the same list. Instead, declare it as None and build it as new whenever it's not passed, i.e.:
def __init__ (self, termos = None, n = 0):
self.termos = termos or []
self.n = [0] * n

You should add a method __getitem__:
def __getitem__(self, i):
return self.termos[i]
And also in your __add__ function, you instantiate a Polinomio by saying n = len(self.termos) but your using something called keyword arguments, so it will not actually instantiate a Polinomio with n as len(self.termos, you should instead say Polinomio([], len(self.termos) or implement keyword arguments if you want to use that syntax to intstantiate it.

Overriding __getitem__ for a nested list?

I'm implementing an exerimental QR code parser and I figured it would be handy to override a list's __getitem__ that takes a given mask into account, like this:
m = [[1, 0], [0, 1]]
def mask(m, i, j):
if i % 2 == 0 or j == 0:
return int(not m[i][j])
return m[i][j]
m2 = list_with_mask(m, mask)
n = m2[0][0]
How can I achieve it in the most Pythonic way?

Quick & dirty implementation, maybe it's better to inherit from the built-in list class.
Not directly what OP asked but but at least it's a start, and you can customize it for your needs.
class CustomNestedObject:
"""Custom weird class to handle __getitem__
TODO: add error handling for strings and other non list/tuple objects
"""
ERRORS = {
'element_doesnt_exist': "You don't have element with such index"
}
def __init__(self, obj):
self._nested = [] # will store nested recursive CustomNestedObject(s)
self._value = None # will store value (for example integer or string)
# recursively parse obj to CustomNestedObject
self._parse_to_self(obj)
def __repr__(self):
"""Method which will return string representation for the nested objects or self._value"""
if not self._nested:
return str(self._value)
else:
return str([x._value for x in self._nested])
def __getitem__(self, index):
# handle error
try:
self._nested[index]
except IndexError:
raise Exception(self.ERRORS['element_doesnt_exist'])
if not self._nested[index]._nested:
# it means that returned object will be self.value
# print(f'trying to access {self._nested[index]._value}')
return self._nested[index]._value
else:
# print('trying to access nested object')
return self._nested[index]
def _parse_to_self(self, obj):
if isinstance(obj, list) or isinstance(obj, tuple):
for item in obj:
self._nested.append(CustomNestedObject(item))
else:
# save as number if obj is not a list or tuple
self._value = obj
if __name__ == '__main__':
x = CustomNestedObject([1, 2, 3, [4, 5]])
print(x[3][1])
print(x[0])
print(x[9])

list with infinite elments

I need to operate on two separate infinite list of numbers, but could not find a way to generate, store and operate on it in python.
Can any one please suggest me a way to handle infinite Arithmetic Progession or any series and how to operate on them considering the fact the minimal use of memory and time.
Thanks every one for their suggestions in advance.

You are looking for a python generator instead:
def infinitenumbers():
count = 0
while True:
yield count
count += 1
The itertools package comes with a pre-built count generator.
>>> import itertools
>>> c = itertools.count()
>>> next(c)
0
>>> next(c)
1
>>> for i in itertools.islice(c, 5):
... print i
...
2
3
4
5
6

This is where the iterator comes in. You can't have an infinite list of numbers, but you can have an infinite iterator.
import itertools
arithmetic_progression = itertools.count(start,step) #from the python docs
The docs for Python2 can be found here

I have another python3 solution (read SICP chapter 3.5)
class Stream:
def __init__(self, head, tail):
self.head = head
self.tail = tail
self.memory = None
self.isDone = False
def car(self):
return self.head
def cdr(self):
if self.isDone:
return self.memory
self.memory = self.tail()
self.isDone = True
return self.memory
def __getitem__(self, pullFrom):
if pullFrom < 1 or self.memory == []:
return []
return [self.car()] + self.cdr()[pullFrom - 1]
def __repr__(self):
return "[" + repr(self.car()) + " x " + repr(self.tail) + "]"
def map(self, func):
if self.memory == []:
return []
return Stream(func(self.car()), lambda: Stream.map(self.cdr(), func))
def from_list(lst):
if lst == []:
return []
return Stream(lst[0], lambda:
Stream.from_list(lst[1:]))
def filter(self, pred):
if self.memory == []:
return []
elif pred(self.car()):
return Stream(self.car(), lambda: Stream.filter(self.cdr(), pred))
else:
return self.cdr().filter(pred)
def sieve(self):
return Stream(self.car(), lambda: self.cdr().filter(lambda n: n % self.car() > 0).sieve())
def foreach(self, action, pull = None):
if pull is None:
action(self.car())
self.cdr().foreach(action, pull)
elif pull <= 0:
return
else:
action(self.car())
self.cdr().foreach(action, pull-1)and run:
a = Stream(0, lambda: a.map((lambda x: x + 1)))
print(a[10])
which returns:
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] .
But streams are lazily evaluated, so:
>>> a = Stream(0, lambda: a.map((lambda x: x + 1)))
>>> print(a)
prints:
[0 x [...]]

To create an object that acts like a "mutable" infinite list, you can overload the __getitem__ and __setitem__ methods in a class:
class infinite_list():
def __init__(self, func):
self.func = func
self.assigned_items = {}
def __getitem__(self, key):
if key in self.assigned_items:
return self.assigned_items[key]
else:
return self.func(key)
def __setitem__(self, key , value):
self.assigned_items[key] = value
Then, you can initialize the "infinite list" with a lambda expression and modify an item in the list:
infinite_thing = infinite_list(lambda a: a*2)
print(infinite_thing[1]) #prints "2"
infinite_thing[1] = infinite_thing[2]
print(infinite_thing[1]) #prints "4"
Similarly, it is possible to create an "infinite dictionary" that provides a default value for each missing key.

Perhaps the natural way to generate an infinite series is using a generator:
def arith(a, d):
while True:
yield a
a += d
This can be used like so:
print list(itertools.islice(arith(10, 2), 100))

My solution is:
from hofs import *
def cons_stream(head,tail):
return [head,tail,False,False]
def stream_cdr(strm):
if strm[2]:
return strm[3]
strm[3] = strm[1]()
strm[2] = True
return strm[3]
def show_stream(stream, num = 10):
if empty(stream):
return []
if num == 0:
return []
return adjoin(stream[0], show_stream(stream_cdr(stream), num - 1))
def add_streams(a , b):
if empty(a):
return b
if empty(b):
return a
return cons_stream(a[0] + b[0] , lambda : add_streams( stream_cdr(a), stream_cdr(b)))
def stream_filter( pred , stream ):
if empty(stream):
return []
if pred(stream[0]):
return cons_stream(stream[0], lambda : stream_filter(pred, stream_cdr(stream)))
else:
return stream_filter( pred , stream_cdr( stream ))
def sieve(stream):
return cons_stream(stream[0] , lambda : sieve(stream_filter(lambda x : x % stream[0] > 0 , stream_cdr(stream))))
ones = cons_stream(1, lambda : ones)
integers = cons_stream(1, lambda : add_streams(ones, integers))
primes = sieve(stream_cdr(integers))
print(show_stream(primes))
Copy the Python code above.
When I tried it, i got [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] which is 10 of an infinite list of primes.

You need hofs.py to be
def empty(data):
return data == []
def adjoin(value,data):
result = [value]
result.extend(data)
return result
def map(func, data):
if empty(data):
return []
else:
return adjoin(func(data[0]), map(func, data[1:]))
def keep(pred, data):
if empty(data):
return []
elif pred(data[0]):
return adjoin( data[0] , keep(pred, data[1:]))
else:
return keep(pred, data[1:])

I assume you want a list of infinite numbers within a range. I have a similar problem, and here is my solution:
c = 0
step = 0.0001 # the difference between the numbers
limit = 100 # The upper limit
myInfList = []
while c <= limit:
myInfList.append(c)
c = c + step
print(myInfList)