We Keep Coding

Is there any way to combine time and date in python using pandas?

I have 2 functions which gives output as date and time
def jdtodatestd (jdate):
if len(jdate) == 5:
fmt = '%y%j'
datestd = datetime.datetime.strptime(jdate, fmt).date()
return(datestd)
elif len(jdate) == 6:
yr = jdate[0:2]
day = jdate[2:len(jdate)]
day = day.lstrip('0')
jdate = yr+day
fmt = '%y%j'
datestd = datetime.datetime.strptime(jdate, fmt).date()
return(datestd)
elif len(jdate) == 7:
fmt = '%Y%j'
datestd = datetime.datetime.strptime(jdate, fmt).date()
return(datestd)
jdtodatestd('120365')
Output: datetime.date(2012, 12, 30)
def jdtotimestd (jtime):
if len(jtime) == 5:
jtime = '0' + jtime
elif len(jtime) == 6:
jtime = jtime
else:
jtime = '000000'
stdtime = jtime[0:2] + ':' + jtime[2:4] + ':' + jtime[4:6]
return stdtime
jdtotimestd('140932')
Output: '14:09:32'
I would like to combine both such as '2012, 12, 30 14:09:32
How can I do?

Modify your function jdtotimestd:
def jdtotimestd(jtime):
"""Returns a dt.time object from string."""
jtime = jtime.zfill(6)
return datetime.time(int(jtime[:2]), int(jtime[2:4]), int(jtime[4:]))
d = jdtodatestd('120365')
t = jdtotimestd('140932')
dt = datetime.datetime.combine(d, t)
Output:
>>> dt
datetime.datetime(2012, 12, 30, 14, 9, 32)

You can use strftime and strptime:
output1 = jdtodatestd('120365')
output2 = jdtotimestd('140932')
>>> datetime.datetime.strptime(datetime.datetime.strftime(output1, "%Y-%m-%d")+output2, "%Y-%m-%d%H:%M:%S")
datetime.datetime(2012, 12, 30, 14, 9, 32)

Getting KeyError when assigning value with __getitem__ method

I want to realize the bert model.
So I built a class with __getitem__ in it.
I can print something like test[0], but when I assign a value, like data = test[0], a KeyError occurs.
import random
"""
corpus_file = 'vocab'
vocab_size = 6
vocab_freq = 1
save_path = 'obj/'
max_sentence = 16
corpus -> org_line -> ope_line
corpus -> org_line -> token_list -> idx_to_token + token_to_idx
"""
class vocab():
def __init__(self, corpus_file, vocab_size, vocab_freq,save_path,max_sentence):
self.max_sentence = max_sentence
self.special_labels = ['PAD', 'UNK', 'SEP', 'CLS', 'MASK']
# output
self.data = []
self.idx_to_token = []
self.token_to_idx = {}
# ope
self.pre_ope(corpus_file,vocab_size,vocab_freq)
#self.save_data(save_path)
#self.print_data()
def pre_ope(self,corpus_file,vocab_size,vocab_freq):
token_list = {}
with open(corpus_file, 'r') as f:
while 1:
new_org_line = f.readline()
if new_org_line != '':
new_org_line = new_org_line.strip('\n')
new_sentence = new_org_line.split('\t')
sentence = []
for tmp in new_sentence:
token_sentence = tmp.split()
sentence.append(token_sentence)
for token in token_sentence:
if token_list.get(token):
token_list[token] += 1
else:
new_token = {token: 1}
token_list.update(new_token)
self.data.append(sentence)
else:
break
f.close()
token_list = sorted(token_list.items(), key=lambda i: (-i[1], i[0]))
self.build_dictionary(token_list,vocab_freq,vocab_size)
'''
Special labels:
PAD
UNK
SEP sentence separator
CLS classifier token
MASK
'''
def build_dictionary(self,token_list,vocab_freq,vocab_size):
for idx, label in enumerate(self.special_labels):
self.idx_to_token.append(label)
self.token_to_idx[label] = idx
for idx, (token, freq) in enumerate(token_list):
if freq >= vocab_freq :
self.idx_to_token.append(token)
self.token_to_idx[token] = idx + len(self.special_labels)
if len(self.idx_to_token) >= vocab_size + len(self.special_labels) and vocab_size != 0 :
break
def __len__(self):
return len(self.data)
def print_data(self):
print(self.data)
print(self.idx_to_token)
print(self.token_to_idx)
def __getitem__(self, item):
s1,s2,is_next_sentence = self.get_random_next_sentence(item)
s1,s1_label = self.get_random_sentence(s1)
s2,s2_label = self.get_random_sentence(s2)
sentence = [self.token_to_idx['CLS']] +s1 +[self.token_to_idx['SEP']] +s2 +[self.token_to_idx['SEP']]
label = [-1] +s1_label +[-1] +s2_label +[-1]
if len(sentence) > self.max_sentence :
print('sentence is greater than the setting of max sentence')
for pos in range(len(sentence),self.max_sentence):
sentence.append(self.token_to_idx['PAD'])
label.append(-1)
return {
'token' : sentence,
'label' : label,
'is_next' : is_next_sentence
}
def get_random_next_sentence(self,item):
s1 = self.data[item][0]
s2 = self.data[item][1]
if random.random() < 0.5 :
is_next = 0
s2 = self.data[self.get_random_line(item)][1]
else:
is_next = 1
return s1,s2,is_next
def get_random_line(self,item):
rand = random.randint(0,len(self.data)-1)
while rand == item :
rand = random.randint(0,len(self.data)-1)
return rand
def get_random_sentence(self,sentence):
label = []
for idx,token in enumerate(sentence):
rand = random.random()
if rand < 0.15:
rand = rand/0.15
if rand < 0.8: #mask
sentence[idx] = self.token_to_idx['MASK']
elif rand < 0.9: #rand
sentence[idx] = random.randint(len(self.special_labels),len(self.token_to_idx)-1)
else: # still
sentence[idx] = self.token_to_idx[token]
label.append(self.token_to_idx[token])
else:
sentence[idx] = self.token_to_idx[token]
label.append(-1)
return sentence,label
if __name__ == '__main__':
test = vocab('vocab', 0, 1,'obj/',16)
print(len(test))
print(test[0])
print(test[1])
data = test[0]
Result:
2
{'token': [3, 4, 18, 12, 15, 11, 2, 7, 9, 13, 2, 0, 0, 0, 0, 0], 'label': [-1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], 'is_next': 0}
{'token': [3, 6, 4, 5, 8, 5, 17, 2, 16, 5, 14, 20, 2, 0, 0, 0], 'label': [-1, -1, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], 'is_next': 0}
Traceback (most recent call last):
File "vocab.py", line 146, in <module>
data = test[0]
File "vocab.py", line 90, in ```__getitem__```
s1,s1_label = self.get_random_sentence(s1)
File "vocab.py", line 136, in get_random_sentence
sentence[idx] = self.token_to_idx[token]
KeyError: 4
vocab file:
hello this is my home nice to meet you
I want to go to school and have lunch

change the code:
def get_random_next_sentence(self,item):
s1 = self.data[item][0]
s2 = self.data[item][1]
if random.random() < 0.5 :
is_next = 0
s2 = self.data[self.get_random_line(item)][1]
else:
is_next = 1
return s1,s2,is_next
to:
def get_random_next_sentence(self,item):
s1 = copy.deepcopy(self.data[item][0])
s2 = copy.deepcopy(self.data[item][1])
if random.random() < 0.5 :
is_next = 0
s2 = copy.deepcopy(self.data[self.get_random_line(item)][1])
print(s2)
else:
is_next = 1
return s1,s2,is_next

List prints letters and not strings?

I'm having a hard time getting the following to print correctly:
core = 1, 2, 3, 4, 5
glutes = 6, 7, 8, 9, 10
upper = 11, 12, 13, 14, 15
lower = 16, 17, 18, 19, 20
conditioning = 21, 22, 23, 24, 25
core_ability = int(input("Core: "))
glute_ability = int(input("Glutes: "))
if core_ability > 4:
upper_ability = int(input("Upper body: "))
else:
""
lower_ability = int(input("Lower body: "))
conditioning_ability = int(input("\nConditioning ability level:"))
newcore = core[0:core_ability]
newglutes = glutes[0:glute_ability]
if core_ability > 4:
newupper = upper[0:upper_ability]
newlower = lower[0:lower_ability]
newconditioning = conditioning[0:conditioning_ability]
if core_ability > 4:
movement_bank = str(newcore) + str(newglutes) + str(newupper) + str(newlower) + str(conditioning_ability)
else:
movement_bank = str(newcore) + str(newglutes) + str(newlower) + str(conditioning_ability)
sections = int(input("\nNumber of GPP sections in the session: "))
print("\nSPECIFY THE NUMBER OF MOVEMENTS PER SECTION")
if sections == 1:
section1_num = int(input("Section 1:"))
print(random.sample(movement_bank[0:], k=section1_num))
I get an output the looks like:
' ', ' ', 'r'
when I'd like to get something like:
'1', '16', '8'
I added "str()" to each list in the "movement_bank" list because without it I got an error of: TypeError: can only concatenate list (not "int") to list.
All help is greatly appreciated.

It seems, you have different lists, and want to combine them all into one list.
Use extend:
core = 1, 2, 3, 4, 5
glutes = 6, 7, 8, 9, 10
upper = 11, 12, 13, 14, 15
lower = 16, 17, 18, 19, 20
conditioning = 21, 22, 23, 24, 25
movement_bank = []
core_ability = int(input("Core: "))
movement_bank.extend(core[:core_ability])
glute_ability = int(input("Glutes: "))
movement_bank.extend(glutes[:glute_ability])
if core_ability > 4:
upper_ability = int(input("Upper body: "))
movement_bank.extend(upper[:upper_ability])
lower_ability = int(input("Lower body: "))
movement_bank.extend(lower[:lower_ability])
conditioning_ability = int(input("\nConditioning ability level:"))
movement_bank.extend(conditioning[:conditioning_ability])
sections = int(input("\nNumber of GPP sections in the session: "))
print("\nSPECIFY THE NUMBER OF MOVEMENTS PER SECTION")
if sections == 1:
section1_num = int(input("Section 1:"))
print(random.sample(movement_bank, k=section1_num))

How to check for duplicate attributes in object list and merge them

I have a list of objects with attributes qt, cons and consper and have to merge all the objects that have the same consper value. What is the best way to do that? The list is already sorted by consper.
Example:
With a list of objects of the class house:
class house():
def __init__(self, qt, cons, consper):
self.qt = qt
self.cons = cons
self.consper = consper
Turn this list:
l = [
house(2, 20, 10),
house(3, 31, 10),
house(6, 70, 11),
house(2, 40, 20),
house(1, 25, 25)]
Into this list:
l_new = [
house(5, 51, 10),
house(6, 70, 11),
house(2, 40, 20),
house(1, 25, 25)]
By adding the first two objects (because their attribute consper is equivalent)

If the items are already sorted by that attribute, you can use itertools.groupby to get the groups and sum to get the sum for the other attributes. You also have to convert the group to list first, as those are iterators.
>>> from itertools import groupby
>>> house.__repr__ = lambda h: "house(%r, %r, %r)" % (h.qt, h.cons, h.consper)
>>> [house(sum(h.qt for h in g), sum(h.cons for h in g), k)
... for k, g in ((k, list(g)) for k, g in groupby(l, key=lambda h: h.consper))]
[house(5, 51, 10), house(6, 70, 11), house(2, 40, 20), house(1, 25, 25)]
Or using a dictionary:
>>> d = {}
>>> for h in l:
... qt, cons = d.get(h.consper, (0, 0))
... d[h.consper] = qt + h.qt, cons + h.cons
...
>>> [house(a, b, c) for a, (b, c) in d.items()]
[house(25, 1, 25), house(10, 5, 51), house(11, 6, 70), house(20, 2, 40)]

Without using itertools, you could do something like this:
class House():
def __init__(self, qt, cons, consper):
self.qt = qt
self.cons = cons
self.consper = consper
def __str__(self):
return "House(" + str(self.qt) + "," + str(self.cons) + "," + str(self.consper) + ")"
def __repr__(self):
return self.__str__()
def merge_dups(house_list):
res = []
house_map = {}
for h in house_list:
if h.consper in house_map:
other_house = house_map[h.consper]
merged_house = House(h.qt + other_house.qt,
h.cons + other_house.cons,
h.consper)
res.remove(other_house)
res.append(merged_house)
else:
house_map[h.consper] = h
res.append(h)
return res
print(merge_dups([
House(2, 20, 10),
House(3, 31, 10),
House(6, 70, 11),
House(2, 40, 20),
House(1, 25, 25)]))
Output
[House(5,51,10), House(6,70,11), House(2,40,20), House(1,25,25)]

You can use itertools.groupby:
import itertools
class house():
def __init__(self, qt, cons, consper):
self.qt = qt
self.cons = cons
self.consper = consper
def __repr__(self):
return self.__class__.__name__+"({qt}, {cons}, {consper})".format(**self.__dict__)
l = [house(2, 20, 10),
house(3, 31, 10),
house(6, 70, 11),
house(2, 40, 20),
house(1, 25, 25)]
new_l = [(a, [(i.qt, i.cons) for i in list(b)]) for a, b in itertools.groupby(sorted(l, key=lambda x:x.consper), key=lambda x:x.consper)]
final_data = [house(*[sum(i) for i in zip(*b)]+[a]) for a, b in new_l]
Output:
[house(5, 51, 10), house(6, 70, 11), house(2, 40, 20), house(1, 25, 25)]

A simple solution is using a Dictionary as follows:
l = [
house(2, 20, 10),
house(3, 31, 10),
house(6, 70, 11),
house(2, 40, 20),
house(1, 25, 25)]
dic= {}
for x in l :
temp = dic.get(x.consper,house(0,0,0))
x.qt += temp.qt
x.cons += temp.cons
dic[x.consper]=x
print('####################')
for x in dic.keys():
print(x)

Python multiprocessing , code keep on executing?

from multiprocessing import Process , Queue
from datetime import datetime
c = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]
out = Queue()
def support(m):
for k in m :
print "%s <-- hi" % k
out.put("done")
all = Queue()
temp = []
total = len(c)
count = 0
for m in c :
count += 1
total = total - 1
temp.append(m)
if count == 5 or total == 0 :
all.put(temp)
count = 0
temp = []
process_count = 3
while all.qsize() != 0 :
process_list = []
try :
for x in range(process_count) :
p = Process(target=support, args=(all.get(),))
process_list.append(p)
for p in process_list :
p.start()
for p in process_list :
p.join()
except Exception as e :
print e
while out.qsize != 0 :
print out.get()
print "all done"
I dont know why it does not end and does not print "all done" , just remain continuously in loop or keep executing .
Will be of great help if you can make this code more efficient but first i want to know why it does not end .

The problem is:
while out.qsize != 0 :
print out.get()
out.qsize is a function, so now you're comparing the function itself (not the return value!) with 0, with is of course always False.
You should use:
while out.qsize() != 0 :
print out.get()