Populate a column ONLY if there is a value on another column - python

I am trying to get a specific value in one column only if the value on the other column is not blank. My current code returns the value for all the column regardless of if its blank or not.
Minimal reproducible example:
df=pd.DataFrame(
{
'col1': [12, 34, 54, 23, 12, 43, 53, 12, 43, 12],
'col2': ['USD', 'CAD', 'USD', 'USD', 'CAD', 'USD', 'USD',
'EURO', 'USD', 'USD'],
}
)
Here's my code:
# Set date/time reply and empty column
def add_days_to_date(date, days):
subtracted_date = pd.to_datetime(date) + timedelta(days=days)
subtracted_date = subtracted_date.strftime("%m-%d")
return subtracted_date
def replied_sent_date(date):
return f"Opened {date} need update {add_days_to_date(date, 1)}"
date = datetime.date.today()
date_format = date.strftime('%m-%d')
df['col3'] = df.apply(lambda _:" ", axis=1)
df.loc[df['col1'] != '', 'col3'] = f"Opened {date_format} will need an update {add_days_to_date(date, 30)}"
Desired output:
df=pd.DataFrame(
{
'col1': [12, 34, 54, 23, 12, 43, 53, 12, 43, 12],
'col2': ['USD', 'CAD', 'USD', 'USD', 'CAD', 'USD', 'USD',
'EURO', 'USD', 'USD'],
'col3': [*Look at note below*]
}
)
** "col3" would output "f"Opened {date_format} will need an update {add_days_to_date(date, 30)}" for every cell in which there exists a value for col1.
Any help is appreciated, thanks.

Related

Python: How to validate and append non-existing row in a dataset/dataframe?

How can we append a non-existing row/value in a dataset? I have here a sample table with list of names and the objective is to validate first the name if this doesn't exist and append it to the dataset.
Please see code below for reference:
import pandas as pd
df = pd.DataFrame.from_dict({
'Name': ['Nik', 'Kate', 'Evan', 'Kyra'],
'Age': [31, 30, 40, 33],
'Location': ['Toronto', 'London', 'Kingston', 'Hamilton']
})
df = df.append({'Name':'Jane', 'Age':25, 'Location':'Madrid'}, ignore_index=True)
print(df)

you can check the condition before insering in the dataframe :
import pandas as pd
df = pd.DataFrame.from_dict({
'Name': ['Nik', 'Kate', 'Evan', 'Kyra'],
'Age': [31, 30, 40, 33],
'Location': ['Toronto', 'London', 'Kingston', 'Hamilton']
})
if 'Jane' not in df.Name.values:
df = df.append({'Name':'Jane', 'Age':25, 'Location':'Madrid'}, ignore_index=True)
print(df)

Complete a pandas data frame with values from other data frames

I have 3 data frames. I need to enrich the data from df with the data columns from df2 and df3 so that df ends up with the columns 'Code', 'Quantity', 'Payment', 'Date', 'Name', 'Size', 'Product','product_id', 'Sector'.
The codes that are in df and not in df2 OR df3, need to receive "unknown" for the string columns and "0" for the numeric dtype columns
import pandas as pd
data = {'Code': [356, 177, 395, 879, 952, 999],
'Quantity': [20, 21, 19, 18, 15, 10],
'Payment': [173.78, 253.79, 158.99, 400, 500, 500],
'Date': ['2022-06-01', '2022-09-01','2022-08-01','2022-07-03', '2022-06-09', '2022-06-09']
}
df = pd.DataFrame(data)
df['Date']= pd.to_datetime(df['Date'])
data2 = {'Code': [356, 177, 395, 893, 697, 689, 687],
'Name': ['John', 'Mary', 'Ann', 'Mike', 'Bill', 'Joana', 'Linda'],
'Product': ['RRR', 'RRT', 'NGF', 'TRA', 'FRT', 'RTW', 'POU'],
'product_id': [189, 188, 16, 36, 59, 75, 55],
'Size': [1, 1, 3, 4, 5, 4, 7],
}
df2 = pd.DataFrame(data2)
data3 = {'Code': [879, 356, 389, 395, 893, 697, 689, 978],
'Name': ['Mark', 'John', 'Marry', 'Ann', 'Mike', 'Bill', 'Joana', 'James'],
'Product': ['TTT', 'RRR', 'RRT', 'NGF', 'TRA', 'FRT', 'RTW', 'DTS'],
'product_id': [988, 189, 188, 16, 36, 59, 75, 66],
'Sector': ['rt' , 'dx', 'sx', 'da', 'sa','sd','ld', 'pc'],
}
df3 = pd.DataFrame(data3)
I was using the following code to obtain the unknown codes by comparing with df2, but now i have to compare with df3 also and also add the data from the columns ['Name', 'Size', 'Product','product_id', 'Sector'].
common = df2.merge(df,on=['Code'])
new_codes = df[(~df['Code'].isin(common['Code']))]

How to filter list of dictionaries in python?

I have a list of dictionaries which is as follow-
VehicleList = [
{
'id': '1',
'VehicleType': 'Car',
'CreationDate': datetime.datetime(2021, 12, 10, 16, 9, 44, 872000)
},
{
'id': '2',
'VehicleType': 'Bike',
'CreationDate': datetime.datetime(2021, 12, 15, 11, 8, 21, 612000)
},
{
'id': '3',
'VehicleType': 'Truck',
'CreationDate': datetime.datetime(2021, 9, 13, 10, 1, 50, 350095)
},
{
'id': '4',
'VehicleType': 'Bike',
'CreationDate': datetime.datetime(2021, 12, 10, 21, 1, 00, 300012)
},
{
'id': '5',
'VehicleType': 'Car',
'CreationDate': datetime.datetime(2021, 12, 21, 10, 1, 50, 600095)
}
]
How can I get a list of the latest vehicles for each 'VehicleType' based on their 'CreationDate'?
I expect something like this-
latestVehicles = [
{
'id': '5',
'VehicleType': 'Car',
'CreationDate': datetime.datetime(2021, 12, 21, 10, 1, 50, 600095)
},
{
'id': '2',
'VehicleType': 'Bike',
'CreationDate': datetime.datetime(2021, 12, 15, 11, 8, 21, 612000)
},
{
'id': '3',
'VehicleType': 'Truck',
'CreationDate': datetime.datetime(2021, 9, 13, 10, 1, 50, 350095)
}
]
I tried separating out each dictionary based on their 'VehicleType' into different lists and then picking up the latest one.
I believe there might be a more optimal way to do this.

Use a dictionary mapping from VehicleType value to the dictionary you want in your final list. Compare the date of each item in the input list with the one your dict, and keep the later one.
latest_dict = {}
for vehicle in VehicleList:
t = vehicle['VehicleType']
if t not in latest_dict or vehicle['CreationDate'] > latest_dict[t]['CreationDate']:
latest_dict[t] = vehicle
latestVehicles = list(latest_dict.values())

Here is a solution using max and filter:
VehicleLatest = [
max(
filter(lambda _: _["VehicleType"] == t, VehicleList),
key=lambda _: _["CreationDate"]
) for t in {_["VehicleType"] for _ in VehicleList}
]
Result
print(VehicleLatest)
# [{'id': '2', 'VehicleType': 'Bike', 'CreationDate': datetime.datetime(2021, 12, 15, 11, 8, 21, 612000)}, {'id': '3', 'VehicleType': 'Truck', 'CreationDate': datetime.datetime(2021, 9, 13, 10, 1, 50, 350095)}, {'id': '5', 'VehicleType': 'Car', 'CreationDate': datetime.datetime(2021, 12, 21, 10, 1, 50, 600095)}]

I think you can acheive what you want using the groupby function from itertools.
from itertools import groupby
# entries sorted according to the key we wish to groupby: 'VehicleType'
VehicleList = sorted(VehicleList, key=lambda x: x["VehicleType"])
latestVehicles = []
# Then the elements are grouped.
for k, v in groupby(VehicleList, lambda x: x["VehicleType"]):
# We then append to latestVehicles the 0th entry of the
# grouped elements after sorting according to the 'CreationDate'
latestVehicles.append(sorted(list(v), key=lambda x: x["CreationDate"], reverse=True)[0])

Sort by 'VehicleType' and 'CreationDate', then create a dictionary from 'VehicleType' and vehicle to get the latest vehicle for each type:
VehicleList.sort(key=lambda x: (x.get('VehicleType'), x.get('CreationDate')))
out = list(dict(zip([item.get('VehicleType') for item in VehicleList], VehicleList)).values())
Output:
[{'id': '2',
'VehicleType': 'Bike',
'CreationDate': datetime.datetime(2021, 12, 15, 11, 8, 21, 612000)},
{'id': '5',
'VehicleType': 'Car',
'CreationDate': datetime.datetime(2021, 12, 21, 10, 1, 50, 600095)},
{'id': '3',
'VehicleType': 'Truck',
'CreationDate': datetime.datetime(2021, 9, 13, 10, 1, 50, 350095)}]

This is very straightforwards in pandas. First load the list of dicts as a pandas dataframe, then sort the values by date, take the top n items (3 in the example below), and export to dict.
import pandas as pd
df = pd.DataFrame(VehicleList)
df.sort_values('CreationDate', ascending=False).head(3).to_dict(orient='records')

You can use the operator to achieve that goal:
import operator
my_sorted_list_by_type_and_date = sorted(VehicleList, key=operator.itemgetter('VehicleType', 'CreationDate'))

A small plea for more readable code:
from operator import itemgetter
from itertools import groupby
vtkey = itemgetter('VehicleType')
cdkey = itemgetter('CreationDate')
latest = [
# Get latest from each group.
max(vs, key = cdkey)
# Sort and group by VehicleType.
for g, vs in groupby(sorted(vehicles, key = vtkey), vtkey)
]

A variation on Blckknght's answer using defaultdict to avoid the long if condition:
from collections import defaultdict
import datetime
from operator import itemgetter
latest_dict = defaultdict(lambda: {'CreationDate': datetime.datetime.min})
for vehicle in VehicleList:
t = vehicle['VehicleType']
latest_dict[t] = max(vehicle, latest_dict[t], key=itemgetter('CreationDate'))
latestVehicles = list(latest_dict.values())
latestVehicles:
[{'id': '5', 'VehicleType': 'Car', 'CreationDate': datetime.datetime(2021, 12, 21, 10, 1, 50, 600095)},
{'id': '2', 'VehicleType': 'Bike', 'CreationDate': datetime.datetime(2021, 12, 15, 11, 8, 21, 612000)},
{'id': '3', 'VehicleType': 'Truck', 'CreationDate': datetime.datetime(2021, 9, 13, 10, 1, 50, 350095)}]

Create subgroups based on dateTime

So I have a list of objects within a list, over 600~ of them.
I have a single object example here:
{'Description': '', 'Encrypted': False, 'OwnerId': '', 'Progress': '100%', 'SnapshotId': '', 'StartTime': datetime.datetime(2021, 7, 16, 22, 47, 50, 383000, tzinfo=tzlocal()) }
The problem is that I have a list/array of these and I want to group them all into subgroups such that each object is grouped into a group with others that are timestamped with the "StartTime" datetime that are within 5 minutes of each other. I've been working on this for over a week and I have no idea how to do this grouping. After I group them, I need to apply some rules to each group to ensure they have the correct tags and information.
Just for reference, these are snapshot objects created by amazon aws boto3 describe_snapshots method. You can read about them here: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_snapshots

You can use pandas for this to group the dataframe with pd.Grouper(key='StartTime', freq='5min'):
import pandas as pd
import datetime
from dateutil.tz import tzlocal
data = [{'Description': '', 'Encrypted': False, 'OwnerId': '', 'Progress': '100%', 'SnapshotId': '', 'StartTime': datetime.datetime(2021, 7, 16, 22, 47, 50, 383000, tzinfo=tzlocal()) },{'Description': '', 'Encrypted': False, 'OwnerId': '', 'Progress': '100%', 'SnapshotId': '', 'StartTime': datetime.datetime(2021, 7, 16, 22, 48, 50, 383000, tzinfo=tzlocal()) },{'Description': '', 'Encrypted': False, 'OwnerId': '', 'Progress': '100%', 'SnapshotId': '', 'StartTime': datetime.datetime(2021, 7, 16, 22, 58, 50, 383000, tzinfo=tzlocal()) },{'Description': '', 'Encrypted': False, 'OwnerId': '', 'Progress': '100%', 'SnapshotId': '', 'StartTime': datetime.datetime(2021, 7, 16, 22, 59, 50, 383000, tzinfo=tzlocal()) }]
df = pd.DataFrame(data)
df_grouped = df.groupby(pd.Grouper(key='StartTime', freq='5min'))
Or you could create an extra row in the original dataframe with the number of the group. Eg:
def bin_number(table):
table['bin'] = list(df_grouped.groups.keys()).index(table.name)
return table
df_grouped = df.groupby(pd.Grouper(key='StartTime', freq='5min'), as_index=False)
df_grouped = df_grouped.apply(bin_number).reset_index()
Output:
index
Description
Encrypted
OwnerId
Progress
SnapshotId
StartTime
bin
0
0
False
100%
2021-07-16 22:47:50.383000+02:00
0
1
1
False
100%
2021-07-16 22:48:50.383000+02:00
0
2
2
False
100%
2021-07-16 22:58:50.383000+02:00
2
3
3
False
100%
2021-07-16 22:59:50.383000+02:00
2

Reorder strings within object based on another data frame lookup

This is my first data frame (df1).
I have to reorder the elements in sequence column (of my df1) based on the minimum count from the second data frame (df2)
so my end results should be like this

I think the code below would do what you want. I've commented inline and put links for further reading...
I know it could be compressed into shorter code, but I wanted the steps to be clear.
import pandas as pd
from pprint import pprint
data1 = {'id': ['A1234', 'A2345'],
'Sequence': ['16 31 17', '51 59 43']}
df1 = pd.DataFrame(data1)
# I assumed the label en count columns are integers
data2 = {'label': [10, 11, 12, 13, 16, 17, 21, 24, 31, 43, 44, 51, 59, 60],
'count': [214, 128, 135, 37, 184, 68, 267, 264, 231, 13, 82, 100, 99, 92]}
df2 = pd.DataFrame(data2)
def seq_value_sort(seq_df, label_df):
new_sequence_list = []
for value in seq_df['Sequence'].values:
print(f'{"":-<40}') # prints a line
# convert string to list of integers
# https://www.geeksforgeeks.org/python-converting-all-strings-in-list-to-integers/
sequence = [int(i) for i in value.split()]
# generate an unsorted list of dict items based on Sequence
data = []
for index, row in label_df.T.iteritems():
if int(row['label']) in sequence:
data.append({'label': int(row['label']),
'count': int(row['count'])})
pprint(data)
# now sort the unsorted list based on key 'count'
# https://stackoverflow.com/a/73050/9267296
data = sorted(data, key=lambda k: k['count'])
pprint(data)
# list comprehension to make list of strings out
# of the list of dict
# https://stackoverflow.com/a/7271523/9267296
sequence_sorted = [ str(item['label']) for item in data ]
pprint(sequence_sorted)
# create the final sequence string from the list
new_sequence_list.append(' '.join(sequence_sorted))
# create return data
return_data = {'id': list(seq_df['id'].values),
'Sequence': new_sequence_list}
pprint(return_data)
# finally return a new df
return pd.DataFrame(return_data)
df3 = seq_value_sort(df1, df2)
print(f'{"":-<40}')
print(df3)
EDIT:
Forgot the output:
----------------------------------------
[{'count': 184, 'label': 16},
{'count': 68, 'label': 17},
{'count': 231, 'label': 31}]
[{'count': 68, 'label': 17},
{'count': 184, 'label': 16},
{'count': 231, 'label': 31}]
['17', '16', '31']
----------------------------------------
[{'count': 13, 'label': 43},
{'count': 100, 'label': 51},
{'count': 99, 'label': 59}]
[{'count': 13, 'label': 43},
{'count': 99, 'label': 59},
{'count': 100, 'label': 51}]
['43', '59', '51']
{'Sequence': ['17 16 31', '43 59 51'], 'id': ['A1234', 'A2345']}
----------------------------------------
id Sequence
0 A1234 17 16 31
1 A2345 43 59 51

Categories