We Keep Coding

MongoDB: Update element in an array where the index of the element is saved in the document

I have the following document structure.
{
_id: ...,
unique_id: 1234,
config_no: 1,
configs: [
{
data: "qwertyuiop" // random string
},
{
data: "asdfghjkl" // random string
}
]
}
I want to update value of data from one of the configs. The index of the config that needs to be updated is available in the config_no key.
Is there any way to update the value without querying the document.
This is what I am currently doing
doc = db.collection.findOne({"unique_id": 1234})
config_no = doc.config_no
db.collection.updateOne(
{"unique_id": 1234},
{"$set": {"configs."+config_no+".data": "zxcvbnm"}} //"configs.1.data"
)
Following is something what i would like to achive.
db.collection.updateOne(
{"unique_id": 1234},
{"$set": {"configs.${config_no}.data": "zxcvbnm"}}
)

You can $unwind with includeArrayIndex option. Use the index to perform conditional update and $merge back into the collection.
db.collection.aggregate([
{
$match: {
unique_id: 1234
}
},
{
"$unwind": {
path: "$configs",
includeArrayIndex: "idx"
}
},
{
$set: {
"configs.data": {
"$cond": {
"if": {
$eq: [
"$config_no",
"$idx"
]
},
"then": "zxcvbnm",
"else": "$configs.data"
}
}
}
},
{
$group: {
_id: "$_id",
config_no: {
$first: "$config_no"
},
configs: {
$push: "$configs"
},
unique_id: {
$first: "$unique_id"
}
}
},
{
"$merge": {
"into": "collection",
"on": "_id",
"whenMatched": "merge"
}
}
])
Mongo Playground

Aggregation $match within a $sum

I was wondering if it was possible to somehow use the $match operator within the $sum function for aggregation.
{ "$unwind": "$info.avatarInfoList" },
{ "$unwind": "$info.avatarInfoList.equipList" },
{ "$unwind": "$info.avatarInfoList.equipList.flat.reliquarySubstats" },
{
"$project": {
"name" : "$name",
"character" : "$info.avatarInfoList.avatarId",
"artifact" : "$info.avatarInfoList.equipList.itemId",
"statValue" : {
"$sum": [
{"$match" : { "$info.avatarInfoList.equipList.flat.reliquarySubstats.appendPropId" : "FIGHT_PROP_CRITICAL_HURT" } },
{"$multiply": [2, {"$match" : { "$info.avatarInfoList.equipList.flat.reliquarySubstats.appendPropId" : "FIGHT_PROP_CRITICAL" } }]}
]
},
}
},
{ "$sort": { "statValue": -1 }},
{ '$limit' : 30 }
]).to_list(length=None)
print(data)
I want to be able to use the value of the $sum operator within the project fields somehow, I just don't really understand what the right approach would be for this.
Sample Input (may be too long):
https://www.toptal.com/developers/hastebin/ixamekaxoq.json
Sample Output:
( 2 * FIGHT_PROP_CRITICAL ) + FIGHT_PROP_CRITICAL_HURT sorted from highest to lowest for each item.
{name: hat, character: Slayer, artifact: 13, statValue : 25.6}

There are still a few ambiguities about how you want to aggregate your data, but using the full document from your link, here's one way to produce the output you want.
N.B.: Weapons in the "equipList" don't have "reliquarySubstats" so they show a "statValue" of null in the output.
db.collection.aggregate([
{"$unwind": "$info.avatarInfoList"},
{"$unwind": "$info.avatarInfoList.equipList"},
{
"$project": {
"_id": 0,
"name": 1,
"character": "$info.avatarInfoList.avatarId",
"artifact": "$info.avatarInfoList.equipList.itemId",
"statValue": {
"$reduce": {
"input": "$info.avatarInfoList.equipList.flat.reliquarySubstats",
"initialValue": 0,
"in": {
"$switch": {
"branches": [
{
"case": {"$eq": ["$$this.appendPropId", "FIGHT_PROP_CRITICAL"]},
"then": {
"$add": [
"$$value",
{"$multiply": [2, "$$this.statValue"]}
]
}
},
{
"case": {"$eq": ["$$this.appendPropId", "FIGHT_PROP_CRITICAL_HURT"]},
"then": {"$add": ["$$value", "$$this.statValue"]}
}
],
"default": "$$value"
}
}
}
}
}
},
{"$sort": {"statValue": -1}}
])
Try it on mongoplayground.net.

It's not quite clear what you want to achieve, but as mentioned you want to be using $cond here.
like so:
{
"$project": {
"statValue": {
"$sum": [
{
$cond: [
{ // if this condition is true (prop id = prop critical hurt )
$eq: [
"$info.avatarInfoList.equipList.flat.reliquarySubstats.appendPropId",
"FIGHT_PROP_CRITICAL_HURT"
]
},
{ // then use this value for the "$sum"
"$multiply": [
2,
"$info.avatarInfoList.equipList.flat.reliquarySubstats.statValue"
]
},
0 // otherwise use this value for the sum.
]
}
]
}
}
Mongo Playground

Find the longest group after groupby on normalized json in pandas

My code below groups by values and creates a list of values that were once the length of arrays. But how can I return the id that has the largest sum of each number in the elements:
Original Json read into df (not same data as printed because it was too long)
{
"kind":"admin#reports#activities",
"etag":"\"5g8\"",
"nextPageToken":"A:1651795128914034:-4002873813067783265:151219070090:C02f6wppb",
"items":[
{
"kind":"admin#reports#activity",
"id":{
"time":"2022-05-05T23:59:39.421Z",
"uniqueQualifier":"5526793068617678141",
"applicationName":"token",
"customerId":"cds"
},
"etag":"\"jkYcURYoi8\"",
"actor":{
"email":"blah#blah.net",
"profileId":"1323"
},
"ipAddress":"107.178.193.87",
"events":[
{
"type":"auth",
"name":"activity",
"parameters":[
{
"name":"api_name",
"value":"admin"
},
{
"name":"method_name",
"value":"directory.users.list"
},
{
"name":"client_id",
"value":"722230783769-dsta4bi9fkom72qcu0t34aj3qpcoqloq.apps.googleusercontent.com"
},
{
"name":"num_response_bytes",
"intValue":"7158"
},
{
"name":"product_bucket",
"value":"GSUITE_ADMIN"
},
{
"name":"app_name",
"value":"Untitled project"
},
{
"name":"client_type",
"value":"WEB"
}
]
}
]
},
{
"kind":"admin#reports#activity",
"id":{
"time":"2022-05-05T23:58:48.914Z",
"uniqueQualifier":"-4002873813067783265",
"applicationName":"token",
"customerId":"df"
},
"etag":"\"5T53xK7dpLei95RNoKZd9uz5Xb8LJpBJb72fi2HaNYM/9DTdB8t7uixvUbjo4LUEg53_gf0\"",
"actor":{
"email":"blah.blah#bebe.net",
"profileId":"1324"
},
"ipAddress":"54.80.168.30",
"events":[
{
"type":"auth",
"name":"activity",
"parameters":[
{
"name":"api_name",
"value":"gmail"
},
{
"name":"method_name",
"value":"gmail.users.messages.list"
},
{
"name":"client_id",
"value":"927538837578.apps.googleusercontent.com"
},
{
"name":"num_response_bytes",
"intValue":"2"
},
{
"name":"product_bucket",
"value":"GMAIL"
},
{
"name":"client_type",
"value":"WEB"
}
]
}
]
}
]
}
current code:
df = pd.json_normalize(response['items'])
df['test'] = df.groupby('actor.profileId')['events'].apply(lambda x: [len(x.iloc[i][0]['parameters']) for i in range(len(x))])
output:
ID
1002306 [7, 7, 7, 5]
1234444 [3,5,6]
1222222 [1,3,4,5]
desired output
id total
1002306 26
Sorry had to fill up more space, as there was so much code

There’s no need to construct the intermediate df and do groupby on it. You can use pass the record and meta paths to json_normalize to directly flatten the json data. Then your job seems to be about counting the number of rows per actor.profileId and finding the maximum.
df = pd.json_normalize(response['items'], ['events','parameters'], ['actor'])
df['actor.profileId'] = df['actor'].str['profileId']
out = df.value_counts('actor.profileId').pipe(lambda x: x.iloc[[0]])
Output:
actor.profileId
1323 7
dtype: int64

MongoDB - Get SUM of values INSIDE of the array

I have JSON document recorded to MongoDB with structure like so:
[{ "SessionKey": "172e3b6b-509e-4ef3-950c-0c1dc5c83bab",
"Query": {"Date": "2020-03-04"},
"Flights": [
{"LegId":"13235",
"PricingOptions": [
{"Agents": [1963108],
"Price": 61763.64 },
{"Agents": [4035868],
"Price": 62395.83 }]},
{"LegId": "13236",
"PricingOptions": [{
"Agents": [2915951],
"Price": 37188.0}]}
...
The result I'm trying to get is "LegId":"sum_per_flight", in this case -> {'13235': (61763.64+62395.83), '13236': 37188.0} and then get flights with price < N
I've tried to run this pipeline for aggregation step (but it returns list of ALL prices - I don't know how to sum them up properly):
result = collection.aggregate([
{'$match': {'Query.Date': '2020-03-01'}},
{'$group': {'_id': {'Flight':'$Flights.LegId', 'Price':'$Flights.PricingOptions.Price'}}} ])
Also I've tried this pipeline, but it returns 0 for 'total_price_per_flight':
result = collection.aggregate({'$project': {
'Flights.LegId':1,
'total_price_per_flight': {'$sum': '$Flights.PricingOptions.Price'}
}})

You need to use $unwind to flatten Flights array to able iterate individually.
With $reduce operator, we iterate PricingOptions array and sum Price fields (accumulate prices).
The last step we return your documents into original structure. Before that, you may apply "get flights with price < N"
db.collection.aggregate([
{
"$match": {
"Query.Date": "2020-03-04"
}
},
{
$unwind: "$Flights"
},
{
$addFields: {
"Flights.LegId": {
$arrayToObject: [
[
{
k: "$Flights.LegId",
v: {
$reduce: {
input: "$Flights.PricingOptions",
initialValue: 0,
in: {
$add: [
"$$value",
"$$this.Price"
]
}
}
}
}
]
]
}
}
},
{
$group: {
_id: "$_id",
SessionKey: {
$first: "$SessionKey"
},
Query: {
$first: "$Query"
},
Flights: {
$push: "$Flights"
}
}
}
])
MongoPlayground

How to merge json objects containing arrays using python?

I have two json files which contain all kinds of levels of properties. I want to write a python script that will replace existing properties and add missing ones, but keep all the other ones in place.
In my attempts until now the entire "configurations" array of the original file is overwritten, including all properties. All examples I could find show merge for objects without arrays. Any help would be appreciated.
Original:
{
"configurations": [
{
"this-needs-to-stay": {
"properties": {
"some_property": "EXISTING"
}
}
},
{
"this-needs-to-be-updated": {
"properties": {
"this.would.stay": "EXISTING",
"this.wont.be.overwritten": "EXISTING"
}
}
}
],
"other-values-1": [
{
"components": [
{
"name": "EXISTING"
}
],
"name": "somename"
}
],
"other-values-2": {
"randomProperties": {
"type": "random"
},
"and_so_on": "you_get_the_point"
}
}
Additional data that should be added to original:
{
"configurations" : [
{
"this-would-be-added": {
"properties": {
"some-property": "ADDED"
}
}
},
{
"this-needs-to-be-updated": {
"properties": {
"this.would.stay": "CHANGED",
"this.would.be.added": "ADDED"
}
}
}
]
}
Result is a merging of the two on the property level:
{
"configurations": [
{
"this-would-be-added": {
"properties": {
"some-property": "ADDED"
}
}
},
{
"this-needs-to-stay": {
"properties": {
"some_property": "EXISTING"
}
}
},
{
"this-needs-to-be-updated": {
"properties": {
"this.would.stay": "CHANGED",
"this.would.be.added": "ADDED"
"this.wont.be.overwritten": "EXISTING"
}
}
}
],
"other-values-1": [
{
"components": [
{
"name": "EXISTING"
}
],
"name": "somename"
}
],
"other-values-2": {
"randomProperties": {
"type": "random"
},
"and_so_on": "you_get_the_point"
}
}

Using funcy.merge:
from funcy import merge
x, y = map(lambda d: {hash(frozenset(c.keys())):c for c in d}, (a['configurations'], b['configurations']))
merged = list(merge(x, y).values())
print(json.dumps(merged, indent=4))
Result:
[
{
"this-needs-to-stay": {
"properties": {
"some_property": "EXISTING"
}
}
},
{
"this-needs-to-be-updated": {
"properties": {
"this.would.stay": "CHANGED",
"this.would.be.added": "ADDED"
}
}
},
{
"this-would-be-added": {
"properties": {
"some-property": "ADDED"
}
}
}
]

In the items of configurations in you sample data, looks like you are using items' only key as a unique key in the array. Therefore, we can convert the list into a dict by using that unique key.
That is turning
[{"ID_1": "VALUE_1"}, {"ID_2": "VALUE_2"}]
into {"ID_1": "VALUE_1", "ID_2": "VALUE_2"}
Then, we just want to merge those two dict. Here I use {**a, **b} to merge them. For this part, you can take a look at How to merge two dictionaries in a single expression?
So
{"ID_1": "value_1", "ID_2": "value_2"}
and
{"ID_2": "new_value_2", "ID_3": "new_value_3"}
would be merged as
{"ID_1": "value_1", "ID_2": "new_value_2", "ID_3": "new_value_3"}
Once they are merged, convert the result dict back into list and that's the final result.
[{"ID_1": "value_1"}, {"ID_2": "new_value_2"}, {"ID_3": "new_value_3"}]
Codes:
def list_to_dict(l):
return {list(item.keys())[0]: list(item.values())[0] for item in l}
def list_item_merge(a, b):
return [{k: v} for k, v in {**list_to_dict(a), **list_to_dict(b)}.items()]
list_item_merge(original['configurations'], additional['configurations'])

I would suggest reviewing your conf structure. A list of dicts with single key doesn't make sense to me. Why not just use a dict?:
{
"configurations": {
"this-needs-to-stay": {
"properties": {
"some_property": "EXISTING"
}
},
"this-needs-to-be-updated": {
"properties": {
"this.would.stay": "EXISTING",
"this.wont.be.overwritten": "EXISTING"
}
}
},
# ...
}
Then you can simply use:
from funcy import merge
conf = base
conf['configurations'] = merge(base['configurations'],
new['configurations'])