I wrote a method to handle mongodb result, where date is coming as datetime.datetime() i used dumps method, which converts the date is not milliseconds, here if the date is before 1970 then date is converted to negative value and i am unable to handle this to change back to date and time after wards.
My sample code is as below:
import datetime
from bson.json_util import dumps
from bson.objectid import ObjectId
import string
def get_json_key_value(mongoResult, key):
# converts mongoResult which is not into proper json format to proper json format
result = dumps(dict(eval(mongoResult)))
# convert unicode format to string format
data = __convert(dict(eval(result)))
# code to get the json value.
value="data"
jsonlist = key.split('.')
for eachKey in jsonlist:
if (eachKey.isdigit()):
value = value + "["+eachKey+ "]"
else:
value = value + "['"+eachKey + "']"
returnValue = eval(value)
#processing the date value, if key has $date
if((string.find(key,"$date"))>=0):
returnValue = datetime.datetime.fromtimestamp(returnValue/1000.0)
returnValue = datetime.datetime.strftime(returnValue , format="%Y-%m-%dT%H:%M:%S")
returnValue = datetime.datetime.strptime(str(returnValue), "%Y-%m-%dT%H:%M:%S")
returnValue = returnValue - datetime.timedelta(minutes=330)
returnValue = datetime.datetime.strftime(returnValue , format="%Y-%m-%dT%H:%M:%S")
return returnValue
sample input data can be as
MongoResult: {'profileDetails': {'basicDetails': {'dateOfBirth': {'$date': -414892800000L}, 'customerCode': 'C017145'}, 'xDirLevel': {'masterCode': 1}, 'createdDate': {'$date': 1467392480962L}}, '_id': {'$oid': '58872e98321a0c863329199d'}}
Key: profileDetails.basicDetails.dateOfBirth.$date
I am getting error ValueError: timestamp out of range for platform localtime()/gmtime() function if the date is before 1970, how to handle this
I got the solution for date convertion before 1970 as below:
need to replace the code in if((string.find(key,"$date"))>=0): block as
ndate = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=(returnValue)/1000)
returnValue = str(ndate).replace(' ','T')
I got the solution from the another question in stackoverflow Timestamp out of range for platform localtime()/gmtime() function
Related
I am reading timestamp strings of format '2014-11-09T01:00Z' from mongo and storing in a python dict. But while retrieving same from the dict I see some of the strings got automatically converted to proto timestamp like seconds: 1511047800.
I have no clue how could this happen. Any insights appreciated.
class SomeInfo:
__school_id_info = defaultdict(
lambda: {'banned': -1,
'school_boy': False,
'teacher': False})
def __init__(self):
mongo_connection_str = os.environ['MONGO_CONNECTION_STRING']
mongo_db_name = os.environ.get('MONGO_DB_NAME', 'coll')
mongo_dao = MongoDao(mongo_connection_str, mongo_db_name)
mongo_collection_name = 'school'
school_records = mongo_dao.find_records(mongo_collection_name)
for school_details in school_records:
self.__school_id_info['date'] = school_details['date']
#If I print the __school_id_info here I see date as '2014-11-09T01:00Z' string format which is same as how it is in Mongo.
self.__dict__ = self.__school_id_info
def get_info_for_both_students(self) -> dict:
#while returning from here I see some as seconds: 1235192400 protobuff timestamp format and some still as it is
# .I am not doing any conversion to protobuf timestamp
return self.__school_id_info
my dynamodb table has timestamp(in YYYY-MM-DD HH:MN:SS) as PrimaryKey column and temperature as sortkey while in data {"humidity" : 42 ,"location":"room" , "temperature":,"thermostat":}
in boto3 python i need to scan based on timestamp (now and 15min ago) with condition if difference(temperature - thermostat) > 5 for more than 10 times then return thermostat-5 and if (temperature - thermostat) < 5 for more than 10 times then returns thermostat+5... following is the code
import boto3
import math
import json
import time
import dateutil.tz
from datetime import datetime,timedelta
from dateutil import tz
from dateutil.tz import tzlocal
from boto3.dynamodb.conditions import Key, Attr
client = boto3.client('dynamodb')
dynamodb = boto3.resource('dynamodb')
def lambda_handler(event, context):
#table_name= "thermostat_dynamo"
table_name= "TableDynamo"
Primary_Column_Name = 'timestamp'
table = dynamodb.Table(table_name)
#key_param = "thermostat"
#thermostatVal = table.get_item(Key={key_param:event[key_param]}) ## get record from dynamodb for this sensor
thermostatVal= 77
south = dateutil.tz.gettz('Asia/Kolkata')
now = datetime.now(tz=south)
fifteen_min_ago = now - timedelta(seconds=900)
now = now.strftime('%F %T')
fifteen_min_ago = fifteen_min_ago.strftime('%F %T')
fe = Key('timeStamp').between(fifteen_min_ago,now);
response = table.scan(FilterExpression=fe & Attr('temperature').lt(thermostatVal))
if response['Count'] == 10:
#return thermostatVal+5
thermonew = thermostatVal + 5
tosensor = '{"thermostat":'+ str(thermonew) + '}'
print(tosensor)
#response = client.publish(topic="updatehomesensor", qos=1, payload=tosensor)
return
elif response['Count'] < 10:
print('{"thermostat":' + str(thermostatVal) + '}')
return
If timestamp was a sort key, you could have used a Query request to scan through all the items with timestamp > now-15min.
However, unfortunately, timestamp is your hash key. The only way you can find the items with timestamp > now-15min is to Scan through all your items. This will cost you a lot of money: You pay Amazon for each item scanned, not each item returned after the filtering.
Another problem is that the DynamoDB filtering syntax (look at the FilterExpression documentation) doesn't actually allow to do addition and subtractions as part of the test. If you always want to do "temperature - thermostat", you can use that as one of the attributes (so you can do a FilterExpression on it), and the second attribute would be "thermostat", and later you can add the two up to get the "temperature".
I want to query MongoDB and return records that have the date of 12/6/2017. The date is in this format:
u'Date': datetime.datetime(2017, 12, 6, 15, 9, 21, 303000)
So, how do I query just the year, month, and day of that entry? I have tried:
date = db.data.find({ 'Date' : {'2017, 12, 6'} },{'Date' : '1'})
for document in date:
print(date)
Which returns: "InvalidDocument: Cannot encode object: set(['2017, 12, 6'])".
I also tried:
date = db.data.find({ 'Date' : {datetime('2017, 12, 6')} },{'Date' : '1'})
for document in date:
print(date)
Which returns: "NameError: name 'datetime' is not defined".
UPDATE...SOLUTION
I was putting the date into Mongo incorrectly. I'm now putting the date into Mongo with Python like this:
import datetime
import dateutil.parser
# the date
now = datetime.datetime.now()
year = now.year
month = now.month
day = now.day
theDate = str(year) + "-" + str(month) + "-" + str(day)
dateStr = theDate
date = dateutil.parser.parse(dateStr)
# Then put that date into your Mongo insert
This is how I'm querying by date. This pulls documents inserted after yersterday (today).
import dateutil.parser, datetime
now = datetime.datetime.now()
year = now.year
month = now.month
yesterday = now.day - 1
dateStr = str(year) + "-" + str(month) + "-" + str(yesterday)
date = dateutil.parser.parse(dateStr)
cursor = db.data.find({'Date' : { '$gt' : date}},{'Date':'1'})
for document in cursor:
print(document)
When you say the object is stored as datetime.datetime, to what are you referring? A custom object?
Per the Mongo docs, this is the only date object they support explicity:
https://docs.mongodb.com/v3.4/reference/method/Date/
From the Docs:
Date() Returns a date either as a string or as a Date object.
Date() returns the current date as a string in the mongo shell. new
Date() returns the current date as a Date object. The mongo shell
wraps the Date object with the ISODate helper. The ISODate is in UTC.
You can specify a particular date by passing to the Date() method a
datetime string. For example:
new Date("") which returns the ISODate with the specified
date. new Date("") which specifies the datetime
in local datetime and returns the ISODate with the specified datetime
in UTC. new Date("") which specifies the
datetime in UTC and returns the ISODate with the specified datetime in
UTC.
To create a query to search on a Date field in mongo, you would instatiate an ISODate like this
db.collection.find({"datefield" : ISODate("YYYY-MM-DDTHH:MM:SS")})
Generally, this will be of somewhat limited use, since the time is measured in milliseconds, so you'll likely need to do a range query similar to:
db.collection.find({"datefield" : {"$gte" : <beginning_of_day_as_ISODate>, "$lte" : <end_of_day_as_ISODate>})
For example:
{createdDate : {$gte : ISODate("2017-12-06T00:00:00"), $lte : ISODate("2017-12-06T23:59:59")}}
If you are using a custom date object, or storing the date in some non-standard format, your query will need to be tailored to that object.
You can try out something like this
date = db.data.find({ 'Date' : {new ISODate("2017-12-6T23:59:59Z") } },{'Date' : '1'})
for document in date:
print(date)
Or you could try out this,
date = db.data.find({ 'Date' : {new Date(2017,12,6) } },{'Date' : '1'})
for document in date:
print(date)
So I am trying to create a function that checks whether or not the contents of a function is true:
def command_add(date, event, calendar):
'''
Add event_details to the list at calendar[date]
Create date if it was not there
:param date: A string as "YYYY-MM-DD"
:param event: A string describing the event
:param calendar: The calendars database
:return: a string indicating any errors, "" for no errors
'''
>>> calendar == {'2015-10-20': ['Python']}
True
>>> command_add("2015-11-01", "Computer Science class", calendar)
''
How would I write such a function? The problem I'm having is how to make the string or how to see if the string for the date is in the format 'YYYY-MM-DD'.
The following code uses strptime to parse the date, if the parsing fails it is not a proper date string. Then it checks if the date is already in the calendar dict or not to see whether to append or add the first entry.
from datetime import datetime
def command_add(date, event, calendar):
'''
Add event_details to the list at calendar[date]
Create date if it was not there
:param date: A string as "YYYY-MM-DD"
:param event: A string describing the event
:param calendar: The calendars database
:return: a string indicating any errors, "" for no errors
'''
try:
datetime.strptime('%Y-%m-%d', date):
except ValueError:
return 'Error parsing date'
else:
if date in calendar:
calendar[date].append(event)
else:
calendar[date] = [event]
return ''
Look at: https://docs.python.org/2/library/time.html#time.strptime
Give strptime the wanted format and if string is not formatted as you wanted you will get an exception
from datetime import datetime
def command_add(date, event, calendar):
# Your Code
calendar = {'2015-10-20': ['Python']}
try:
Date = "2015-11-01"
date_object = datetime.strptime(Date, '%Y-%m-%d')
command_add(Date, "Computer Science class", calendar)
except:
print "Date Format is not correct"
I am writing a method in a Python module which tries to make live easier to the users. This method implements the creation of events in that calendar.
def update_event(start_datetime=None, end_datetime=None, description=None):
'''
Args:
start_date: string datetime in the format YYYY-MM-DD or in RFC 3339
end_date: string datetime in the format YYYY-MM-DD or in RFC 3339
description: string with description (\n are transforrmed into new lines)
'''
If the user specifies the start_date or the end_date a check up should be made in order to determine if the date is in YYYY-MM-DD format or in the datetime RFC 3339 format.
if (start_date is not None):
# Check whether we have an date or datetime value
# Whole day events are represented as YYYY-MM-DD
# Other events are represented as 2014-04-25T10:47:00.000-07:00
whole_day_event = False
try:
new_start_time = datetime.datetime.strptime(start_date,'YYYY-MM-DD')
# Here the event date is updated
try:
new_start_time = datetime.datetime.strptime(start_date,'%Y-%m-%dT%H:%M:%S%z')
#Here the event date is updated
except ValueError:
return (ErrorCodeWhatever)
except ValueError:
return (ErrorCodeWhatever)
Is this a good way of doing this? Can I check what kind of date I am receiving in a nicer way?
Thanks!
dateutil.parser.parse can be used to attempt to parse strings into datetime objects for you.
from dateutil.parser import parse
def update_event(start_datetime=None, end_datetime=None, description=None):
if start_datetime is not None:
new_start_time = parse(start_datetime)
return new_start_time
d = ['23/04/2014', '24-04-2013', '25th April 2014']
new = [update_event(i) for i in d]
for date in new:
print(date)
# 2014-04-23 00:00:00
# 2013-04-24 00:00:00
# 2014-04-25 00:00:00
Extending #Ffisegydd answer you can also specify your target datetime format that you want like this :-
from dateutil.parser import parse
def update_event(start_datetime=None, end_datetime=None, description=None):
if start_datetime is not None:
new_start_time = parse(start_datetime)
return new_start_time
d = ['06/07/2021 06:40:23.277000','06/07/2021 06:40','06/07/2021']
new = [update_event(i) for i in d]
for date in new:
print(date.strftime('%Y/%m/%d %H:%M:%S.%f'))