Related
I wrote a script which will basically import data from a json file and store it in the database of my Django Application.
I'm simply pointing to a json file and i'm adding "facilities" if they don't exist or they get updated if the last modified date changes.
It worked perfectly fine until i couldn't import the json file locally anymore and made some smaller changes to use an external json file.
When i run the importer now it will only import the first two facilities and then tell me that all others are up to date even though they don't exist. I even modified and tested the json file manually to make sure it is not caused by a bug inside the json file.
I will add the old code plus the modified code below.
One of the main differences is that this part is now at the very bottom in the new version after the if statement:
for key, data_object in data.items():
And also that i'm using "import requests" in the new version with the following code at the bottom of the file. My feeling is that i made a mistake right there:
def handle(self, *args, **options):
"""
Call the function to import data from json url
"""
headers = {'Content-Type': 'application/json'}
response = requests.get(
url=IMPORT_URL,
headers=headers,
)
data = response.json()
for key, data_object in data.items():
self.import_facility_from_file(data_object)
New Version:
import requests
import json
from leads.models import Facility, FacilityAddress, FacilityInspectionInfo, FacilityComplaints
from django.core.management.base import BaseCommand
IMPORT_URL = 'https://importerdomain.test/test.json'
class Command(BaseCommand):
def import_facility_from_file(self, data):
UUID = data.get('UUID', None)
Name = data.get('Name', None)
IssuedNumber = data.get('IssuedNumber', None)
Capacity = data.get('Capacity', None)
Licensee = data.get('Licensee', None)
Email = data.get('Email', None)
AdministratorName = data.get('AdministratorName', None)
TelephoneNumber = data.get('TelephoneNumber', None)
ClosedTimestamp = data.get('ClosedTimestamp', None)
MostRecentLicenseTimestamp = data.get('MostRecentLicenseTimestamp', None)
ImporterLastModifiedTimestamp = data.get('ImporterLastModifiedTimestamp', None)
PrimaryAddress = data["AddressInfo"]["PrimaryAddress"]
SecondaryAddress = data["AddressInfo"]["SecondaryAddress"]
City = data["AddressInfo"]["City"]
RegionOrState = data["AddressInfo"]["RegionOrState"]
PostalCode = data["AddressInfo"]["PostalCode"]
Geolocation = data["AddressInfo"]["Geolocation"]
ComplaintRelatedVisits = data["InspectionInfo"]["ComplaintRelatedVisits"]
InspectionRelatedVisits = data["InspectionInfo"]["InspectionRelatedVisits"]
NumberOfVisits = data["InspectionInfo"]["NumberOfVisits"]
LastVisitTimestamp = data["InspectionInfo"]["LastVisitTimestamp"]
ComplaintsTypeA = data["Complaints"]["ComplaintsTypeA"]
ComplaintsTypeB = data["Complaints"]["ComplaintsTypeB"]
SubstantiatedAllegations = data["Complaints"]["SubstantiatedAllegations"]
TotalAllegations = data["Complaints"]["TotalAllegations"]
if Facility.objects.filter(ImporterLastModifiedTimestamp=ImporterLastModifiedTimestamp):
msg = "\n\nfacility exists and is up to date: {}\n{}".format(Name, str())
print(msg)
else:
print("UPDATE")
facility, facility_created = Facility.objects.update_or_create(UUID=UUID,
defaults={
'Name': Name,
'IssuedNumber': IssuedNumber,
'Capacity': Capacity,
'Licensee': Licensee,
'Email': Email,
'AdministratorName': AdministratorName,
'TelephoneNumber': TelephoneNumber,
'ClosedTimestamp': ClosedTimestamp,
'MostRecentLicenseTimestamp': MostRecentLicenseTimestamp,
'ImporterLastModifiedTimestamp': ImporterLastModifiedTimestamp
}
)
facility_address, address_created = FacilityAddress.objects.update_or_create(AddressInfo=facility,
defaults={
'PrimaryAddress': PrimaryAddress,
'SecondaryAddress': SecondaryAddress,
'City': City,
'RegionOrState': RegionOrState,
'PostalCode': PostalCode,
'Geolocation': Geolocation,
'AddressInfo': facility
}
)
facility_inspection, inspection_created = FacilityInspectionInfo.objects.update_or_create(InspectionInfo=facility,
defaults={
'ComplaintRelatedVisits': ComplaintRelatedVisits,
'InspectionRelatedVisits': InspectionRelatedVisits,
'NumberOfVisits': NumberOfVisits,
'LastVisitTimestamp': LastVisitTimestamp,
'InspectionInfo': facility
}
)
facility_complaints, complaints_created = FacilityComplaints.objects.update_or_create(Complaints=facility,
defaults={
'ComplaintsTypeA': ComplaintsTypeA,
'ComplaintsTypeB': ComplaintsTypeB,
'SubstantiatedAllegations': SubstantiatedAllegations,
'TotalAllegations': TotalAllegations,
'Complaints': facility
}
)
def handle(self, *args, **options):
"""
Call the function to import data from json url
"""
headers = {'Content-Type': 'application/json'}
response = requests.get(
url=IMPORT_URL,
headers=headers,
)
data = response.json()
for key, data_object in data.items():
self.import_facility_from_file(data_object)
Old Version
import os
import json
import traceback
from data_import.models import Facility, FacilityAddress, FacilityInspectionInfo, FacilityComplaints
from django.core.management.base import BaseCommand
from datetime import datetime
from jsontest.settings import BASE_DIR, STATIC_URL
class Command(BaseCommand):
def import_facility_from_file(self):
print("BASE", BASE_DIR)
data_folder = os.path.join(BASE_DIR, 'import_data', 'resources')
for data_file in os.listdir(data_folder):
with open(os.path.join(data_folder, data_file), encoding='utf-8') as data_file:
data = json.loads(data_file.read())
for key, data_object in data.items():
UUID = data_object.get('UUID', None)
Name = data_object.get('Name', None)
IssuedNumber = data_object.get('IssuedNumber', None)
Capacity = data_object.get('Capacity', None)
Licensee = data_object.get('Licensee', None)
Email = data_object.get('Email', None)
AdministratorName = data_object.get('AdministratorName', None)
TelephoneNumber = data_object.get('TelephoneNumber', None)
ClosedTimestamp = data_object.get('ClosedTimestamp', None)
MostRecentLicenseTimestamp = data_object.get('MostRecentLicenseTimestamp', None)
PrimaryAddress = data_object["AddressInfo"]["PrimaryAddress"]
SecondaryAddress = data_object["AddressInfo"]["SecondaryAddress"]
City = data_object["AddressInfo"]["City"]
RegionOrState = data_object["AddressInfo"]["RegionOrState"]
PostalCode = data_object["AddressInfo"]["PostalCode"]
Geolocation = data_object["AddressInfo"]["Geolocation"]
ComplaintRelatedVisits = data_object["InspectionInfo"]["ComplaintRelatedVisits"]
InspectionRelatedVisits = data_object["InspectionInfo"]["InspectionRelatedVisits"]
NumberOfVisits = data_object["InspectionInfo"]["NumberOfVisits"]
LastVisitTimestamp = data_object["InspectionInfo"]["LastVisitTimestamp"]
ComplaintsTypeA = data_object["Complaints"]["ComplaintsTypeA"]
ComplaintsTypeB = data_object["Complaints"]["ComplaintsTypeB"]
SubstantiatedAllegations = data_object["Complaints"]["SubstantiatedAllegations"]
TotalAllegations = data_object["Complaints"]["TotalAllegations"]
LatestUpdateTimestamp = data_object.get('LatestUpdateTimestamp', None)
if Facility.objects.filter(LatestUpdateTimestamp=LatestUpdateTimestamp):
msg = "\n\nfacility exists and is up to date: {}\n{}".format(Name, str())
print(msg)
else:
print("UPDATE")
facility, facility_created = Facility.objects.update_or_create(UUID=UUID,
defaults={
'Name': Name,
'IssuedNumber': IssuedNumber,
'Capacity': Capacity,
'Licensee': Licensee,
'Email': Email,
'AdministratorName': AdministratorName,
'TelephoneNumber': TelephoneNumber,
'ClosedTimestamp': ClosedTimestamp,
'MostRecentLicenseTimestamp': MostRecentLicenseTimestamp,
'LatestUpdateTimestamp': LatestUpdateTimestamp
}
)
def handle(self, *args, **options):
"""
Call the function to import data
"""
self.import_facility_from_file()
The json i'm importing
{"00016ed7be4872a19d6e16afc98a7389b2bb324a2":
{"UUID":"00016ed7be4872a19d6e1ed6f36b647f3eb41cadedd2130b103a5851caebc26fbbbf24c2f1a64d2cf34ac4e03aaa30309816f58c397e6afc98a7389b2bb324a2","Name":"Test Facility","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"test#example.com","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","ImporterLastModifiedTimestamp":"1362985200",
"AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
"InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
"Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}},
"00016ed7be4872a15435435435b2bb324a2":
{"UUID":"000c93dcb7a0b3d5783bb330892aff6abdb9fb57a7d3701c2d903f3640877579f3173ecd8a80532f6c3d53dbacde78a6a54ae42fef321a5793f5a01934f8de7a","Name":"Test Facility 2","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"test#example.com","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","ImporterLastModifiedTimestamp":"1362985200",
"AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
"InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
"Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}},
"00234324324343243afc98a7389b2bb324a2":
{"UUID":"fffd4dec10054e6e1deb2a2266a7c6bb0136ba46222e734ceed5855651f735cfbe0bb66cfaf27c3d175ae261a8f6df0c36b5390d15c70b07d67e35e1081aaf6d","Name":"Test Facility 3","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"test#example.com","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","ImporterLastModifiedTimestamp":"1362985200",
"AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
"InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
"Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}}}
if Facility.objects.filter(ImporterLastModifiedTimestamp=ImporterLastModifiedTimestamp):
The if statement above is True as soon as you check it with a timestamp that is the same as one of the objects inserted before
You need to filter UUID and timestamp to catch the one single object you want to check if it has changed.
this is my first question here. I try to rewrite telegram bot in Python. And I cant solve one problem. When I test my bot by myself everything fine, but if another user connect with my bot script doesn`t start for him with initial state of variables. For example, in this code script increment x variable when user send "1". But for new user x already not null. Please help me to fix it.
import StringIO
import json
import logging
import random
import urllib
import urllib2
# for sending images
from PIL import Image
import multipart
# standard app engine imports
from google.appengine.api import urlfetch
from google.appengine.ext import ndb
import webapp2
TOKEN = 'TOKEN HERE'
BASE_URL = 'https://api.telegram.org/bot' + TOKEN + '/'
x = 0
# ================================
class EnableStatus(ndb.Model):
# key name: str(chat_id)
enabled = ndb.BooleanProperty(indexed=False, default=False)
# ================================
def setEnabled(chat_id, yes):
es = EnableStatus.get_or_insert(str(chat_id))
es.enabled = yes
es.put()
def getEnabled(chat_id):
es = EnableStatus.get_by_id(str(chat_id))
if es:
return es.enabled
return False
# ================================
class MeHandler(webapp2.RequestHandler):
def get(self):
urlfetch.set_default_fetch_deadline(60)
self.response.write(json.dumps(json.load(urllib2.urlopen(BASE_URL + 'getMe'))))
class GetUpdatesHandler(webapp2.RequestHandler):
def get(self):
urlfetch.set_default_fetch_deadline(60)
self.response.write(json.dumps(json.load(urllib2.urlopen(BASE_URL + 'getUpdates'))))
class SetWebhookHandler(webapp2.RequestHandler):
def get(self):
urlfetch.set_default_fetch_deadline(60)
url = self.request.get('url')
if url:
self.response.write(json.dumps(json.load(urllib2.urlopen(BASE_URL + 'setWebhook', urllib.urlencode({'url': url})))))
class WebhookHandler(webapp2.RequestHandler):
def post(self):
urlfetch.set_default_fetch_deadline(60)
body = json.loads(self.request.body)
logging.info('request body:')
logging.info(body)
self.response.write(json.dumps(body))
update_id = body['update_id']
message = body['message']
message_id = message.get('message_id')
date = message.get('date')
text = message.get('text')
fr = message.get('from')
chat = message['chat']
chat_id = chat['id']
if not text:
logging.info('no text')
return
def reply(msg=None, img=None):
if msg:
resp = urllib2.urlopen(BASE_URL + 'sendMessage', urllib.urlencode({
'chat_id': str(chat_id),
'text': msg.encode('utf-8'),
'disable_web_page_preview': 'true',
#'reply_to_message_id': str(message_id),
})).read()
elif img:
resp = multipart.post_multipart(BASE_URL + 'sendPhoto', [
('chat_id', str(chat_id)),
#('reply_to_message_id', str(message_id)),
], [
('photo', 'image.jpg', img),
])
else:
logging.error('no msg or img specified')
resp = None
logging.info('send response:')
logging.info(resp)
if text.startswith('/'):
if text == '/start':
reply('Bot start')
setEnabled(chat_id, True)
elif text == '/stop':
reply('Bot disabled')
setEnabled(chat_id, False)
else:
reply('What command?')
elif '1' in text:
global x
x = x + 1
reply(str(x))
else:
if getEnabled(chat_id):
reply('?')
else:
logging.info('not enabled for chat_id {}'.format(chat_id))
app = webapp2.WSGIApplication([
('/me', MeHandler),
('/updates', GetUpdatesHandler),
('/set_webhook', SetWebhookHandler),
('/webhook', WebhookHandler),
], debug=True)
Instead of variable x, you may want to use python dictionary like this -
x = {}
and then use it like this -
key = str(chat_id)
if key in x:
x[key] += 1
else:
x[key] = 1
We have about 5,000 objects of class Domain in Google App Engine, and we want to export the list of domains to CSV. Each domain is linked to an object of class DomainStateData:
class DomainStateData(db.Expando, ExpandoEntity):
plan = db.ReferenceProperty(Plan)
plan_expiration = db.DateTimeProperty()
trial_expiration = db.DateTimeProperty()
date_created = db.DateTimeProperty(auto_now_add=True, indexed=True)
last_modified = db.DateTimeProperty(auto_now=True)
class Domain(db.Expando, ExpandoEntity, SocialIconsEntity):
"""
Domain Model
"""
domain = db.StringProperty(required=True)
...
_state_data = db.ReferenceProperty(DomainStateData)
#property
def state_data(self):
try:
if not self._state_data:
# try to get it, if not, build it
sd = DomainStateData.get_by_key_name(self.key().name())
if not sd:
sd = DomainStateData(key_name=self.key().name()).put()
self._state_data = sd
self.put()
return self._state_data
else:
return self._state_data
except ReferencePropertyResolveError:
self._state_data = DomainStateData(key_name=self.key().name()).put()
self.put()
return self._state_data
I wrote a code which exports 100 domains to CSV (it takes 5 seconds), but if I try to fetch all the 5,000 domains I get a timeout, which is 60 seconds. Is it possible to fetch all the DomainStateData objects together without a timeout? Here is my code that exports the domains to CSV:
import sys
sys.path.insert(0, 'libs')
import webapp2
import datetime
import csv
from models import Domain
class ExportAllDomainsToCsvHandler(webapp2.RequestHandler):
def get(self):
self.response.headers['Content-Type'] = 'text/csv'
self.response.headers['Content-Disposition'] = 'attachment; filename="All Domains [{0}].csv"'.format(str(datetime.date.today()))
writer = csv.writer(self.response.out)
writer.writerow(["Domain", "Current state", "Plan expiration date", "Trial expiration date", "Current oauth user"])
all_domains = Domain.all().fetch(100)
all_domains.sort(key=lambda domain: (0 if domain.state_data.plan_expiration is None else 1, domain.state_data.plan_expiration, 0 if domain.state_data.trial_expiration is None else 1, domain.state_data.trial_expiration, domain.domain))
for domain in all_domains:
if (domain.state_data.plan_expiration is None):
domain_plan_expiration = "No plan expiration date"
else:
domain_plan_expiration = domain.state_data.plan_expiration.strftime('%Y-%m-%d')
if (domain.state_data.trial_expiration is None):
domain_trial_expiration = "No trial expiration date"
else:
domain_trial_expiration = domain.state_data.trial_expiration.strftime('%Y-%m-%d')
writer.writerow([domain.domain, domain.cur_state.name, domain_plan_expiration, domain_trial_expiration, domain.admin])
app = webapp2.WSGIApplication([
("/csv/export_all_domains_to_csv", ExportAllDomainsToCsvHandler)
], debug=True)
OK, I found a solution. I fetched all the DomainStateData objects directly from the database and now it takes 35 seconds to create the CSV with all the domains. Here is my code, I didn't change the models:
import sys
sys.path.insert(0, 'libs')
import webapp2
import datetime
import csv
from models import DomainStateData, Domain
class ExportAllDomainsToCsvHandler(webapp2.RequestHandler):
def get(self):
self.response.headers['Content-Type'] = 'text/csv'
self.response.headers['Content-Disposition'] = 'attachment; filename="All Domains [{0}].csv"'.format(str(datetime.date.today()))
writer = csv.writer(self.response.out)
writer.writerow(["Domain", "Current state", "Plan expiration date", "Trial expiration date", "Current oauth user"])
all_domain_state_data_dict = dict()
all_domain_state_data = DomainStateData.all().fetch(1000000)
all_domains = Domain.all().fetch(1000000)
for domain_state_data in all_domain_state_data:
all_domain_state_data_dict[domain_state_data.key().name()] = domain_state_data
for domain in all_domains:
if (domain.key().name() in all_domain_state_data_dict):
domain.__state_data = all_domain_state_data_dict[domain.key().name()]
all_domains.sort(key=lambda domain: (0 if domain.__state_data.plan_expiration is None else 1, domain.__state_data.plan_expiration, 0 if domain.__state_data.trial_expiration is None else 1, domain.__state_data.trial_expiration, domain.domain))
for domain in all_domains:
if (domain.__state_data.plan_expiration is None):
domain_plan_expiration = "No plan expiration date"
else:
domain_plan_expiration = domain.__state_data.plan_expiration.strftime('%Y-%m-%d')
if (domain.__state_data.trial_expiration is None):
domain_trial_expiration = "No trial expiration date"
else:
domain_trial_expiration = domain.__state_data.trial_expiration.strftime('%Y-%m-%d')
writer.writerow([domain.domain, domain.cur_state.name, domain_plan_expiration, domain_trial_expiration, domain.admin])
app = webapp2.WSGIApplication([
("/csv/export_all_domains_to_csv", ExportAllDomainsToCsvHandler)
], debug=True)
How to configure jinja2 in Appengine to:
Auto reload when template is updated.
Enable bytecode cache, so it can be share among each instances. I prefer jinja2 to produce bytecode when compiling template, and store it to datastore. So next instance will load bytecode instead of repeatedly compile the template.
I have added the bcc like this, using the app engine memcache Client()::
loader = dynloaders.DynLoader() # init Function loader
bcc = MemcachedBytecodeCache(memcache.Client(), prefix='jinja2/bytecode/', timeout=None)
return Environment(auto_reload=True, cache_size=100, loader=FunctionLoader(loader.load_dyn_all),
bytecode_cache=bcc)
My function loader:
def html(self, cid):
def _html_txt_up_to_date(): # closure to check if template is up to date
return CMSUpdates.check_no_update(cid, template.modified)
template = ndb.Key('Templates', cid, parent=self.parent_key).get()
if not template:
logging.error('DynLoader (HTML/TXT): %s' % cid)
return None # raises TemplateNotFound exception
return template.content, None, _html_txt_up_to_date
The template model uses template.modified : ndb.DateTimeProperty(auto_now=True)
The closure function:
class CMSUpdates(ndb.Model):
updates = ndb.JsonProperty()
#classmethod
def check_no_update(cls, cid, cid_modified):
cms_updates = cls.get_or_insert('cms_updates', updates=dict()).updates
if cid in cms_updates: # cid modified has dt microseconds
if cid_modified >= datetime.strptime(cms_updates[cid], '%Y-%m-%d %H:%M:%S'):
if (datetime.now() - timedelta(days=1)) > cid_modified:
del cms_updates[cid]
cls(id='cms_updates', updates=cms_updates).put_async()
return True
return False # reload the template
return True
Been few weeks i looking for the solution. And finally i figured it out, i would like to share my code for everyone. There are 4 python source files in my code.
TemplateEngine.py, ContentRenderer.py, TestContent.py & Update_Template.py
File: TemplateEngine.py
Note:
i use now = datetime.utcnow() + timedelta(hours=8) because my timezone is GMT+8
You must use ndb.BlobProperty to store the bytecode, ndb.TextProperty will not work!
from google.appengine.ext import ndb
from datetime import datetime,timedelta
class SiteTemplates(ndb.Model):
name = ndb.StringProperty(indexed=True, required=True)
data = ndb.TextProperty()
uptodate = ndb.BooleanProperty(required=True)
class SiteTemplateBytecodes(ndb.Model):
key = ndb.StringProperty(indexed=True, required=True)
data = ndb.BlobProperty(required=True)
mod_datetime = ndb.DateTimeProperty(required=True)
class LocalCache(jinja2.BytecodeCache):
def load_bytecode(self, bucket):
q = SiteTemplateBytecodes.query(SiteTemplateBytecodes.key == bucket.key)
if q.count() > 0:
r = q.get()
bucket.bytecode_from_string(r.data)
def dump_bytecode(self, bucket):
now = datetime.utcnow() + timedelta(hours=8)
q = SiteTemplateBytecodes.query(SiteTemplateBytecodes.key == bucket.key)
if q.count() > 0:
r = q.get()
r.data = bucket.bytecode_to_string()
r.mod_datetime = now
else:
r = SiteTemplateBytecodes(key=bucket.key, data=bucket.bytecode_to_string(), mod_datetime=now)
r.put()
def Update_Template_Source(tn, source):
try:
q = SiteTemplates.query(SiteTemplates.name == tn)
if q.count() == 0:
u = mkiniTemplates(name=tn, data=source, uptodate=False)
else:
u = q.get()
u.name=tn
u.data=source
u.uptodate=False
u.put()
return True
except Exception,e:
logging.exception(e)
return False
def Get_Template_Source(tn):
uptodate = False
def Template_Uptodate():
return uptodate
try:
q = SiteTemplates.query(SiteTemplates.name == tn)
if q.count() > 0:
r = q.get()
uptodate = r.uptodate
if r.uptodate == False:
r.uptodate=True
r.put()
return r.data, tn, Template_Uptodate
else:
return None
except Exception,e:
logging.exception(e)
return None
File: ContentRenderer.py
Note: It is very important to set cache_size=0, otherwise bytecode cache function will be disable. I have no idea why.
from TemplateEngine import Get_Template_Source
import jinja2
def Render(tn,tags):
global te
return te.Render(tn, tags)
bcc = LocalCache()
te = jinja2.Environment(loader=jinja2.FunctionLoader(Get_Template_Source), cache_size=0, extensions=['jinja2.ext.autoescape'], bytecode_cache=bcc)
File: Update_Template.py
Note: Use Update_Template_Source() to update template source to datastore.
from TemplateEngine import Update_Template_Source
template_source = '<html><body>hello word to {{title}}!</body></html>'
if Update_Template_Source('my-template.html', template_source):
print 'template is updated'
else:
print 'error when updating template source'
File: TestContent.py
Note: Do some test
from ContentRenderer import Render
print Render('my-template.htmnl', {'title':'human'})
'hello world to human!'
You will realize, even you have more than 20 instances in your application, the latency time will not increase, even you update your template. And the template source will update in 5 to 10 seconds.
I have a little problem with web.py. Exacly I have problem with sessions.
Link to my app:
http://len.iem.pw.edu.pl/~witkowr1/apps/demo/
Login/password: wtq/wtq
Code:
# -*- coding: utf-8 -*-
import web
import json
from web.contrib.template import render_jinja
import datetime
prefix = '/~witkowr1/apps/demo'
urls = (
prefix + '/login','Login',
prefix + '/logout','Logout',
prefix + '/', 'Index',
)
app = web.application(urls, globals())
wsgi = app.wsgifunc()
session = web.session.Session(app, web.session.DiskStore('sessions'),initializer={'time':datetime.datetime.now()})
render = render_jinja('static', encoding = 'utf-8')
render._lookup.globals.update(assets=prefix+'/static')
class Login:
def GET(self):
web.seeother(prefix+'/')
def POST(self):
login = web.input().login
password = web.input().passwd
if login == 'wtq' and password == 'wtq':
session.logged_in = True
session.time = datetime.datetime.now()
last_login = web.cookies().get('time')
if last_login == None:
last_login_data = u'Zalogowałeś się pierwszy raz.'
else:
last_login_data = last_login
return render.logged(name=login, date_last_login=last_login_data)
else:
session.logged_in = False
error=u'Niepoprawne dane. SprĂłbuj jeszcze raz.'
return render.login(error_msg=error)
class Logout:
def GET(self):
web.seeother(prefix+'/')
def POST(self):
session.logged_in = False
web.setcookie('time',session.time)
message = u'Zostałeś poprawnie wylogowany.'
session.kill()
return render.login(error_msg=message)
class Index:
def GET(self):
return render.login()
if __name__ == "__main__":
app.run()
I would like to verify session and if I login early, I see site with latest login date.
Now, when I refresh the site, I must login again.
I think, that I check session, when I rendering HTML site, but I don't know, what I do it.
Please help!
The problem here is that you are not checking whether they are logged in if they access the page with GET method.
You would need to make a change like:
def GET(self):
if session.logged_in:
last_login = web.cookies().get('time')
if last_login == None:
last_login_data = u'Zalogowałeś się pierwszy raz.'
else:
last_login_data = last_login
return render.logged(name=login, date_last_login=last_login_data)
else:
web.seeother(prefix+'/')
But you should rewrite this, a lot, so that you are taken to another page once you are logged in, and that page is responsible for rendering this information. There is a lot of room for improvement in the structure of your application.
That said, the simple answer is - even though you store the session, the "GET" method of login is entirely unaware of sessions, and will always return the login prompt.
Not sure if you solved your problem already but since it looks like there are often some problems with sessions in web.py I pushed a small and crude demo of it to bitbucket. Works fine for me so hope this works for you.
You can get it via:
git clone https://victorkohler#bitbucket.org/victorkohler/webpy-login.git
I solved my problem.
Very stupid mistakes :)
Code:
# -*- coding: utf-8 -*-
import web
from web.contrib.template import render_jinja
import datetime
prefix = ''
urls = (
prefix + '/', 'Index',
prefix + '/login','Login',
prefix + '/logout','Logout',
)
app = web.application(urls, globals())
wsgi = app.wsgifunc()
web.config.debug = False
session = web.session.Session(app, web.session.DiskStore('sessions'),initializer={'time':datetime.datetime.now()})
render = render_jinja('static', encoding = 'utf-8')
render._lookup.globals.update(assets=prefix+'/static')
allowed = (
('user','user'),
)
class Login:
def GET(self):
web.seeother(prefix+'/')
def POST(self):
login = web.input().login
passwd = web.input().passwd
if(login,passwd) in allowed:
session.logged_in = True
session.login = login
session.time = datetime.datetime.now()
last_login = web.cookies().get('time')
if last_login == None:
last_login_data = u'Zalogowałeś się pierwszy raz.'
else:
last_login_data = last_login
return render.logged(name=login, date_last_login = last_login_data)
else:
session.logged_in = False
error=u'Niepoprawne dane. Spróbuj jeszcze raz.'
return render.login(error_msg=error)
class Logout:
def GET(self):
web.seeother(prefix+'/')
def POST(self):
session.logged_in = False
web.setcookie('time',session.time)
message = u'Zostałeś poprawnie wylogowany.'
session.kill()
return render.login(error_msg=message)
class Index:
def GET(self):
if session.get ('logged_in') == True:
last_login = web.cookies().get('time')
if last_login == None:
last_login_data = u'Zalogowałeś się pierwszy raz.'
else:
last_login_data = last_login
return render.logged(name=session.get ('login'), date_last_login = last_login_data)
else:
return render.login()
if __name__ == "__main__":
app.run()
This is not an answer to your question but an extension of the question. I have a similar solution to the login from the webpy cookbook (http://webpy.org/cookbook/userauthbasic) but would like to load the variable allowed from a database.
allowed = (
('user','user'),
)
When I tried a read to assign the value to the variable it comes out as "None" when executed in the login class.
def readUsersDb():
def POST(self):
# load user data into allowed variable
usersDbFilePath = 'userdata/irf/default/'
usersDbFile = 'usersDb.db'
usersDbFilePath = usersDbFilePath + usersDbFile
conn = sqlite3.connect(usersDbFilePath)
c = conn.cursor()
c.execute('SELECT * FROM users')
output = c.fetchall()
conn.close()
global allowed
allowed = output
readUsersDb()
The login functions when the variable allowed is hard coded. I used the format from the database read and it still functions as expected so it is not a format issue.
#response: [(u'user1', 'user1'), (u'user2', 'user2'), (u'user3', 'user3')]
Hopefully someone has tried to do this previously. If it is a faux pas to add a question as an answer I apologize ahead of time.