We Keep Coding

google cloud: downloading as an CSV

I have a download button that should return a users CSV. The code for it below:
class StartCSVHandler(ThreeScaleResourceHandler):
""" Starts generating a user's CSV file """
allowed_methods = ('new',)
requires_input = False
def post(self, *args, **kwargs):
user = self.current_user()
if not user:
self.abort(401, detail=ERROR_NOT_AUTHORIZED)
task_url = '/jobs/users/%s/data/csv' % user.key()
taskqueue.add(url=task_url, queue_name='users')
return {}
def generate_data_from_query(query, user, format, handler, filename, mission=None):
batch_size = 500
cursor = None
spottings = []
count = 0
mime_type = 'text/csv' if format == CSV_FORMAT else 'text/xml'
timestamp = '%0.6f' % time.time()
gcs_filename_template = '/{bucket}/{user}/{timestamp}/{filename}'
from global_config import config
gcs_filename = gcs_filename_template.format(
bucket='project-noah-backups',
# bucket=config['cloudstorage']['export_files_bucket'],
user=str(user.key()),
timestamp=timestamp,
filename=filename
)
logging.debug(str(user.key()))
f = cloudstorage.open(gcs_filename, mode='w', content_type=mime_type)
# blobstore_file_name = files.blobstore.create(mime_type=mime_type, _blobinfo_uploaded_filename=filename.encode('utf-8'))
while True:
if format == CSV_FORMAT:
writer = utils.UnicodeWriter(f)
if count == 0:
writer.writerow(csv_display_names)
elif format == KML_FORMAT and count == 0:
f.write(template.render('spotting_export_pre.kml', {}))
if cursor:
query.with_cursor(cursor)
spottings = query.fetch(batch_size)
if format == CSV_FORMAT:
dicts = [s.as_dict() for s in spottings]
logging.debug(dicts)
for spotting_dict in dicts:
writer.writerow([spotting_dict[k] for k in csv_keys])
elif format == KML_FORMAT:
output = template.render('spotting_export_mid.kml', {'spottings' : spottings, 'server_url' : utils.server_url(handler.request)})
f.write(output.encode('utf-8'))
cursor = query.cursor()
logging.info('written spottings %d to %d' % (count, count + len(spottings)))
count += len(spottings)
if not mission:
push_to_beacon_user(user, {'format':format,'progress':count})
else:
push_to_beacon_user_mission(user, mission, {'format':format,'progress':count})
if len(spottings) < batch_size:
break
if format == KML_FORMAT:
f.write(template.render('spotting_export_post.kml', {}))
blob_key = BlobKey(blobstore.create_gs_key(u'/gs' + gcs_filename))
logging.debug(blob_key)
return blob_key
def generate_data_from_user_spottings(user, format, handler):
filename = u'My-Spottings.%s' % format
# query = user.mySpottings
query = user.mySpottings
logging.debug(query)
return generate_data_from_query(query, user, format, handler, filename)
class GenerateUserDataHandler(NoahHandler):
def post(self, user_key=None, format=None):
if not user_key:
return
user = NoahUser.get(user_key)
if not user:
return
if format not in (CSV_FORMAT, KML_FORMAT):
return
blob_key = generate_data_from_user_spottings(user, format, self)
user = NoahUser.get(user_key)
if format == CSV_FORMAT:
if user.csv:
user.csv.delete()
user.csv = blob_key
user.put()
elif format == KML_FORMAT:
if user.kml:
user.kml.delete()
user.kml = blob_key
user.put()
logging.debug(user.recent_spottings)
logging.debug(str(blob_key))
push_to_beacon_user(user, {'format': format,'url':'/data?key=%s' % str(blob_key)})
class ThreeScaleResourceHandler(ResourceHandler):
#three_scale_authenticate
def get(self, *args, **kwargs):
super(ThreeScaleResourceHandler, self).get(*args, **kwargs)
#three_scale_authenticate
def post(self, *args, **kwargs):
super(ThreeScaleResourceHandler, self).post(*args, **kwargs)
#three_scale_authenticate
def put(self, *args, **kwargs):
super(ThreeScaleResourceHandler, self).put(*args, **kwargs)
#three_scale_authenticate
def delete(self, *args, **kwargs):
super(ThreeScaleResourceHandler, self).delete(*args, **kwargs)
This should download users data in the form of an CSV. The problem i am getting is two fold; firstly, the end point that this generates is '/api/v1/users/me/data/csv' and when visiting it, i receive the following error
{"error": {"title": "Unauthorized", "status": 401, "message": "You are not authorized to perform that action. Please use the api_key parameter with your registered key."}}
Secondly, the link it provides for the user to save cannot be found:
http://localhost:8080/data?key=encoded_gs_file:cHJvamVjdC1ub2FoLWJhY2t1cHMvYWdoa1pYWi1UbTl1WlhJVkN4SUlUbTloYUZWelpYSVlnSUNBZ0lDQWdBb00vMTU4MTAxODk3My4wODgyODEvTXktU3BvdHRpbmdzLmNzdg==
I am not entirely sure what i need to correct.

firstly, the end point that this generates is '/api/v1/users/me/data/csv' and when visiting it, i receive the following error: {"error": {"title": "Unauthorized", "status": 401, "message": "You are not authorized to perform that action. Please use the api_key parameter with your registered key."}}
Which handler in your code snippet handles /api/v1/users/me/data/csv? Is it StartCSVHandler? Are you sure it isn't being thrown because of this line? self.abort(401, detail=ERROR_NOT_AUTHORIZED)
Secondly, the link it provides for the user to save cannot be found: http://localhost:8080/data?key=encoded_gs_file:cHJvamVjdC1ub2FoLWJhY2t1cHMvYWdoa1pYWi1UbTl1WlhJVkN4SUlUbTloYUZWelpYSVlnSUNBZ0lDQWdBb00vMTU4MTAxODk3My4wODgyODEvTXktU3BvdHRpbmdzLmNzdg==
In what way? like you are trying to find the file on your machine or this link is throwing a 404?
On your localhost, encoded_gs_file files can be found here: http://localhost:8000/blobstore
If it's a 404, then what does your handler for /data do? It doesnt look like it's in your code snippet

TypeError : encode() missing 1 required positional argument: 'iterations'

i dont know what is triggering this error. i dont know why i keep getting this error . i already change a few parts of code and still i get this error . i have been trying to fix it for 2 days .
Traceback:
File "C:\Users\Adila\AppData\Local\Programs\Python\Python35\lib\site-packages\django\core\handlers\exception.py" in inner
41. response = get_response(request)
File "C:\Users\Adila\AppData\Local\Programs\Python\Python35\lib\site-packages\django\core\handlers\base.py" in _get_response
187. response = self.process_exception_by_middleware(e, request)
File "C:\Users\Adila\AppData\Local\Programs\Python\Python35\lib\site-packages\django\core\handlers\base.py" in _get_response
185. response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\Adila\Documents\tryFOUR\src\register\views.py" in register
13. user = form.save()
File "C:\Users\Adila\Documents\tryFOUR\src\custom_user\forms.py" in save
50. user.set_password(self.cleaned_data["password2"])
File "C:\Users\Adila\AppData\Local\Programs\Python\Python35\lib\site-packages\django\contrib\auth\base_user.py" in set_password
105. self.password = make_password(raw_password)
File "C:\Users\Adila\AppData\Local\Programs\Python\Python35\lib\site-packages\django\contrib\auth\hashers.py" in make_password
84. return hasher.encode(password, salt)
Exception Type: TypeError at /accounts/register/
Exception Value: encode() missing 1 required positional argument: 'iterations'
hashers.py :
from django.contrib.auth.hashers import PBKDF2PasswordHasher
from django.utils.crypto import (get_random_string, pbkdf2)
from honeywordHasher.models import Sweetwords
class MyHoneywordHasher(PBKDF2PasswordHasher):
algorithm = "honeyword_base9_tweak3_pbkdf2_sha256"
iterations = PBKDF2PasswordHasher.iterations*3
def hash(self, password, salt, iterations):
hash = pbkdf2(password, salt, iterations, digest=self.digest)
return base64.b64encode(hash).decode('ascii').strip()
def salt(self):
salt = get_random_string()
while Sweetwords.objects.filter(salt=salt).exists():
salt = get_random_string()
return salt
def verify(self, password, encoded):
algorithm, iterations, salt, dummy=encoded.split('$',3)
hashes = pickle.loads(Sweetwords.objects.get(salt=salt).sweetwords)
hash = self.hash(password, salt, int(iterations))
if hash in hashes:
return honeychecker.check_index(salt, hashes.index(hash))
return False
def encode(self, password, salt, iterations):
sweetwords = ['hilman95']
sweetwords.extend(honey_gen.gen(password, base64, ["passfiles.txt"]))
for i in range(base64 + 1):
sweetwords.extend(honeywordtweak.tweak(password[i], 3))
random.shuffle(sweetwords)
hashes = []
for swd in sweetwords:
hashes.append(self.hash(swd, salt, iterations))
self.honeychecker.update_index(salt, sweetwords.index(password))
h = Sweetwords(salt=salt, sweetwords = pickle.dumps(hashes))
h.save()
return "%s$%d$%s$%s" %(self.algorithm, iterations, salt, hashes[0])
can someone explain why i keep getting this error please ? i have spend hours searching all the related questions here and nothing solves it .

def encode(self, password, salt, iterations=None):
if iterations is None:
iterations = self.iterations
# then ur code......

A simple way I used to Serialize and Deserialize JSON in python is as below.
It might not be the best way but as a newbie, in python, it helps me to get the job done.
import json
import requests
class UserModel:
GET_USER_URL = "https://jsonplaceholder.typicode.com/todos/1"
#{'userId': 1, 'id': 1, 'title': 'delectus aut autem', 'completed': False}
def __init__(self) -> None:
pass
def __init__(self, userId, id, title, completed):
self.userId = userId
self.id = id
self.title = title
self.completed = completed
def getUser():
response = requests.get(UserModel.GET_USER_URL)
res = response.json()
print("Response\n",res)
#dict_obj = json.loads(res)
#print("Type of dict_obj", type(res))
return json.dumps(res, indent=4, cls=UserEndoder)
# subclass
class UserEndoder(json.JSONEncoder):
#I removed self as it was causing:
#TypeError: JSONEncoder.encode() missing 1 required positional argument: 'o'
def default(o):#I
return o.__dict__
Sample usage
userModel = UserModel.getUser()
print("Serialized results:\n",userModel)
userModel = UserModel(123, 321, "test title", True)
print("Deserialized:\n",UserEndoder.default(userModel))
Have a nice coding day!!!

AttributeError: 'Driver' object has no attribute 'id'

i have a deployment on appengine, im getting this error when i'm tryng to assign a trip
def get(self):
from trips.end_trip import notify_cancelation
"""
GET /2/drivers/report_trip_arrival
> trip_id /custom_id
> device_id
> driver_id
> event_timestamp
< 200 application/json
Changes the trip status to 4, your taxi has arrived
"""
responseJSON = {}
self.response.headers['Content-Type'] = "application/json"
trip_id = self.request.get("trip_id")
custom_trip_id = self.request.get("custom_id")
device_id = self.request.get("device_id")
logging.info("Report arrival")
logging.info(str(trip_id))
logging.info(str(custom_trip_id))
logging.info(str(device_id))
#Get the trip
# fixing
try:
trip = trip_finder.find_trip(trip_id=trip_id, custom_id=custom_trip_id)
except:
trip = None
responseJSON["Status"] = "OK"
return
driver_id = self.request.get('driver_id')
if driver_id:
driver_id = int(driver_id)
if trip.selected_driver.get().id() != driver_id:
responseJSON = {'status': 'OK', 'Staus': 'OK'}
json.dump(responseJSON, self.response.out)
return
if trip:
#deferred.defer(notify_arrival, trip.key.id(), _queue="delivery")
params = {}
params["trip_id"] = trip.key.id()
taskqueue.add(url="/2/delivery/task/notify_arrival", params=params, queue_name="delivery")
last_status = trip.last_status()
#If we are already on status 4, we just return OK
if last_status.id != "4":
#Update the status
#ripStatus(trip=trip.key, id="4", message=trip_statuses["4"], timestamp=now()).put()
trip.change_status("4")
#Alert the passenger
name = trip.traveler_name.split()[0] if trip.traveler_name else ""
message_status = name + ", ha llegado tu taxi :)"
responseJSON["Status"] = "OK"
responseJSON["status"] = "OK"
# if the trip was canceled notify the driver
if last_status.id == "16":
#deferred.defer(notify_cancelation, trip.key.id(), _queue="endtrip")
params = {'trip_id': trip.key.id()}
taskqueue.add(url="/2/trips/task/notify_cancelation", params=params, queue_name="endtrip")
#deferred.defer(notify_cancelation, trip.key.id(), _queue="endtrip")
else:
responseJSON["Status"] = "OK"
responseJSON["status"] = "OK"
self.response.out.write(json.dumps(responseJSON))
This is the full details:
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.2/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~precise-line-762/v-4-4-03.383164018327881359/security/security.py", line 47, in wrapper
return handler(request)
File "/base/data/home/apps/s~precise-line-762/v-4-4-03.383164018327881359/drivers/handlers.py", line 141, in get
if trip.selected_driver.get().id() != driver_id:
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/model.py", line 3640, in __getattr__
return super(Expando, self).__getattribute__(name)
AttributeError: 'Driver' object has no attribute 'id'

You have to pick object's ID from key property.
Looks like your trip.selected_driver is a Key already. If you want to ensure the object exist then call trip.selected_driver.get().key.id()

Adding ManytoManyField breaks Django

In my project, I have a model, Project, that when saved creates another model, Access, which contains a manytomanyfield, access_list, where users can add other users onto their project as collaborators. It works - when I create a new project, I can add additional users into it, but if I add a 2nd user, it will no longer serve the page with the error,
Exception Value: get() returned more than one Access -- it returned 2!"
If I switch to an account I've added to the project, then add other users on with that account, they add fine and it does not break the page.
When the page breaks, it also creates an additional instance of the project on my Projects page, even though there's only one instance of the project in the database.
My code:
Models.py:
class Project(models.Model):
created_by = models.ForeignKey(User)
super(Project, self).save()
Access.objects.get_or_create(project=self)
class Access(models.Model):
project = models.ForeignKey(Project)
access_list = models.ManyToManyField(User)
pubdate = models.DateTimeField(default=timezone.now)
Views.py:
#login_required
def access(request, project_id=1):
thisuser = request.user
if Access.objects.filter(Q(access_list=thisuser) | Q(project__created_by=thisuser), project__id=project_id).exists():
accesspermission = Access.objects.filter(Q(access_list=thisuser) | Q(project__created_by=thisuser), project__id=project_id).order_by('-project__project_pubdate')[0]
else:
accesspermission = None
if Entry.objects.filter(project_id=project_id).exists():
anyentries = Entry.objects.filter(project_id=project_id, entry_unique=1).order_by('-entry_pubdate')[0]
else:
anyentries = None
if Entry.objects.filter(project_id=project_id, entry_unique=1).exists():
firstentry = Entry.objects.filter(project_id=project_id, entry_unique=1).order_by('-entry_pubdate')[0]
else:
firstentry = None
if Entry.objects.filter(project_id=project_id).exists():
lastentry = Entry.objects.filter(project_id=project_id).order_by('-entry_pubdate')[0]
lastentrynumber = lastentry.entry_unique
else:
lastentrynumber = None
if request.method == "POST":
form = AddAccessForm(request.POST)
if form.is_valid():
p = form.save(commit=False)
adduserfromform = p.accessupdate
if User.objects.filter(username=adduserfromform).exists():
usertoadd = User.objects.get(username=adduserfromform)
projecttoadd = Access.objects.filter(project__id=project_id).order_by('-project__project_pubdate')[0]
projecttoadd.access_list.add(usertoadd)
else:
usertoadd = None
removeuserfromform = p.accessremove
if User.objects.filter(username=removeuserfromform).exists():
usertoremove = User.objects.get(username=removeuserfromform)
projecttoremove = Access.objects.filter(project__id=project_id).order_by('-project__project_pubdate')[0]
projecttoremove.access_list.remove(usertoremove)
else:
usertoremove = None
form.save()
return HttpResponseRedirect('/projects/get/%s/access' % project_id)
Traceback:
Traceback (most recent call last):
File "/webapps/filmeditdb/local/lib/python2.7/site-packages/django/core/handlers/base.py", line 114, in get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/webapps/filmeditdb/local/lib/python2.7/site-packages/django/contrib/auth/decorators.py", line 22, in _wrapped_view
return view_func(request, *args, **kwargs)
File "/webapps/filmeditdb/filmeditdb/docproject/views.py", line 284, in access
def access(request, project_id=1):
File "/webapps/filmeditdb/local/lib/python2.7/site-packages/django/db/models/manager.py", line 151, in get
return self.get_queryset().get(*args, **kwargs)
File "/webapps/filmeditdb/local/lib/python2.7/site-packages/django/db/models/query.py", line 310, in get
(self.model._meta.object_name, num))
MultipleObjectsReturned: get() returned more than one Access -- it returned 2!
2014-03-25 22:48:26 [17264] [INFO] Handling signal: winch
2014-03-25 22:48:26 [17264] [INFO] SIGWINCH ignored. Not daemonized

Adding a full traceback is generally recommeded, but from the error shown the issue is most likely on this line:
projecttoremove = Access.objects.get(project__id=project_id).order_by('-project__project_pubdate')[0]
Djangos get() always expects to return only 1 object. If there is 0 objects, it throws a DoesNotExist exception if there is more than 0, it throws MultipleObjectsReturned Exception.
Change that line to match the earlier line here:
projecttoadd = Access.objects.filter(project__id=project_id).order_by('-project__project_pubdate')[0]
Failing this, identify the erroneous call. The exception you are getting is quite clear.
At some point in the code, you are calling Access.objects.get() with some parameters and it is finding 2 objects instead of 1. Either fix the parameters, or moer likely switch it to use filter().

KeyError at /app/myRoute/ 'Status' - AJAX Post Django

From my view, I am simply returning
context = {'message':'Request Accepted'}
return HttpResponse(context)
I am calling this view with AJAX call and its returning Http 500
with error message
KeyError at /routes/AcceptRequest/
'status'
Request Method: POST
when i try to stringify the response in javascript it gives :
{"readyState":4,"responseText":"KeyError at /routes/AcceptRequest/\n'status'\n\nRequest Method: POST\nRequest URL: http://localhost:8000/routes/AcceptRequest/\nDjango Version: 1.4\nPython Executable: /home/subodh/joinwheelsv6/env/bin/python\nPython Version: 2.7.3\nPython Path: ['/home/subodh/joinwheelsv6/env/joinwheels', '/home/subodh/joinwheelsv6/env/lib/python2.7/site-packages/PIL-1.1.7-py2.7-linux-x86_64.egg', '/home/subodh/joinwheelsv6/env/lib/python2.7', '/home/subodh/joinwheelsv6/env/lib/python2.7/plat-linux2', '/home/subodh/joinwheelsv6/env/lib/python2.7/lib-tk', '/home/subodh/joinwheelsv6/env/lib/python2.7/lib-old', '/home/subodh/joinwheelsv6/env/lib/python2.7/lib-dynload', '/usr/lib/python2.7', '/usr/lib/python2.7/plat-linux2', '/usr/lib/python2.7/lib-tk', '/home/subodh/joinwheelsv6/env/local/lib/python2.7/site-packages', '/home/subodh/joinwheelsv6/env/lib/python2.7/site-packages']\nServer time: Thu, 17 Oct 2013 15:48:36 +0530\nInstalled Applications:\n('django.contrib.auth',\n 'django.contrib.contenttypes',\n 'django.contrib.sessions',\n 'django.contrib.sites',\n 'django.contrib.messages',\n 'django.contrib.staticfiles',\n 'django.contrib.admin',\n 'django.contrib.admindocs',\n 'south',\n 'account',\n 'socialaccounts',\n 'routes',\n 'Groups')\nInstalled Middleware:\n('django.middleware.common.CommonMiddleware',\n 'django.contrib.sessions.middleware.SessionMiddleware',\n 'django.middleware.csrf.CsrfViewMiddleware',\n 'django.contrib.auth.middleware.AuthenticationMiddleware',\n 'django.contrib.messages.middleware.MessageMiddleware')\n\nTraceback:\nFile \"/home/subodh/joinwheelsv6/env/local/lib/python2.7/site-packages/django/core/handlers/base.py\" in get_response\n 111. response = callback(request, *callback_args, **callback_kwargs)\nFile \"/home/subodh/joinwheelsv6/env/local/lib/python2.7/site-packages/django/views/generic/base.py\" in view\n 48. return self.dispatch(request, *args, **kwargs)\nFile \"/home/subodh/joinwheelsv6/env/local/lib/python2.7/site-packages/django/views/generic/base.py\" in dispatch\n 69. return handler(request, *args, **kwargs)\nFile \"/home/subodh/joinwheelsv6/env/joinwheels/routes/views.py\" in post\n 1157. return HttpResponse(response['Status'])\nFile \"/home/subodh/joinwheelsv6/env/local/lib/python2.7/site-packages/django/http/__init__.py\" in __getitem__\n 615. return self._headers[header.lower()][1]\n\nException Type: KeyError at /routes/AcceptRequest/\nException Value: 'status'\nRequest information:\nGET: No GET data\n\nPOST:\ncsrfmiddlewaretoken = u'Dxm5R7omqawuGtJ2v8AuOGsDXXdTCXCa'\nRequestID = u'7'\n\nFILES: No FILES data\n\nCOOKIES:\ncsrftoken = 'Dxm5R7omqawuGtJ2v8AuOGsDXXdTCXCa'\nsessionid = '139167c44021568cf40311753e0480fc'\n\nMETA:\nwsgi.version = \nwsgi.multiprocess = False\nRUN_MAIN = 'true'\nHTTP_REFERER = 'http://localhost:8000/account/TravelCard/Ride/2059/7/'\nGNOME_DESKTOP_SESSION_ID = 'this-is-deprecated'\nSERVER_PROTOCOL = 'HTTP/1.1'\nSERVER_SOFTWARE = 'WSGIServer/0.1 Python/2.7.3'\nSCRIPT_NAME = u''\nLESSOPEN = '| /usr/bin/lesspipe %s'\nREQUEST_METHOD = 'POST'\nLOGNAME = 'subodh'\nUSER = 'subodh'\nHTTP_ORIGIN = 'http://localhost:8000'\nPATH = '/home/subodh/joinwheelsv6/env/bin:/usr/lib/lightdm/lightdm:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games'\nQUERY_STRING = ''\nGNOME_KEYRING_CONTROL = '/tmp/keyring-KIXbNa'\nPS1 = '(env)\\\\[\\\\e]0;\\\\u#\\\\h: \\\\w\\\\a\\\\]${debian_chroot:+($debian_chroot)}\\\\u#\\\\h:\\\\w\\\\$ '\nDISPLAY = ':0'\nSSH_AGENT_PID = '2016'\nLANG = 'en_IN'\nTERM = 'xterm'\nSHELL = '/bin/bash'\nXDG_SESSION_PATH = '/org/freedesktop/DisplayManager/Session0'\nXAUTHORITY = '/home/subodh/.Xauthority'\nLANGUAGE = 'en_IN:en'\nSESSION_MANAGER = 'local/dell:#/tmp/.ICE-unix/1981,unix/dell:/tmp/.ICE-unix/1981'\nSHLVL = '1'\nMANDATORY_PATH = '/usr/share/gconf/ubuntu.mandatory.path'\nwsgi.url_scheme = 'http'\nCOMPIZ_CONFIG_PROFILE = ...

Well, you seem to be doing something else that you haven't explained. From your error, cleaned up slightly:
File "/home/subodh/joinwheelsv6/env/joinwheels/routes/views.py" in post
1157. return HttpResponse(response['Status'])
So you have some code in views.py that is looking for a Status field that doesn't exist. Of course, we have no idea why.

Change the return line:
import json
return HttpResponse(json.dumps(context))