We Keep Coding

How to differentiate access to data at the database server level?

In DB i have three roles: guest, client and admin.
In my django project, there are three connections under these roles
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': 'test',
'USER': 'guest',
'PASSWORD': 'guest',
'HOST': 'localhost',
'PORT': 5432,
},
'admin': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': 'test',
'USER': 'admin',
'PASSWORD': 'admin',
'HOST': 'localhost',
'PORT': 5432,
},
'customer': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': 'test',
'USER': 'customer',
'PASSWORD': 'customer',
'HOST': 'localhost',
'PORT': 5432,
}
}
How and where can I change the connection to the database depending on whether the user is authenticated or not?

I am presuming that you are using psycopg2 to connect to the Postgresql RDBMS. What I would do is specify what Postgresql user you want to use before you execute your query.
For example:
import psycopg2
def func1():
conn = psycopg2.connect(database = "exampledb", user = "user1", password = "user1password", host = "127.0.0.1", port = "5432")
cur = conn.cursor()
cur.execute("SELECT * FROM schema_name.table_name;")
rows = cur.fetchall()
for row in rows:
print(row)
def func2():
conn = psycopg2.connect(database = "exampledb", user = "user2", password = "user2password", host = "127.0.0.1", port = "5432")
cur = conn.cursor()
cur.execute("INSERT INTO schema_name.table_name (col1, col2) VALUES(1, 2);")
rows = cur.fetchall()
for row in rows:
print(row)
I would also be very careful with the admin user, from a security standpoint I would not allow this account to be used for the server-side scripting, this is because if and sql injection is executed, then a lot of harm could be caused. For prevention of sql injections in python I would recommend this: https://realpython.com/prevent-python-sql-injection/

Using quicksight boto3 update_data_source

Summary
What is the right syntax to use update_data_source Quicksight boto3 to change credentials ?
Context
I am trying to use update data source method for Quicksight on boto3 to update my Redshift credentials in Quicksight.
My issue is that it is passing a dictionary as key to another dictionary. How can I unpack that to get to the username / password for Redshift ?
Code
My code looks like this :
def main():
qs = boto3.client('quicksight', region_name=region_name)
response = qs.update_data_source(
AwsAccountId='awsaccountid',
DataSourceId='datasourceid',
Name='qs_test',
Credentials={
{
'CredentialPair':{
'Username': 'test_user'
'Password': 'my_pass'
}
}
}
)
print(response)
main()
Also tried the below
response = qs.update_data_source(
AwsAccountId='awsaccountid',
DataSourceId='datasourceid',
Name='qs_test',
Credentials={CredentialPair
{
RedshiftParameters=[
{
'Database': 'dbname',
'ClusterId': 'clusterid'
}
}
],
Credentials={
'CredentialPair': {
'Username': 'test_user',
'Password': 'my_pass'
}
}
)
print(response)

The below syntax works :
def main():
qs = boto3.client('quicksight', region_name=region_name)
response = qs.update_data_source(
AwsAccountId='awsaccountid',
DataSourceId='datasourceid',
Name='qs_test',
DataSourceParameters={
'RedshiftParameters'={
'Database': 'dbname',
'ClusterId': 'clusterid'
}
}
}
Credentials={
'CredentialPair':{
'Username': 'test_user'
'Password': 'my_pass'
}
}
)
print(response)
main()

Django+mysqlclient not closing database connections

I'm using 2 mysql database connections. After every page request in django, the connection count ( as displayed by SHOW STATUS LIKE 'Conn%' increases by 2 each time.
Python 3.4.0
django 1.8.2
mysqlclient 1.3.6
( Both Windows and Linux seem to have the issue )
Initially i accessed the second database directly through MySQLdb but now i've switched to using django.db.connections['...'] to access it.
This is my settings.py
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'myapp',
'USER': 'root',
'PASSWORD': '',
'HOST': '',
'PORT': '',
},
'analytics': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'myapp_analytics',
'USER': 'root',
'PASSWORD': '',
'HOST': '',
'PORT': '',
},
}
And these are the classes that i wrote to for easy usage:
import MySQLdb as DBConnector
import django
class DictCursorEmulator:
def __init__(self, connection):
self.cursor = connection.cursor()
def _execute(self,query_str,query_args):
self.cursor.execute(query_str,query_args)
def fetchall(self):
"Returns all rows from a cursor as a dict"
desc = self.cursor.description
return [
dict(zip([col[0] for col in desc], row))
for row in self.cursor.fetchall()
]
def fetchone(self):
desc = self.cursor.description
return dict(zip([col[0] for col in desc], row))
#I probably should have used instance methods here but there's too much code to update ;_;
class DatabaseInterface:
#classmethod
def execute(self, queryStr, args):
cursor = DictCursorEmulator(django.db.connections['analytics'])
cursor._execute(queryStr,args)
return cursor
#classmethod
def autocommit(cls, set_to):
django.db.connections['analytics'].autocommit(set_to)
#classmethod
def commit(cls):
django.db.connections['analytics'].commit()
#classmethod
def rollback(cls):
django.db.connections['analytics'].rollback()
I've tried excluding the entire file but it still makes the connections count go up by 2, So i'm starting to think it's not a a problem with my code. Does anyone know how to fix this?
Update: Apparently, Connections shows the total number of connections
Threads connected shows 2, which is expected. That solves my problem. I just had a lot of weird errors popping up on the DB side.

Flask-RESTful - don't return object property instead of returning null

Let's say I've got a clients table with id, name and email fields. An email field is optional.
The code looks like this:
client_fields = {
'id' : fields.String,
'name' : fields.String,
'email' : fields.String
}
And for displaying:
class ClientList(Resource):
#marshal_with(client_fields)
def get(self):
return model.Client.query.all()
When email is not provided, API returns JSON like this:
{
"id": "1",
"name": "John Doe",
"email": null
}
But instead I want it to return this object:
{
"id": "1",
"name": "John Doe"
}
Which basically means that instead of a property with null value I want it to return no such property at all. Is there a way to achieve that?

I would use the marshal function instead of the marshal_with decorator:
class ClientList(Resource):
def get(self):
clients = []
for client in model.Client.query.all():
if client.email:
clients.append(marshal(client_fields))
else:
clients.append(marshal(client_fields_no_email))
return clients
Or even better
class ClientList(Resource):
def get(self):
return [client_marshal(client) for client in model.Client.query.all()]
with
def client_marshal(client):
if client.email:
return {'id' : fields.String,
'name' : fields.String,
'email' : fields.String}
else:
return {'id' : fields.String,
'name' : fields.String}

There are two ways it can be done, pre-marshalling and post-marshalling modification. Pre-marshalling removes any default values given to field names in the client_fields dict but post-marshalling preserves them.
In pre-marshalling method, you have to pass a modified fields dict to marshal function if client's email is None.
For example;
import json
from flask_restful import fields, marshal, marshal_with
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
client_fields = {
'id': fields.String,
'name': fields.String,
'email': fields.String
}
def get():
clients =[Client(1, 'Tom'), Client(2, 'John', 'john#example.com')]
return [marshal(client, client_fields if client.email else {k: v for k, v in client_fields.items() if k != 'email'}) for client in clients]
print(json.dumps(get()))
Output;
[{"id": "1", "name": "Tom"}, {"email": "john#example.com", "id": "2", "name": "John"}]
In post-marshalling you have to remove the email field of the OrderedDict returned by marshal_with if it is None.
The de_none function by default removes all fields those are None or you have to explicitly pass the field names if that's not desired and you have to pass envelope argument too if marshal_with takes the same.
from functools import wraps
import json
from flask_restful import fields, marshal, marshal_with
client_fields = {
'id': fields.String,
'name': fields.String,
#'email': fields.String(default='user#example.com')
'email': fields.String
}
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
def de_none(envelope=None, *fields):
def decorator(func):
def dict_remove(d):
if fields:
for field in fields:
if d[field] is None:
d.pop(field)
else:
for k, v in d.items():
if v is None:
d.pop(k)
#wraps(func)
def decorated(*args, **kwargs):
data = result = func(*args, **kwargs)
if isinstance(result, tuple):
data = result[0]
if envelope:
data = data[envelope]
if isinstance(data, (list, tuple)):
for d in data:
dict_remove(d)
else:
dict_remove(data)
return result
return decorated
return decorator
#de_none()
#marshal_with(client_fields)
def get():
#return [Client(1, 'Tom'), Client(2, 'john', 'john#example.com')], 200, {'Etag': 'blah'}
#return [Client(1, 'Tom'), Client(2, 'john', 'john#example.com')]
#return Client(1, 'Tom'), 200, {'Etag': 'foo'}
return Client(1, 'Tom')
print(json.dumps(get()))
#de_none()
#marshal_with(client_fields)
def get():
return Client(2, 'John', 'john#example.com'), 201, {'Etag': 'ok'}
print(json.dumps(get()))
Output;
{"id": "1", "name": "Tom"}
{"email": "john#example.com", "id": "2", "name": "John"}
UPDATE Request hooks
The app.after_request decorator can be used to modify response object. Any default values given to fields are preserved.
The remove_none_fields request hook takes fields parameter which can be None to remove all fields with None value or a list of field names to selectively remove.
import json
from flask import Flask, Response
from flask_restful import fields, marshal_with, Api, Resource
app = Flask(__name__)
api = Api(app)
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
client_fields = {
'id': fields.String,
'name': fields.String,
'email': fields.String,
'age': fields.String
}
class ClientList(Resource):
#marshal_with(client_fields)
def get(self):
clients =[Client(1, 'Tom'), Client(2, 'John', 'john#example.com')]
return clients, 200
#app.after_request
def remove_none_fields(resp, fields=('email',)):
"""
removes all None fields
"""
if not 'application/json' in resp.content_type:
return resp
def dict_remove(d, fields):
if fields:
for field in fields:
if d[field] is None:
d.pop(field)
else:
for k, v in tuple(d.items()):
if v is None:
d.pop(k)
data = json.loads(resp.get_data())
if isinstance(data, list):
for obj in data:
dict_remove(obj, fields)
else:
dict_remove(data, fields)
resp.set_data(json.dumps(data, indent=1))
resp.content_length = resp.calculate_content_length()
return resp
api.add_resource(ClientList, '/')
if __name__ == '__main__':
app.run(debug=True)
Output;
[
{
"age": null,
"name": "Tom",
"id": "1"
},
{
"age": null,
"email": "john#example.com",
"name": "John",
"id": "2"
}
]
update patching flask_restful.marshal
I filter out None values in a genexp inside marshal function and replace flask_restful.marshal with marshal defined here.
from collections import OrderedDict
from flask import Flask
import flask_restful
from flask_restful import fields, marshal_with, Api, Resource
app = Flask(__name__)
api = Api(app)
class Client(object):
def __init__(self, id, name, email=None):
self.id = id
self.name = name
self.email = email
client_fields = {
'id': fields.String,
'name': fields.String,
'email': fields.String,
}
def marshal(data, fields, envelope=None):
def make(cls):
if isinstance(cls, type):
return cls()
return cls
if isinstance(data, (list, tuple)):
return (OrderedDict([(envelope, [marshal(d, fields) for d in data])])
if envelope else [marshal(d, fields) for d in data])
items = ((k, marshal(data, v) if isinstance(v, dict)
else make(v).output(k, data))
for k, v in fields.items())
#filtering None
items = ((k,v) for k, v in items if v is not None)
return OrderedDict([(envelope, OrderedDict(items))]) if envelope else OrderedDict(items)
flask_restful.marshal = marshal
class ClientList(Resource):
#marshal_with(client_fields)
def get(self):
clients =[Client(1, 'Tom'), Client(2, 'John', 'john#example.com')]
return clients, 200
api.add_resource(ClientList, '/')
if __name__ == '__main__':
app.run(debug=True)
Output;
[
{
"id": "1",
"name": "Tom"
},
{
"email": "john#example.com",
"id": "2",
"name": "John"
}
]

You should use the skip_none property of the #marshal decorator. Its much more convenient than the approaches suggested in the other answers.
#marshal(some_model, skip_none=True)
def get():
...
The documentation can be found here: https://flask-restplus.readthedocs.io/en/stable/marshalling.html
This is also possible using Flask Restx :)

Flask: Decorator to verify JSON and JSON Schema

I have a flask application with calls expecting JSON payload. Before each call is processed, I have a 2-step error checking process:
Assert that the payload is a valid JSON
Assert that the JSON payload complies with a specific schema
Which is implemented in the following fashion:
#app.route('/activate', methods=['POST'])
def activate():
request_id = request.__hash__()
# Assert that the payload is a valid JSON
try:
input = request.json
except BadRequest, e:
msg = "payload must be a valid json"
return jsonify({"error": msg}), 400
# JSON Schema Validation
try:
validate(request.json, app.config['activate_schema'])
except ValidationError, e:
return jsonify({"error": e.message}), 400
Since this code is duplicated over many calls, I wonder If I can elegantly move it to a decorator, something in the formof:
#validate_json
#validate_schema(schema=app.config['activate_schema'])
#app.route('/activate', methods=['POST'])
def activate():
....
The problem is that the request argument is implicit: I can refer to it within the function, but it is not a parameter to it. Therefore, I am not sure how to use it within the decorator.
How can I implement the validation checks using Python decorators?

Just use the request context global in your decorator. It is available during any request.
from functools import wraps
from flask import (
current_app,
jsonify,
request,
)
def validate_json(f):
#wraps(f)
def wrapper(*args, **kw):
try:
request.json
except BadRequest, e:
msg = "payload must be a valid json"
return jsonify({"error": msg}), 400
return f(*args, **kw)
return wrapper
def validate_schema(schema_name):
def decorator(f):
#wraps(f)
def wrapper(*args, **kw):
try:
validate(request.json, current_app.config[schema_name])
except ValidationError, e:
return jsonify({"error": e.message}), 400
return f(*args, **kw)
return wrapper
return decorator
Apply these decorators before applying the #route decorator; you want to register the wrapped function, not the original function for the route:
#app.route('/activate', methods=['POST'])
#validate_json
#validate_schema('activate_schema')
def activate():
input = request.json

now you can use #expect_json directly
For Example
from flask import Flask, jsonify, g, url_for
from flask_expects_json import expects_json
# example imports
from models import User
from orm import NotUniqueError
app = Flask(__name__)
schema = {
'type': 'object',
'properties': {
'name': {'type': 'string'},
'email': {'type': 'string'},
'password': {'type': 'string'}
},
'required': ['email', 'password']
}
#app.route('/register', methods=['POST'])
#expects_json(schema)
def register():
# if payload is invalid, request will be aborted with error code 400
# if payload is valid it is stored in g.data
# do something with your data
user = User().from_dict(g.data)
try:
user.save()
except NotUniqueError as e:
# exception path: duplicate database entry
return jsonify(dict(message=e.message)), 409
# happy path: json response
resp = jsonify(dict(auth_token=user.encode_auth_token(), user=user.to_dict()})
resp.headers['Location'] = url_for('users.get_user', user_id=user.id)
return resp, 201
or
from flask import Flask
from flask_expects_json import expects_json
app = Flask(__name__)
schema = {
'type': 'object',
'properties': {
'name': {'type': 'string', "minLength": 4, "maxLength": 15},
'mobile': {'type': 'string', "pattern": "^[1-9]{1}[0-9]{9}$"},
'email': {'type': 'string', "pattern": "[^#]+#[^#]+\.[^#]"},
'password': {'type': 'string', "pattern": "^.*(?=.{8,})(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[!##$%^&+=]).*$"}
},
'required': ['name', 'mobile', 'email', 'password']
}
#app.route('/', methods=['POST'])
#expects_json(schema)
def index():
values = request.get_json()
print(values)
return values
get more from here

A late answer, but you're probably looking for something like marshmallow (flask-marshmallow) or toastedmarshmallow.