Here is a very basic class for handling sessions on App Engine:
"""Lightweight implementation of cookie-based sessions for Google App Engine.
Classes:
Session
"""
import os
import random
import Cookie
from google.appengine.api import memcache
_COOKIE_NAME = 'app-sid'
_COOKIE_PATH = '/'
_SESSION_EXPIRE_TIME = 180 * 60
class Session(object):
"""Cookie-based session implementation using Memcached."""
def __init__(self):
self.sid = None
self.key = None
self.session = None
cookie_str = os.environ.get('HTTP_COOKIE', '')
self.cookie = Cookie.SimpleCookie()
self.cookie.load(cookie_str)
if self.cookie.get(_COOKIE_NAME):
self.sid = self.cookie[_COOKIE_NAME].value
self.key = 'session-' + self.sid
self.session = memcache.get(self.key)
if self.session:
self._update_memcache()
else:
self.sid = str(random.random())[5:] + str(random.random())[5:]
self.key = 'session-' + self.sid
self.session = dict()
memcache.add(self.key, self.session, _SESSION_EXPIRE_TIME)
self.cookie[_COOKIE_NAME] = self.sid
self.cookie[_COOKIE_NAME]['path'] = _COOKIE_PATH
print self.cookie
def __len__(self):
return len(self.session)
def __getitem__(self, key):
if key in self.session:
return self.session[key]
raise KeyError(str(key))
def __setitem__(self, key, value):
self.session[key] = value
self._update_memcache()
def __delitem__(self, key):
if key in self.session:
del self.session[key]
self._update_memcache()
return None
raise KeyError(str(key))
def __contains__(self, item):
try:
i = self.__getitem__(item)
except KeyError:
return False
return True
def _update_memcache(self):
memcache.replace(self.key, self.session, _SESSION_EXPIRE_TIME)
I would like some advices on how to improve the code for better security.
Note: In the production version it will also save a copy of the session in the datastore.
Note': I know there are much more complete implementations available online though I would like to learn more about this subject so please don't answer the question with "use that" or "use the other" library.
Here is a suggestion for simplifying your implementation.
You are creating a randomized temporary key that you use as the session's key in the memcache. You note that you will be storing the session in the datastore as well (where it will have another key).
Why not randomize the session's datastore key, and then use that as the one and only key, for both the database and the memcache (if necessary)? Does this simplification introduce any new security issues?
Here's some code for creating a randomized datastore key for the Session model:
# Get a random integer to use as the session's datastore ID.
# (So it can be stored in a cookie without being 'guessable'.)
random.seed();
id = None;
while None==id or Session.get_by_id( id ):
id = random.randrange( sys.maxint );
seshKey = db.Key.from_path( 'Session', id );
session = Session( key = seshKey );
To get the ID from the session (i.e. to store in the cookie) use:
sid = session.key().id();
To retrieve the session instance after the 'sid' has been read from the cookie:
session = Session.get_by_id( sid );
Here are a couple of additional security measures you could add.
First, I think it is pretty common to use information stored in the session instance to validate each new request. For example, you could verify that the IP address and user-agent don't change during a session:
newip = str( request.remote_addr );
if sesh.ip_addr != newip:
logging.warn( "Session IP has changed to %s." % newip);
newua = rh.request.headers.get( 'User-Agent', None );
if sesh.agent != newua:
logging.warn( "Session UA has changed to %s." % newua );
Also, perhaps it would be better to prevent the session from being renewed indefinitely? I think that sites such as Google will eventually ask you to sign-in again if you try to keep a session going for a long time.
I guess it would be easy to slowly decrease the _SESSION_EXPIRE_TIME each time the session gets renewed, but that isn't really a very good solution. Ideally the choice of when to force the user to sign-in again would take into account the flow and security requirements of your site.
Related
I am trying to pull data from Bloomberg using Python API. API package comes with example codes and the programs that only requires local host work perfectly. However, the programs that uses other authorization ways are always stuck with the error:
Connecting to port 8194 on localhost
TokenGenerationFailure = {
reason = {
source = "apitkns (apiauth) on ebbdbp-ob-053"
category = "NO_AUTH"
errorCode = 12
description = "User not in emrs userid=NA\mds firm=22691"
subcategory = "INVALID_USER"
}
}
Failed to get token
No authorization
I saw one more person having similar problem but instead of solving it he chose to just use local host. I can't always use localhost because I will have to assist and troubleshoot for other users. So I need a hint how to overcome this error.
My question is how can I set the userid anything other than OS_LOGON which automatically uses the login credentials of my account so that I can use other users' name when needed? I tried to change OS_LOGON with the user name but it didn't work.
The full program I am trying to run is:
"""SnapshotRequestTemplateExample.py"""
from __future__ import print_function
from __future__ import absolute_import
import datetime
from optparse import OptionParser, OptionValueError
import blpapi
TOKEN_SUCCESS = blpapi.Name("TokenGenerationSuccess")
TOKEN_FAILURE = blpapi.Name("TokenGenerationFailure")
AUTHORIZATION_SUCCESS = blpapi.Name("AuthorizationSuccess")
TOKEN = blpapi.Name("token")
def authOptionCallback(_option, _opt, value, parser):
vals = value.split('=', 1)
if value == "user":
parser.values.auth = "AuthenticationType=OS_LOGON"
elif value == "none":
parser.values.auth = None
elif vals[0] == "app" and len(vals) == 2:
parser.values.auth = "AuthenticationMode=APPLICATION_ONLY;"\
"ApplicationAuthenticationType=APPNAME_AND_KEY;"\
"ApplicationName=" + vals[1]
elif vals[0] == "userapp" and len(vals) == 2:
parser.values.auth = "AuthenticationMode=USER_AND_APPLICATION;"\
"AuthenticationType=OS_LOGON;"\
"ApplicationAuthenticationType=APPNAME_AND_KEY;"\
"ApplicationName=" + vals[1]
elif vals[0] == "dir" and len(vals) == 2:
parser.values.auth = "AuthenticationType=DIRECTORY_SERVICE;"\
"DirSvcPropertyName=" + vals[1]
else:
raise OptionValueError("Invalid auth option '%s'" % value)
def parseCmdLine():
"""parse cli arguments"""
parser = OptionParser(description="Retrieve realtime data.")
parser.add_option("-a",
"--ip",
dest="hosts",
help="server name or IP (default: localhost)",
metavar="ipAddress",
action="append",
default=[])
parser.add_option("-p",
dest="port",
type="int",
help="server port (default: %default)",
metavar="tcpPort",
default=8194)
parser.add_option("--auth",
dest="auth",
help="authentication option: "
"user|none|app=<app>|userapp=<app>|dir=<property>"
" (default: %default)",
metavar="option",
action="callback",
callback=authOptionCallback,
type="string",
default="user")
(opts, _) = parser.parse_args()
if not opts.hosts:
opts.hosts = ["localhost"]
if not opts.topics:
opts.topics = ["/ticker/IBM US Equity"]
return opts
def authorize(authService, identity, session, cid):
"""authorize the session for identity via authService"""
tokenEventQueue = blpapi.EventQueue()
session.generateToken(eventQueue=tokenEventQueue)
# Process related response
ev = tokenEventQueue.nextEvent()
token = None
if ev.eventType() == blpapi.Event.TOKEN_STATUS or \
ev.eventType() == blpapi.Event.REQUEST_STATUS:
for msg in ev:
print(msg)
if msg.messageType() == TOKEN_SUCCESS:
token = msg.getElementAsString(TOKEN)
elif msg.messageType() == TOKEN_FAILURE:
break
if not token:
print("Failed to get token")
return False
# Create and fill the authorization request
authRequest = authService.createAuthorizationRequest()
authRequest.set(TOKEN, token)
# Send authorization request to "fill" the Identity
session.sendAuthorizationRequest(authRequest, identity, cid)
# Process related responses
startTime = datetime.datetime.today()
WAIT_TIME_SECONDS = 10
while True:
event = session.nextEvent(WAIT_TIME_SECONDS * 1000)
if event.eventType() == blpapi.Event.RESPONSE or \
event.eventType() == blpapi.Event.REQUEST_STATUS or \
event.eventType() == blpapi.Event.PARTIAL_RESPONSE:
for msg in event:
print(msg)
if msg.messageType() == AUTHORIZATION_SUCCESS:
return True
print("Authorization failed")
return False
endTime = datetime.datetime.today()
if endTime - startTime > datetime.timedelta(seconds=WAIT_TIME_SECONDS):
return False
def main():
"""main entry point"""
global options
options = parseCmdLine()
# Fill SessionOptions
sessionOptions = blpapi.SessionOptions()
for idx, host in enumerate(options.hosts):
sessionOptions.setServerAddress(host, options.port, idx)
sessionOptions.setAuthenticationOptions(options.auth)
sessionOptions.setAutoRestartOnDisconnection(True)
print("Connecting to port %d on %s" % (
options.port, ", ".join(options.hosts)))
session = blpapi.Session(sessionOptions)
if not session.start():
print("Failed to start session.")
return
subscriptionIdentity = None
if options.auth:
subscriptionIdentity = session.createIdentity()
isAuthorized = False
authServiceName = "//blp/apiauth"
if session.openService(authServiceName):
authService = session.getService(authServiceName)
isAuthorized = authorize(authService, subscriptionIdentity,
session, blpapi.CorrelationId("auth"))
if not isAuthorized:
print("No authorization")
return
else:
print("Not using authorization")
.
.
.
.
.
finally:
session.stop()
if __name__ == "__main__":
print("SnapshotRequestTemplateExample")
try:
main()
except KeyboardInterrupt:
print("Ctrl+C pressed. Stopping...")
This example is intended for Bloomberg's BPIPE product and as such includes the necessary authorization code. For this example, if you're connecting to the Desktop API (typically localhost:8194) you would want to pass an auth parameter of "none". Note that this example is for the mktdata snapshot functionality which isn't supported by Desktop API.
You state you're trying to troubleshoot on behalf of other users, presumably traders using BPIPE under their credentials. In this case you would need to create an Identity object to represent that user.
This would be done thusly:
# Create and fill the authorization request
authRequest = authService.createAuthorizationRequest()
authRequest.set("authId", STRING_CONTAINING_USERS_EMRS_LOGON)
authRequest.set("ipAddress", STRING_OF_IP_ADDRESS_WHERE_USER_IS_LOGGED_INTO_TERMINAL)
# Send authorization request to "fill" the Identity
session.sendAuthorizationRequest(authRequest, identity, cid)
Please be aware of potential licensing compliance issues when using this approach as this can have serious consequences. If in any doubt, approach your firm's market data team who will be able to ask their Bloomberg contacts.
Edit:
As asked in the comments, it's useful to elaborate on the other possible parameters for the AuthorizationRequest.
"uuid" + "ipAddress"; this would be the default method of authenticating users for Server API. On BPIPE this would require Bloomberg to explicitly enable it for you. The UUID is the unique integer identifier assigned to each Bloomberg Anywhere user. You can look this up in the terminal by running IAM
"emrsId" + "ipAddress"; "emrsId" is a deprecated alias for "authId". This shouldn't be used anymore.
"authId" + "ipAddress"; "authId" is the String defined in EMRS (the BPIPE Entitlements Management and Reporting System) or SAPE (the Server API's equivalent of EMRS) that represents each user. This would typically be that user's OS login details (e.g. DOMAIN/USERID) or Active Directory property (e.g. mail -> blah#blah.blah)
"authId" + "ipAddress" + "application"; "application" is the application name defined on EMRS/SAPE. This will check to see whether the user defined in authId is enabled for the named application on EMRS. Using one of these user+app style Identity objects in requests should record usage against both the user and application in the EMRS usage reports.
"token"; this is the preferred approach. Using the session.generateToken functionality (which can be seen in the original question's code snippet) will result in an alphanumeric string. You'd pass this as the only parameter into the Authorization request. Note that the token generation system is virtualization-aware; if it detects it's running in Citrix or a remote desktop it will report the IP address of the display machine (or one hop towards where the user actually is).
I am using Sanic with 2 workers. I am trying to get a billing system working, i.e. Counting how many times a user hit the API endpoint. Following is my code:
class User(object):
def __init__(self, id, name, age, address, mobile, credits=0):
self.id = id
self.name = name
self.credits = count
self.details = {"age": age, "address": address, "mobile_number": mobile}
The above Users Class is used to make objects that I have uploaded onto Redis using another python script as follows:
user = User(..., credits = 10)
string_obj = json.dumps(user)
root.set(f"{user.user_id}", string_obj)
The main issue arises when I want to maintain a count of the number of hits an endpoint receives and track it withing the user object and upload it back onto Redis. My code is as follows:
from sanic_redis_ext import RedisExtension
app = Sanic("Testing")
app.config.update(
{
"REDIS_HOST": "127.0.0.1",
"REDIS_PORT": 6379,
"REDIS_DATABASE": 0,
"REDIS_SSL": None,
"REDIS_ENCODING": "utf-8",
"REDIS_MIN_SIZE_POOL": 1,
"REDIS_MAX_SIZE_POOL": 10,
})
#app.route("/test", methods=["POST"])
#inject_user()
#protected()
async def foo(request, user):
user.credits -= 1
if user.credits < 0:
user.credits = 0
return sanic.response.text("Credits Exhausted")
result = process(request)
if not result:
user.credits += 1
await app.redis.set(f"{user.user_id}", json.dumps(user))
return sanic.response.text(result)
And this is how I am retrieving the user:
async def retrieve_user(request, *args, **kwargs):
if "user_id" in kwargs:
user_id = kwargs.get("user_id")
else:
if "payload" in kwargs:
payload = kwargs.get("payload")
else:
payload = await request.app.auth.extract_payload(request)
if not payload:
raise exceptions.MissingAuthorizationHeader()
user_id = payload.get("user_id")
user = json.loads(await app.redis.get(user_id))
return user
When I use JMeter to test the API endpoint with 10 threads acting as the same user, the credit system does not seem to work. In this case, as the user starts with 10 credits, they may end up with 7 or 8 (not predictable) credits left whereas they should have 0 left. According to me, this is due to the workers not sharing the user object and not having the updated copy of the variable which is causing them to overwrite each others update. Can anyone help me find a way out of this so that even if the same user simultaneously hits the endpoint, he/she should be billed perfectly and the user object should be saved back into Redis.
The problem is that you read the credits info from Redis, deduct it, then save it back it to Redis, which is not an atomic process. It's a concurrency issue.
I don't know about Python, so I'll just use pseudo code.
First set 10 credits for user {user_id}.
app.redis.set("{user_id}:credits", 10)
Then this user comes in
# deduct 1 from the user credits and get the result
int remaining_credits=app.redis.incryBy ("{user_id}:credits",-1)
if(remaining_credits<=0){
return sanic.response.text("Credits Exhausted")} else{
return "sucess" # or some other result}
Save your user info with payload somewhere else and retrieve the "{user_id}:credits"and combine them when you retrieve the user.
This is only my second task (bug I need to fix) in a Python\Flask\SQLAlchemy\Marshmallow system I need to work on. So please try to be easy with me :)
In short: I'd like to approve an apparently invalid request.
In details:
I need to handle a case in which a user might send a request with some json in which he included by mistake a duplicate value in a list.
For example:
{
"ciphers": [
"TLS_AES_256_GCM_SHA384",
"AES256-SHA256"
],
"is_default": true,
"tls_versions": [
"tls10",
"tls10",
"tls11",
]
}
What I need to do is to eliminate one of the duplicated tls1.0 values, but consider the request as valid, update the db with the correct and distinct tls versions, and in the response return the non duplicated json in body.
Current code segments are as follows:
tls Controller:
...
#client_side_tls_bp.route('/<string:tls_profile_id>', methods=['PUT'])
def update_tls_profile_by_id(tls_profile_id):
return update_entity_by_id(TlsProfileOperator, entity_name, tls_profile_id)
...
general entity controller:
...
def update_entity_by_id(operator, entity_name, entity_id):
"""flask route for updating a resource"""
try:
entity_body = request.get_json()
except Exception:
return make_custom_response("Bad Request", HTTPStatus.BAD_REQUEST)
entity_obj = operator.get(g.tenant, entity_id, g.correlation)
if not entity_obj:
response = make_custom_response(http_not_found_message(entity_name, entity_id), HTTPStatus.NOT_FOUND)
else:
updated = operator.update(g.tenant, entity_id, entity_body, g.correlation)
if updated == "accepted":
response = make_custom_response("Accepted", HTTPStatus.ACCEPTED)
else:
response = make_custom_response(updated, HTTPStatus.OK)
return response
...
tls operator:
...
#staticmethod
def get(tenant, name, correlation_id=None):
try:
tls_profile = TlsProfile.get_by_name(tenant, name)
return schema.dump(tls_profile)
except NoResultFound:
return None
except Exception:
apm_logger.error(f"Failed to get {name} TLS profile", tenant=tenant,
consumer=LogConsumer.customer, correlation=correlation_id)
raise
#staticmethod
def update(tenant, name, json_data, correlation_id=None):
schema.load(json_data)
try:
dependant_vs_names = VirtualServiceOperator.get_dependant_vs_names_locked_by_client_side_tls(tenant, name)
# locks virtual services and tls profile table simultaneously
to_update = TlsProfile.get_by_name(tenant, name)
to_update.update(json_data, commit=False)
db.session.flush() # TODO - need to change when 2 phase commit will be implemented
snapshots = VirtualServiceOperator.get_snapshots_dict(tenant, dependant_vs_names)
# update QWE
# TODO handle QWE update atomically!
for snapshot in snapshots:
QWEController.update_abc_services(tenant, correlation_id, snapshot)
db.session.commit()
apm_logger.info(f"Update successfully {len(dependant_vs_names)} virtual services", tenant=tenant,
correlation=correlation_id)
return schema.dump(to_update)
except Exception:
db.session.rollback()
apm_logger.error(f"Failed to update {name} TLS profile", tenant=tenant,
consumer=LogConsumer.customer, correlation=correlation_id)
raise
...
and in the api schema class:
...
#validates('_tls_versions')
def validate_client_side_tls_versions(self, value):
if len(noDuplicatatesList) < 1:
raise ValidationError("At least a single TLS version must be provided")
for tls_version in noDuplicatatesList:
if tls_version not in TlsProfile.allowed_tls_version_values:
raise ValidationError("Not a valid TLS version")
...
I would have prefer to solve the problem in the schema level, so it won't accept the duplication.
So, as easy as it is to remove the duplication from the "value" parameter value, how can I propagate the non duplicates list back in order to use it to update the db and the response?
Thanks.
I didn't test but I think mutating value in the validation function would work.
However, this is not really guaranteed by marshmallow's API.
The proper way to do it would be to add a post_load method to de-duplicate.
#post_load
def deduplicate_tls(self, data, **kwargs):
if "tls_versions" in data:
data["tls_version"] = list(set(data["tls_version"]))
return data
This won't maintain the order, so if the order matters, or for issues related to deduplication itself, see https://stackoverflow.com/a/7961390/4653485.
So I launched 20 uswgi processes, and I'm using Nginx to load balance them like detailed here
What I'm noticing is that when one worker is hit it will load up memcached after not seeing the results it needs in memcached. Thereafter that workder will find them in memcached. But other workers still won't see the memcached results until they load them up themselves.
Any idea what could be going on? Here's the relevant code:
import memcache
import msgpack
from flask import Flask
app = Flask(__name__, static_url_path = "")
app.config['PROPAGATE_EXCEPTIONS'] = True
class MemCachedWrapper(object):
"""Simple wrapper around memcached to handle translating keys"""
def make_key(self, key):
#is this crazy?? memcached only wants strings and won't accept spaces
return str(key).replace(' ','')
def __init__(self, servers, debug=0, server_max_value_length=1024*1024*3,
flush_on_start=False):
#Set up memcached
#see README.md for instructions on setting server_max_value_length on memcached (-I option)
self.cache = memcache.Client(servers, debug=debug,
server_max_value_length=server_max_value_length)
if flush_on_start:
self.cache.flush_all()
def get(self, key):
return self.cache.get(self.make_key(key))
def get_multi(self, keys):
m = self.make_key
return self.cache.get_multi([m(k) for k in keys])
def set(self, key, value):
return self.cache.set(self.make_key(key), value)
def delete(self, key):
return self.cache.delete(self.make_key(key))
cache =MemCachedWrapper([MEMCACHE_URL], debug=0,
server_max_value_length=MEMCACHE_MAX_VALUE_LENGTH)
def _get_range(key_beg, key_end):
keys = get_keys(key_beg, key_end)
def stream_results():
mc_results = None
for scality_key in keys:
if not mc_results:
mc_results = cache.get_multi([k[0].encode('utf-8') for k in keys])
scality_key = scality_key.encode('utf-8')
obj = mc_results.get(scality_key)
if obj:
yield dvid_key, obj
else:
print "key miss"
response = session.get(SCALITY_URL % scality_key)
cache.set(scality_key, response.content)
yield dvid_key, response.content
return stream_results()
#app.route('/api/keyvalue_range/<key_beg>/<key_end>/', methods=['GET'])
def get_range(key_beg, key_end):
results = _get_range(key_beg, key_end)
packed = msgpack.packb(list(results), use_bin_type=True)
return Response(packed, mimetype='application/x-msgpack')
if __name__ == '__main__':
app.run(host=host, port=PORT, debug=True)
This isn't a great answer, but I ended up using pylibmc as my memcached client library instead and it seems to have fixed the issue. No idea what was going on.
As for changes to the above code, basically just these lines:
import pylibmc as memcache
...
self.cache = memcache.Client(servers, binary=True,
behaviors={"tcp_nodelay": True, "ketama": True, "no_block": True})
def manage_bread_crumb(self, testvar):
stzr = ''
if self.session.get('temp_sesison') != None:
stzr = pickle.loads(str(self.session.get('temp_sesison')))
string = stzr + testvar
self.session['temp_sesison'] = pickle.dumps(string)
self.temp_session = pickle.loads(str(self.session.get('temp_sesison')))
def __init__(self, request):
RequestHandler.__init__(self, request)
Jinja2Mixin.__init__(self)
if self.session.get('profile_user') is not None:
self.profile_user = pickle.loads(str(self.session.get('profile_user')))
else:
self.profile_user = None
self.temp_session = pickle.loads(str(self.session.get('temp_sesison')))\
if self.session.get('temp_sesison') else None
I concatenated a string and append it to a session created by tipfy for each and every request. But the session does not get updated.
This is how I call the session in another handler:
def some_hanlder(self, secure_page_handler):
self.manage_bread_crumb('some name')
print self.temp_session
Can anyone help me?
All I have to do was change the tipfy session management from cookies to memcache . After that it works fine,