I'm using an interceptor to check the validity of a token passed by the user in my tornado application.
def token_authenticate():
def wrapper(self, transforms, *args, **kwargs):
def _throw_error(self):
print 'writing basic auth'
if self._headers_written:
raise Exception('headers have already been written')
self.write(json.dumps({'auth': 'false'}))
self.finish()
return False
request = self.request
try:
token = request.arguments.get('token')[0]
if not token:
return _throw_error(self)
session = Instance().get_session()
user_token = session.query(UserToken)\
.filter(UserToken.token == token)\
.filter(UserToken.expires > datetime.utcnow())\
.one()
if user_token:
self.token = user_token
self.user = user_token.user
else:
print 'no user token'
return _throw_error(self)
except Exception, e:
print 'exception ' + e
return _throw_error(self)
return True
return wrapper
def interceptor(func):
def classwrapper(cls):
def wrapper(old):
def inner(self, transforms, *args, **kwargs):
log.debug('Invoking wrapper %s', func)
ret = func(self, transforms, *args, **kwargs)
if ret:
return old(self, transforms, *args, **kwargs)
else:
return ret
return inner
cls._execute = wrapper(cls._execute)
return cls
return classwrapper
## HANDLER
#interceptor(token_authenticate())
class SampleAuthenticatedRequestHandler(BaseHandler):
def get(self):
self.write({'response': self.user.as_dict()})
The request receives an empty response when the token is missing/invalid.
> curl localhost:8888/test -I -v
* Adding handle: conn: 0x7fb18b004000
* Adding handle: send: 0
* Adding handle: recv: 0
* Curl_addHandleToPipeline: length: 1
* - Conn 0 (0x7fb18b004000) send_pipe: 1, recv_pipe: 0
* About to connect() to localhost port 8888 (#0)
* Trying ::1...
* Connected to localhost (::1) port 8888 (#0)
> HEAD /test HTTP/1.1
> User-Agent: curl/7.30.0
> Host: localhost:8888
> Accept: */*
>
* Empty reply from server
* Connection #0 to host localhost left intact
curl: (52) Empty reply from server
Is there something missing? Is this the best way to abstract authentication away from my handler? I was hoping to replicate this model for different kinds of authentication (token based, session based etc).
RequestHandler._execute is an internal method and this will break in Tornado 4.0. prepare() and the HTTP verb methods get()/post()/etc are the supported methods to be overridden in subclasses. I don't see any reason that this code would return an empty response, although the server logs (which you did not post) might have something useful.
This is a working version of the above function. Just for anyone else who's having the same trouble -
def token_authenticate():
"""
This is a basic authentication interceptor which
protects the desired URIs and requires
authentication as per configuration
"""
def wrapper(self, transforms, *args, **kwargs):
request = self.request
try:
token = request.arguments.get('token')[0]
if not token:
return False
user_token = get_user(token)
if user_token:
self.token = user_token
self.user = user_token.user
return True
except:
pass
return False
return wrapper
def interceptor(func):
"""
This is a class decorator which is helpful in configuring
one or more interceptors which are able to intercept, inspect,
process and approve or reject further processing of the request
"""
def classwrapper(cls):
def wrapper(old):
def inner(self, transforms, *args, **kwargs):
ret = func(self, transforms, *args, **kwargs)
if ret:
return old(self, transforms, *args, **kwargs)
else:
self._transforms = transforms
return self._unauthorized()
return inner
cls._execute = wrapper(cls._execute)
return cls
return classwrapper
Related
I have a few conditions to implement for rotating proxies in scrapy middleware:
If response is not 200 try that request with another random proxy from a list.
I have two lists of proxies let's say I'd like to start crawling with first list of proxies and retry about 10 times with that list and after that as a last resort I want to try second proxy list.
I have tried creating the middleware but it is not working as expected it is not rotating proxies as well as not picking up the second proxy list as last resort. Here is the code:
class SFAProxyMiddleware(object):
#classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings)
def __init__(self, settings):
self.packetstream_proxies = [
settings.get("PS_PROXY_USA"),
settings.get("PS_PROXY_CA"),
settings.get("PS_PROXY_IT"),
settings.get("PS_PROXY_GLOBAL"),
]
self.unlimited_proxies = [
settings.get("UNLIMITED_PROXY_1"),
settings.get("UNLIMITED_PROXY_2"),
settings.get("UNLIMITED_PROXY_3"),
settings.get("UNLIMITED_PROXY_4"),
settings.get("UNLIMITED_PROXY_5"),
settings.get("UNLIMITED_PROXY_6"),
]
def add_proxy(self, request, host):
request.meta["proxy"] = host
def process_request(self, request, spider):
retries = request.meta.get("retry_times", 0)
if "proxy" in request.meta.keys():
return None
if retries <= 10:
self.add_proxy(request, random.choice(self.unlimited_proxies))
else:
self.add_proxy(request, random.choice(self.packetstream_proxies))
Am I doing something wrong implementing the middleware? Thanks
I think based on the conditions at the beginning of your question, that you also need to process the response to check for it's status code and if it isn't a 200 then to increase the retry count and send it make to the scheduler.
You might need to set the dont_filter parameter in the request to True, and you should also probably set a maximum for the number of retries.
for example
from scrapy.exceptions import IgnoreRequest
MAX_RETRY = 20
class SFAProxyMiddleware(object):
#classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings)
def __init__(self, settings):
self.packetstream_proxies = [
settings.get("PS_PROXY_USA"),
settings.get("PS_PROXY_CA"),
settings.get("PS_PROXY_IT"),
settings.get("PS_PROXY_GLOBAL"),
]
self.unlimited_proxies = [
settings.get("UNLIMITED_PROXY_1"),
settings.get("UNLIMITED_PROXY_2"),
settings.get("UNLIMITED_PROXY_3"),
settings.get("UNLIMITED_PROXY_4"),
settings.get("UNLIMITED_PROXY_5"),
settings.get("UNLIMITED_PROXY_6"),
]
def add_proxy(self, request, host):
request.meta["proxy"] = host
def process_request(self, request, spider):
retries = request.meta.get("retry_times", 0)
if "proxy" in request.meta.keys():
return None
if retries <= 10:
self.add_proxy(request, random.choice(self.unlimited_proxies))
else:
self.add_proxy(request, random.choice(self.packetstream_proxies))
def process_response(self, response, spider):
if response.status_code != 200:
request = response.request
request.meta.setdefault("retry_times", 1)
request.meta["retry_times"] += 1
if request.meta["retry_times"] > MAX_RETRY:
raise IgnoreRequest
request.dont_filter = True
return request
return response
I have implemented the following retry decorator.
def retry(delay=10, retries=4):
def retry_decorator(f):
#wraps(f)
def f_retry(*args, **kwargs):
while retries > 1:
try:
return f(*args, **kwargs)
except Exception as e:
msg = "Exception: {}, Retrying in {} seconds...'.format(e, delay)"
print(msg)
time.sleep(delay)
retries -= 1
return f(*args, **kwargs)
return f_retry
return retry_decorator
I get the error that retries is not defined. However, retries is mentioned in the function definition. I am unable to figure out what went wrong here. Any help will be appreciated.
I made it work by collecting the variables retry and delay in a dictionary and then using that inside the function.
def retry(delay=10, retries=4):
def retry_decorator(f):
#wraps(f)
def f_retry(*args, **kwargs):
opt_dict = {'retries': retries, 'delay': delay}
while opt_dict['retries'] > 1:
try:
return f(*args, **kwargs)
except Exception as e:
msg = "Exception: {}, Retrying in {} seconds...".format(e, delay)
print(msg)
time.sleep(opt_dict['delay'])
opt_dict['retries'] -= 1
return f(*args, **kwargs)
return f_retry
return retry_decorator
I wrote a method in a class that tries to read some data from a database. I've also decorated this method with a decorator which checks if the connection to the database was open, and in case it was not, restarts it.
class HiveConnection(object):
def __init__(self, host, user, password):
"""Instantiate a HiveConnector object."""
self.host = host
self.user = user
self.password = password
self.port = 10000
self.auth_mechanism = 'PLAIN'
self._connection = self._connect()
def _connect(self):
"""Start the connection to database."""
try:
return connect(host=self.host, port=self.port,
user=self.user, password=self.password,
auth_mechanism=self.auth_mechanism)
except TTransportException as error:
print('Failed attempt to connect')
self._connect()
def _disconnect(self):
"""Close connection to database."""
self._connection.close()
def hadoop_connection_handler(function):
"""Start a database connection if not already open."""
#wraps(function)
def wrapper(inst, *args, **kwargs):
if not inst._connection:
inst._connect()
return function(inst, *args, **kwargs)
return wrapper
#hadoop_connection_handler
def read(self, query):
"""Execute a query to pull the data.
Args:
query: [str] Query to pull the data.
Returns:
A list of namedtuple (`Row`).
"""
columns = self._columns(query)
cursor = self._connection.cursor()
cursor.execute(query)
Record = namedtuple("Record", columns)
data = map(Record._make, cursor.fetchall())
cursor.close()
return data
Now I want to write a unit test to make sure this actually works.
from unittest.mock import patch, MagicMock
from nose.tools import assert_equal, raises
from services.db_connections import HiveConnection
class TestHiveConnection:
"""Integration test suite for HiveConnection class."""
def setUp(self):
self.hive = HiveConnection(user='username', password='password', host='myhost.net')
def test_reconnect(self):
"""If the connection drops, the object should be able to establish a
new connection.
"""
query = 'SELECT * FROM database.table 1'
self.hive._connect = MagicMock()
self.hive._disconnect()
self.hive.read(query)
assert_equal(self.hive._connect.called, True)
The above test always fails. self.hive._connect.called equals in fact to False. This I think it's because the connect() method is called within the decorator. How should I change the test to account for that?
I've been writing a MUD in python, using asyncio.Protocol, however I have a problem with when users close their client (typically a terminal, since you connect via telnet) without disconnecting properly.
The server doesn't recognize the user as disconnected, and they remain in the game.
The problem only occurs when the client is connected remotely (for some reason, maybe someone can explain...) it doesn't happen when connecting from the localhost.
Is there a neat way to check a user is still actually connected (without additional software client-side), or on failing that how do I incorporate a timeout?
My Protocol looks something like this currently:
class User(Protocol):
def connection_made(self, transport):
self.transport = transport
self.addr = transport.get_extra_info('peername')
self.authd = False
self.name = None
self.admin = False
self.room = None
self.table = None
self.db = None
self.flags = []
print("Connected: {}".format(self.addr))
server.connected.append(self)
actions['help'](self, ['welcome'])
self.get_prompt()
def data_received(self, data):
msg = data.decode().strip()
args = msg.split()
if self.authd is False:
actions['login'](self, args)
return
if msg:
if args[0] in self.db.aliases:
args[0] = str(self.db.aliases[args[0]])
msg = ' '.join(args)
args = msg.split()
if msg[0] in server.channels:
ch = db.session.query(db.models.Channel).get(msg[0])
if msg[1] =='#':
channels.send_to_channel(self, ch, msg[2:], do_emote=True)
else:
channels.send_to_channel(self, ch, msg[1:])
self.get_prompt()
return
if args[0] in actions:
if self.is_frozen():
self.send_to_self("You're frozen solid!")
else:
actions[args[0]](self, args[1:] if len(args) > 1 else None)
self.get_prompt()
return
self.send_to_self("Huh?")
else:
if self.table is not None:
actions['table'](self, None)
elif self.room is not None:
actions['look'](self, None)
def send_to_self(self, msg):
msg = "\r\n" + msg
msg = colourify(msg)
self.transport.write(msg.encode())
#staticmethod
def send_to_user(user, msg):
msg = "\r\n"+msg
msg = colourify(msg)
user.transport.write(msg.encode())
#staticmethod
def send_to_users(users, msg):
msg = "\r\n"+msg
msg = colourify(msg)
for user in users:
user.transport.write(msg.encode())
def connection_lost(self, ex):
print("Disconnected: {}".format(self.addr))
server.connected.remove(self)
if self.authd:
self.save()
server.users.remove(self)
self.room.occupants.remove(self)
Note: I've chopped a lot of superfluous stuff out. If you want the full code, it's here.
You may schedule a new timeout handler on every data_received() call (with cancelling previous timeout handler, sure). I found the approach too cumbersome.
Or, as an option, switch to asyncio streams -- you may use asyncio.wait_for or brand new not released yet asyncio.timeout.
I have a multithreading client-server software and it's a bit hard to debug when 10 client is connected. I have a few questions regarding the tracing.
How can I trace a only a certain file or class in the software?
Is there a possibility to have a log with a timestamp and the function name, without any other information and separated by threads?
Is there a graphical trace generator?
import logging
import time
import pymongo
import hashlib
import random
DEBUG_MODE = True
class logger(object):
def __new__(cls, *args, **kwargs):
if DEBUG_MODE:
return object.__new__(cls, *args, **kwargs)
else:
return args[0]
def __init__(self, foo):
self.foo = foo
logging.basicConfig(filename='exceptions.log', format='%(levelname)s %(asctime)s: %(message)s')
self.log = logging.getLogger(__name__)
def __call__(self, *args, **kwargs):
def _log():
try:
t = time.time()
func_hash = self._make_hash(t)
col = self._make_db_connection()
log_record = {'func_name':self.foo.__name__, 'start_time':t, 'func_hash':func_hash}
col.insert(log_record)
res = self.foo(*args, **kwargs)
log_record = {'func_name':self.foo.__name__, 'exc_time':round(time.time() - t,4), 'end_time':time.time(),'func_hash':func_hash}
col.insert(log_record)
return res
except Exception as e:
self.log.error(e)
return _log()
def _make_db_connection(self):
connection = pymongo.Connection()
db = connection.logger
collection = db.log
return collection
def _make_hash(self, t):
m = hashlib.md5()
m.update(str(t)+str(random.randrange(1,10)))
return m.hexdigest()
Its use mongo as storage but you can write any backend. Just wrap function which you need and follow the log.