I'm using python suds library to make a SOAP client based on a local wsdl file. My goal is to use Twisted as the backend so I query the SOAP servers in a asyncronous way.
I know this topic has been covered different times (here1, here2), but I still have some questions.
I've seen three different approaches to use twisted with suds:
a) Applying this patch to the suds library.
b) Use twisted-suds, which is a fork of suds.
c) Influenced by this post, I implemented Client_Async suds client using the twisted deferToThread operation, (fully working gist can be found here. I also implemented a Client_Sync suds client as well to do some benchmarks)
# Init approach c) code
from suds.client import Client as SudsClient
from twisted.internet.threads import deferToThread
class MyClient(SudsClient):
def handleFailure(self, f, key, stats):
stats.stop_stamp(error=True)
logging.error("%s. Failure: %s" % (key, str(f)))
def handleResult(self, result, key, stats):
stats.stop_stamp(error=False)
success, text, res = False, None, None
try:
success = result.MessageResult.MessageResultCode == 200
text = result.MessageResult.MessageResultText
res = result.FooBar
except Exception, err:
pass
logging.debug('%40s : %5s %10s \"%40s\"' % (key, success, text, res))
logging.debug('%40s : %s' % (key, self.last_sent()))
logging.debug('%40s : %s' % (key, self.last_received()))
def call(stats, method, service, key, *a, **kw):
stats.start_stamp()
logging.debug('%40s : calling!' % (key))
result = service.__getattr__(method)(*a, **kw)
return result
class Client_Async(MyClient):
""" Twisted based async client"""
def callRemote(self, stats, method, key, *args, **kwargs):
logging.debug('%s. deferring to thread...' % key)
d = deferToThread(call, stats, method, self.service, key, *args, **kwargs)
d.addCallback(self.handleResult, key, stats)
d.addErrback(self.handleFailure, key, stats)
return d
class Client_Sync(MyClient):
def callRemote(self, stats, method, key, *args, **kwargs):
result = None
try:
result = call(stats, method, self.service, key, *args, **kwargs)
except Exception, err:
self.handleFailure(err, key, stats)
else:
self.handleResult(result, key, stats)
# End approach c) code
Doing a small benchmark using the c) approach points the benefits of the Async model:
-- Sync model using Client_Sync of approach c).
# python soap_suds_client.py -t 200 --sync
Total requests:800/800. Success:794 Errors:6
Seconds elapsed:482.0
Threads used:1
-- Async model using Client_Async of approach c).
# python soap_suds_client.py -t 200
Total requests:800/800. Success:790 Errors:10
Seconds elapsed:53.0
Threads used:11
I haven't tested approaches a) or b), my question is:
What am I really gaining from them apart from the use of just one thread?
I'm using suds in my projects. I didn't have to do any patches, or use twisted-suds.
I'm using the 0.4.1-2 version of python-suds package (on ubuntu) and it comes with very usefull nosend option.
# This parses the wsdl file. The autoblend option you'd probably skip,
# its needed when name spaces are not strictly preserved (case for Echo Sign).
from suds import client
self._suds = client.Client('file://' + config.wsdl_path, nosend=True,
autoblend=True)
....
# Create a context for the call, example sendDocument() call. This doesn't yet
# send anything, only creates an object with the request and capable of parsing
# the response
context = self._suds.service.sendDocument(apiKey=....)
# Actually send the request. Use any web client you want. I actually use
# something more sophisticated, but below I put the example using
# standard twisted web client.
from twisted.web import client
d = client.getPage(url=context.client.location(),
postdata=str(context.envelope),
method='POST',
headers=context.client.headers())
# The callback() of the above Deferred is fired with the body of the
# http response. I parse it using the context object.
d.addCallback(context.succeeded)
# Now in the callback you have the actual python object defined in
# your WSDL file. You can print...
from pprint import pprint
d.addCallback(pprint)
# I the response is a failure, your Deferred would be errbacked with
# the suds.WebFault exception.
Related
hello i guess have problem with client and member config which config should i use as you can see i am inserting json as data when i call get_data it returns with no problem but when i try to use predicate-sql it gives me error "hazelcast.errors.HazelcastSerializationError: Exception from server: com.hazelcast.nio.serialization.HazelcastSerializationException: There is no suitable de-serializer for type -120. This exception is likely caused by differences in t
he serialization configuration between members or between clients and members."
#app.route('/insert_data/<database_name>/<collection_name>', methods=['POST'])
def insert_data(database_name, collection_name):
client = hazelcast.HazelcastClient(cluster_members=[
url
])
dbname_map = client.get_map(f"{database_name}-{collection_name}").blocking()
if request.json:
received_json_data = request.json
received_id = received_json_data["_id"]
del received_json_data["_id"]
dbname_map.put(received_id, received_json_data)
client.shutdown()
return jsonify()
else:
client.shutdown()
abort(400)
#app.route('/get_data/<database_name>/<collection_name>', methods=['GET'])
def get_all_data(database_name, collection_name):
client = hazelcast.HazelcastClient(cluster_members=[
url
])
dbname_map = client.get_map(f"{database_name}-{collection_name}").blocking()
entry_set = dbname_map.entry_set()
output = dict()
datas = []
for key, value in entry_set:
value['_id'] = key
output = value
datas.append(output)
client.shutdown()
return jsonify({"Result":datas})
#bp.route('/get_query/<database_name>/<collection_name>/<name>', methods=['GET'])
def get_query_result(database_name, collection_name,name):
client = hazelcast.HazelcastClient(cluster_members=[
url
])
predicate_map = client.get_map(f"{database_name}-{collection_name}").blocking()
predicate = and_(sql(f"name like {name}%"))
entry_set = predicate_map.values(predicate)
#entry_set = predicate_map.entry_set(predicate)
send_all_data = ""
for x in entry_set:
send_all_data += x.to_string()
send_all_data += "\n"
print(send_all_data)
# print("Retrieved %s values whose age is less than 30." % len(result))
# print("Entry is", result[0].to_string())
# value=predicate_map.get(70)
# print(value)
return jsonify()
i try to change hazelcast.xml according to hazelcast-full-example.xml but i can't start hazelcast after
the changes and do i really have to use serialization ? hazelcast version:4.1 python:3.9
This is most likely happening because you are putting entries of the type dictionary to the map, which is serialized by the pickle because you didn't specify a serializer for that and the client does not know how to handle that correctly, so it fallbacks to the default serializer. However, since pickle serialization is Python-specific, servers cannot deserialize it and throw such an exception.
There are possible solutions to that, see the https://hazelcast.readthedocs.io/en/stable/serialization.html chapter for details.
I think the most appropriate solution for your use case would be Portable serialization which does not require a configuration change or code on the server-side. See the https://hazelcast.readthedocs.io/en/stable/serialization.html#portable-serialization
BTW, client objects are quite heavyweight, so you shouldn't be creating them on demand like this. You can construct it once in your application and share and use it in your endpoints or business-logic code freely since it is thread-safe. The same applies to the map proxy you get from the client. It can also be re-used.
Following advice from Jean-Paul Calderone here on SO, I'm trying to modify the twisted "starttls_server" sample below to support the use of ssl.ClientCertificateOptions, to allow me to specify my private key, certificate, and trusted roots, as per http://twistedmatrix.com/documents/14.0.0/api/twisted.internet.ssl.CertificateOptions.html
from twisted.internet import ssl, protocol, defer, task, endpoints
from twisted.protocols.basic import LineReceiver
from twisted.python.modules import getModule
class TLSServer(LineReceiver):
def lineReceived(self, line):
print("received: " + line)
if line == "STARTTLS":
print("-- Switching to TLS")
self.sendLine('READY')
self.transport.startTLS(self.factory.options)
def main(reactor):
certData = getModule(__name__).filePath.sibling('server.pem').getContent()
cert = ssl.PrivateCertificate.loadPEM(certData)
factory = protocol.Factory.forProtocol(TLSServer)
factory.options = cert.options()
endpoint = endpoints.TCP4ServerEndpoint(reactor, 8000)
endpoint.listen(factory)
return defer.Deferred()
if __name__ == '__main__':
import starttls_server
task.react(starttls_server.main)
My understanding is that I effectively need to replace the cert = ssl.PrivateCertificate... and cert.options = ssl.PrivateCertificate.... lines with something like certopts = ssl.CertificateOptions(privateKey=pKeyData, certificate=certData, trustRoot=caCertsData) (having read the appropriate files in to certData, caCertsData, and pKeyData) and then pass this in to factory.options - but without pasting every variant of code I've tried, I've yet to work this out correctly - my efforts have produced varying results from the classic "OpenSSL.crypto.Error: []" - through to seemingly just dumping the contents of my 3 PEM files to screen and exiting!
Can anyone enlighten me? Thank you :)
cert.options() is already returning a CertificateOptions. The problem is that options takes authorities (as Certificate objects) as positional args, and doesn't let you pass through all the other configuration values through, so you probably want to construct a CertificateOptions directly.
Just change the factory.options = cert.options() line to factory.options = ssl.CertificateOptions(...).
However, CertificateOptions takes a pyOpenSSL PKey object as its privateKey, not the key data. So you'll need to use OpenSSL APIs to load that key, or you can extract it from a PrivateCertificate.
If you read the signature of CertificateOptions very carefully, the required types should be fairly clear. You may also need to consult the pyOpenSSL documentation as well.
[EDIT: I'm running Python 2.7.3]
I'm a network engineer by trade, and I've been hacking on ncclient (the version on the website is old, and this was the version I've been working off of) to make it work with Brocade's implementation of NETCONF. There are some tweaks that I had to make in order to get it to work with our Brocade equipment, but I had to fork off the package and make tweaks to the source itself. This didn't feel "clean" to me so I decided I wanted to try to do it "the right way" and override a couple of things that exist in the package*; three things specifically:
A "static method" called build() which belongs to the HelloHandler class, which itself is a subclass of SessionListener
The "._id" attribute of the RPC class (the original implementation used uuid, and Brocade boxes didn't like this very much, so in my original tweaks I just changed this to a static value that never changed).
A small tweak to a util function that builds XML filter attributes
So far I have this code in a file brcd_ncclient.py:
#!/usr/bin/env python
# hack on XML element creation and create a subclass to override HelloHandler's
# build() method to format the XML in a way that the brocades actually like
from ncclient.xml_ import *
from ncclient.transport.session import HelloHandler
from ncclient.operations.rpc import RPC, RaiseMode
from ncclient.operations import util
# register brocade namespace and create functions to create proper xml for
# hello/capabilities exchange
BROCADE_1_0 = "http://brocade.com/ns/netconf/config/netiron-config/"
register_namespace('brcd', BROCADE_1_0)
brocade_new_ele = lambda tag, ns, attrs={}, **extra: ET.Element(qualify(tag, ns), attrs, **extra)
brocade_sub_ele = lambda parent, tag, ns, attrs={}, **extra: ET.SubElement(parent, qualify(tag, ns), attrs, **extra)
# subclass RPC to override self._id to change uuid-generated message-id's;
# Brocades seem to not be able to handle the really long id's
class BrcdRPC(RPC):
def __init__(self, session, async=False, timeout=30, raise_mode=RaiseMode.NONE):
self._id = "1"
return super(BrcdRPC, self).self._id
class BrcdHelloHandler(HelloHandler):
def __init__(self):
return super(BrcdHelloHandler, self).__init__()
#staticmethod
def build(capabilities):
hello = brocade_new_ele("hello", None, {'xmlns':"urn:ietf:params:xml:ns:netconf:base:1.0"})
caps = brocade_sub_ele(hello, "capabilities", None)
def fun(uri): brocade_sub_ele(caps, "capability", None).text = uri
map(fun, capabilities)
return to_xml(hello)
#return super(BrcdHelloHandler, self).build() ???
# since there's no classes I'm assuming I can just override the function itself
# in ncclient.operations.util?
def build_filter(spec, capcheck=None):
type = None
if isinstance(spec, tuple):
type, criteria = spec
# brocades want the netconf prefix on subtree filter attribute
rep = new_ele("filter", {'nc:type':type})
if type == "xpath":
rep.attrib["select"] = criteria
elif type == "subtree":
rep.append(to_ele(criteria))
else:
raise OperationError("Invalid filter type")
else:
rep = validated_element(spec, ("filter", qualify("filter")),
attrs=("type",))
# TODO set type var here, check if select attr present in case of xpath..
if type == "xpath" and capcheck is not None:
capcheck(":xpath")
return rep
And then in my file netconftest.py I have:
#!/usr/bin/env python
from ncclient import manager
from brcd_ncclient import *
manager.logging.basicConfig(filename='ncclient.log', level=manager.logging.DEBUG)
# brocade server capabilities advertising as 1.1 compliant when they're really not
# this will stop ncclient from attempting 1.1 chunked netconf message transactions
manager.CAPABILITIES = ['urn:ietf:params:netconf:capability:writeable-running:1.0', 'urn:ietf:params:netconf:base:1.0']
# BROCADE_1_0 is the namespace defined for netiron configs in brcd_ncclient
# this maps to the 'brcd' prefix used in xml elements, ie subtree filter criteria
with manager.connect(host='hostname_or_ip', username='username', password='password') as m:
# 'get' request with no filter - for brocades just shows 'show version' data
c = m.get()
print c
# 'get-config' request with 'mpls-config' filter - if no filter is
# supplied with 'get-config', brocade returns nothing
netironcfg = brocade_new_ele('netiron-config', BROCADE_1_0)
mplsconfig = brocade_sub_ele(netironcfg, 'mpls-config', BROCADE_1_0)
filterstr = to_xml(netironcfg)
c2 = m.get_config(source='running', filter=('subtree', filterstr))
print c2
# so far it only looks like the supported filters for 'get-config'
# operations are: 'interface-config', 'vlan-config' and 'mpls-config'
Whenever I run my netconftest.py file, I get timeout errors because in the log file ncclient.log I can see that my subclass definitions (namely the one that changes the XML for hello exchange - the staticmethod build) are being ignored and the Brocade box doesn't know how to interpret the XML that the original ncclient HelloHandler.build() method is generating**. I can also see in the generated logfile that the other things I'm trying to override are also being ignored, like the message-id (static value of 1) as well as the XML filters.
So, I'm kind of at a loss here. I did find this blog post/module from my research, and it would appear to do exactly what I want, but I'd really like to be able to understand what I'm doing wrong via doing it by hand, rather than using a module that someone has already written as an excuse to not have to figure this out on my own.
*Can someone explain to me if this is "monkey patching" and is actually bad? I've seen in my research that monkey patching is not desirable, but this answer and this answer are confusing me quite a bit. To me, my desire to override these bits would prevent me from having to maintain an entire fork of my own ncclient.
**To give a little more context, this XML, which ncclient.transport.session.HelloHandler.build() generates by default, the Brocade box doesn't seem to like:
<?xml version='1.0' encoding='UTF-8'?>
<nc:hello xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0">
<nc:capabilities>
<nc:capability>urn:ietf:params:netconf:base:1.0</nc:capability>
<nc:capability>urn:ietf:params:netconf:capability:writeable-running:1.0</nc:capability>
</nc:capabilities>
</nc:hello>
The purpose of my overridden build() method is to turn the above XML into this (which the Brocade does like:
<?xml version="1.0" encoding="UTF-8"?>
<hello xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
<capabilities>
<capability>urn:ietf:params:netconf:base:1.0</capability>
<capability>urn:ietf:params:netconf:capability:writeable-running:1.0</capability>
</capabilities>
</hello>
So it turns out that the "meta info" should not have been so hastily removed, because again, it's difficult to find answers to what I'm after when I don't fully understand what I want to ask. What I really wanted to do was override stuff in a package at runtime.
Here's what I've changed brcd_ncclient.py to (comments removed for brevity):
#!/usr/bin/env python
from ncclient import manager
from ncclient.xml_ import *
brcd_new_ele = lambda tag, ns, attrs={}, **extra: ET.Element(qualify(tag, ns), attrs, **extra)
brcd_sub_ele = lambda parent, tag, ns, attrs={}, **extra: ET.SubElement(parent, qualify(tag, ns), attrs, **extra)
BROCADE_1_0 = "http://brocade.com/ns/netconf/config/netiron-config/"
register_namespace('brcd', BROCADE_1_0)
#staticmethod
def brcd_build(capabilities):
hello = brcd_new_ele("hello", None, {'xmlns':"urn:ietf:params:xml:ns:netconf:base:1.0"})
caps = brcd_sub_ele(hello, "capabilities", None)
def fun(uri): brcd_sub_ele(caps, "capability", None).text = uri
map(fun, capabilities)
return to_xml(hello)
def brcd_build_filter(spec, capcheck=None):
type = None
if isinstance(spec, tuple):
type, criteria = spec
# brocades want the netconf prefix on subtree filter attribute
rep = new_ele("filter", {'nc:type':type})
if type == "xpath":
rep.attrib["select"] = criteria
elif type == "subtree":
rep.append(to_ele(criteria))
else:
raise OperationError("Invalid filter type")
else:
rep = validated_element(spec, ("filter", qualify("filter")),
attrs=("type",))
if type == "xpath" and capcheck is not None:
capcheck(":xpath")
return rep
manager.transport.session.HelloHandler.build = brcd_build
manager.operations.util.build_filter = brcd_build_filter
And then in netconftest.py:
#!/usr/bin/env python
from brcd_ncclient import *
manager.logging.basicConfig(filename='ncclient.log', level=manager.logging.DEBUG)
manager.CAPABILITIES = ['urn:ietf:params:netconf:capability:writeable-running:1.0', 'urn:ietf:params:netconf:base:1.0']
with manager.connect(host='host', username='user', password='password') as m:
netironcfg = brcd_new_ele('netiron-config', BROCADE_1_0)
mplsconfig = brcd_sub_ele(netironcfg, 'mpls-config', BROCADE_1_0)
filterstr = to_xml(netironcfg)
c2 = m.get_config(source='running', filter=('subtree', filterstr))
print c2
This gets me almost to where I want to be. I still have to edit the original source code to change the message-id's from being generated with uuid1().urn because I haven't figured out or don't understand how to change an object's attributes before __init__ happens at runtime (chicken/egg problem?); here's the offending code in ncclient/operations/rpc.py:
class RPC(object):
DEPENDS = []
REPLY_CLS = RPCReply
def __init__(self, session, async=False, timeout=30, raise_mode=RaiseMode.NONE):
self._session = session
try:
for cap in self.DEPENDS:
self._assert(cap)
except AttributeError:
pass
self._async = async
self._timeout = timeout
self._raise_mode = raise_mode
self._id = uuid1().urn # Keeps things simple instead of having a class attr with running ID that has to be locked
Credit goes to this recipe on ActiveState for finally cluing me in on what I really wanted to do. The code I had originally posted I don't think was technically incorrect - if what I wanted to do was fork off my own ncclient and make changes to it and/or maintain it, which wasn't what I wanted to do at all, at least not right now.
I'll edit my question title to better reflect what I had originally wanted - if other folks have better or cleaner ideas, I'm totally open.
Is there an easy way to cache things when using urllib2 that I am over-looking, or do I have to roll my own?
If you don't mind working at a slightly lower level, httplib2 (https://github.com/httplib2/httplib2) is an excellent HTTP library that includes caching functionality.
You could use a decorator function such as:
class cache(object):
def __init__(self, fun):
self.fun = fun
self.cache = {}
def __call__(self, *args, **kwargs):
key = str(args) + str(kwargs)
try:
return self.cache[key]
except KeyError:
self.cache[key] = rval = self.fun(*args, **kwargs)
return rval
except TypeError: # incase key isn't a valid key - don't cache
return self.fun(*args, **kwargs)
and define a function along the lines of:
#cache
def get_url_src(url):
return urllib.urlopen(url).read()
This is assuming you're not paying attention to HTTP Cache Controls, but just want to cache the page for the duration of the application
This ActiveState Python recipe might be helpful:
http://code.activestate.com/recipes/491261/
I've always been torn between using httplib2, which does a solid job of handling HTTP caching and authentication, and urllib2, which is in the stdlib, has an extensible interface, and supports HTTP Proxy servers.
The ActiveState recipe starts to add caching support to urllib2, but only in a very primitive fashion. It fails to allow for extensibility in storage mechanisms, hard-coding the file-system-backed storage. It also does not honor HTTP cache headers.
In an attempt to bring together the best features of httplib2 caching and urllib2 extensibility, I've adapted the ActiveState recipe to implement most of the same caching functionality as is found in httplib2. The module is in jaraco.net as jaraco.net.http.caching. The link points to the module as it exists at the time of this writing. While that module is currently part of the larger jaraco.net package, it has no intra-package dependencies, so feel free to pull the module out and use it in your own projects.
Alternatively, if you have Python 2.6 or later, you can easy_install jaraco.net>=1.3 and then utilize the CachingHandler with something like the code in caching.quick_test().
"""Quick test/example of CacheHandler"""
import logging
import urllib2
from httplib2 import FileCache
from jaraco.net.http.caching import CacheHandler
logging.basicConfig(level=logging.DEBUG)
store = FileCache(".cache")
opener = urllib2.build_opener(CacheHandler(store))
urllib2.install_opener(opener)
response = opener.open("http://www.google.com/")
print response.headers
print "Response:", response.read()[:100], '...\n'
response.reload(store)
print response.headers
print "After reload:", response.read()[:100], '...\n'
Note that jaraco.util.http.caching does not provide a specification for the backing store for the cache, but instead follows the interface used by httplib2. For this reason, the httplib2.FileCache can be used directly with urllib2 and the CacheHandler. Also, other backing caches designed for httplib2 should be usable by the CacheHandler.
I was looking for something similar, and came across "Recipe 491261: Caching and throttling for urllib2" which danivo posted. The problem is I really dislike the caching code (lots of duplication, lots of manually joining of file paths instead of using os.path.join, uses staticmethods, non very PEP8'sih, and other things that I try to avoid)
The code is a bit nicer (in my opinion anyway) and is functionally much the same, with a few additions - mainly the "recache" method (example usage can be seem here, or in the if __name__ == "__main__": section at the end of the code).
The latest version can be found at http://github.com/dbr/tvdb_api/blob/master/cache.py, and I'll paste it here for posterity (with my application specific headers removed):
#!/usr/bin/env python
"""
urllib2 caching handler
Modified from http://code.activestate.com/recipes/491261/ by dbr
"""
import os
import time
import httplib
import urllib2
import StringIO
from hashlib import md5
def calculate_cache_path(cache_location, url):
"""Checks if [cache_location]/[hash_of_url].headers and .body exist
"""
thumb = md5(url).hexdigest()
header = os.path.join(cache_location, thumb + ".headers")
body = os.path.join(cache_location, thumb + ".body")
return header, body
def check_cache_time(path, max_age):
"""Checks if a file has been created/modified in the [last max_age] seconds.
False means the file is too old (or doesn't exist), True means it is
up-to-date and valid"""
if not os.path.isfile(path):
return False
cache_modified_time = os.stat(path).st_mtime
time_now = time.time()
if cache_modified_time < time_now - max_age:
# Cache is old
return False
else:
return True
def exists_in_cache(cache_location, url, max_age):
"""Returns if header AND body cache file exist (and are up-to-date)"""
hpath, bpath = calculate_cache_path(cache_location, url)
if os.path.exists(hpath) and os.path.exists(bpath):
return(
check_cache_time(hpath, max_age)
and check_cache_time(bpath, max_age)
)
else:
# File does not exist
return False
def store_in_cache(cache_location, url, response):
"""Tries to store response in cache."""
hpath, bpath = calculate_cache_path(cache_location, url)
try:
outf = open(hpath, "w")
headers = str(response.info())
outf.write(headers)
outf.close()
outf = open(bpath, "w")
outf.write(response.read())
outf.close()
except IOError:
return True
else:
return False
class CacheHandler(urllib2.BaseHandler):
"""Stores responses in a persistant on-disk cache.
If a subsequent GET request is made for the same URL, the stored
response is returned, saving time, resources and bandwidth
"""
def __init__(self, cache_location, max_age = 21600):
"""The location of the cache directory"""
self.max_age = max_age
self.cache_location = cache_location
if not os.path.exists(self.cache_location):
os.mkdir(self.cache_location)
def default_open(self, request):
"""Handles GET requests, if the response is cached it returns it
"""
if request.get_method() is not "GET":
return None # let the next handler try to handle the request
if exists_in_cache(
self.cache_location, request.get_full_url(), self.max_age
):
return CachedResponse(
self.cache_location,
request.get_full_url(),
set_cache_header = True
)
else:
return None
def http_response(self, request, response):
"""Gets a HTTP response, if it was a GET request and the status code
starts with 2 (200 OK etc) it caches it and returns a CachedResponse
"""
if (request.get_method() == "GET"
and str(response.code).startswith("2")
):
if 'x-local-cache' not in response.info():
# Response is not cached
set_cache_header = store_in_cache(
self.cache_location,
request.get_full_url(),
response
)
else:
set_cache_header = True
#end if x-cache in response
return CachedResponse(
self.cache_location,
request.get_full_url(),
set_cache_header = set_cache_header
)
else:
return response
class CachedResponse(StringIO.StringIO):
"""An urllib2.response-like object for cached responses.
To determine if a response is cached or coming directly from
the network, check the x-local-cache header rather than the object type.
"""
def __init__(self, cache_location, url, set_cache_header=True):
self.cache_location = cache_location
hpath, bpath = calculate_cache_path(cache_location, url)
StringIO.StringIO.__init__(self, file(bpath).read())
self.url = url
self.code = 200
self.msg = "OK"
headerbuf = file(hpath).read()
if set_cache_header:
headerbuf += "x-local-cache: %s\r\n" % (bpath)
self.headers = httplib.HTTPMessage(StringIO.StringIO(headerbuf))
def info(self):
"""Returns headers
"""
return self.headers
def geturl(self):
"""Returns original URL
"""
return self.url
def recache(self):
new_request = urllib2.urlopen(self.url)
set_cache_header = store_in_cache(
self.cache_location,
new_request.url,
new_request
)
CachedResponse.__init__(self, self.cache_location, self.url, True)
if __name__ == "__main__":
def main():
"""Quick test/example of CacheHandler"""
opener = urllib2.build_opener(CacheHandler("/tmp/"))
response = opener.open("http://google.com")
print response.headers
print "Response:", response.read()
response.recache()
print response.headers
print "After recache:", response.read()
main()
This article on Yahoo Developer Network - http://developer.yahoo.com/python/python-caching.html - describes how to cache http calls made through urllib to either memory or disk.
#dbr: you may need to add also https responses caching with :
def https_response(self, request, response):
return self.http_response(request,response)
In simpy, I'm using the store as my resource due to the nature of the problem.
I have a few get request for a store item. However, some get request have higher priority and I wish for it to be processed first. I do not wish to follow the FIFO rules for such special get request.
yield Store_item.get()
I tried following this question. However, I'm unable to create a subclass that suits this requirement.
I want something like this:(but this is an example of priority resource and not store resource).
def resource_user(name, env, resource, wait, prio):
yield env.timeout(wait)
with resource.request(priority=prio) as req:
print('%s requesting at %s with priority=%s'% (name,env.now,prio))
yield req
print('%s got resource at %s' % (name, env.now))
yield env.timeout(3)
However, I need it for store resource class and not a generic get for store.
The outcome will be:
yield Store_item.priority_get()
I realize I'm late, but this is what worked for me.
First, define a PriorityGet class (this code was adapted from the source of simpy):
class PriorityGet(simpy.resources.base.Get):
def __init__(self, resource, priority=10, preempt=True):
self.priority = priority
"""The priority of this request. A smaller number means higher
priority."""
self.preempt = preempt
"""Indicates whether the request should preempt a resource user or not
(:class:`PriorityResource` ignores this flag)."""
self.time = resource._env.now
"""The time at which the request was made."""
self.usage_since = None
"""The time at which the request succeeded."""
self.key = (self.priority, self.time, not self.preempt)
"""Key for sorting events. Consists of the priority (lower value is
more important), the time at which the request was made (earlier
requests are more important) and finally the preemption flag (preempt
requests are more important)."""
super().__init__(resource)
Then, assemble your PriorityStore resource:
from simpy.core import BoundClass
class PriorityBaseStore(simpy.resources.store.Store):
GetQueue = simpy.resources.resource.SortedQueue
get = BoundClass(PriorityGet)
No priority_get method is bound to the class, but you can achieve the same result with a .get(priority = 1) (or any other number lower than 10, the base priority defined in the PriorityGet class). Alternatively, you can bound the method explicitly.