This has been driving me crazy and I can't figure out where the issue is. AWS is having issues right now, but this problem was happening before all that and continues to exhibit same behavior.
Summary: I have an admin field that uploads images to S3 and stores the path in the database. On a fresh apache restart, it works fine for about 20 minutes. Then it stops working and just hangs on 'waiting for response from domain' That's it, no error message, just sits there, waiting, and won't work again until I restart apache. And after restarting apache it works flawlessly until it stops, which happens after 20 minutes best I can tell and is not cause by more or less traffic or a certain number of inserts, deletes, etc.
Here's code for the model:
class SampleImage(models.Model):
def __unicode__(self):
return self.name
name = models.CharField(max_length=50)
front_image = S3EnabledImageField(upload_to='samples')
back_image = S3EnabledImageField(upload_to='samples')
Code for S3EnabledImageField:
class S3EnabledImageField(models.ImageField):
def generate_filename(self, instance, filename):
path_join = os.path.join(self.get_directory_name(), self.get_filename(filename))
return path_join.replace("\\", "/")
def __init__(self, bucket=settings.BUCKET_NAME, verbose_name=None, name=None, width_field=None, height_field=None, **kwargs):
if settings.USE_AMAZON_S3:
self.connection = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
if not self.connection.lookup(bucket):
self.connection.create_bucket(bucket)
self.bucket = self.connection.get_bucket(bucket)
kwargs['storage'] = S3Storage(self.bucket)
super(S3EnabledImageField, self).__init__(verbose_name, name, width_field, height_field, **kwargs)
Code for S3Storage:
class S3Storage(FileSystemStorage):
def __init__(self, bucket=None, location=None, base_url=None):
if location is None:
location = settings.MEDIA_ROOT
if base_url is None:
base_url = settings.MEDIA_URL
self.location = os.path.abspath(location)
self.bucket = bucket
self.base_url = base_url
def _open(self, name, mode='rb'):
class S3File(File):
def __init__(self, key):
self.key = key
def size(self):
return self.key.size
def read(self, *args, **kwargs):
return self.key.read(*args, **kwargs)
def write(self, content):
self.key.set_contents_from_string(content)
def close(self):
self.key.close()
return S3File(Key(self.bucket, name))
def _save(self, name, content):
key = Key(self.bucket, name)
if hasattr(content, 'temporary_file_path'):
key.set_contents_from_filename(content.temporary_file_path())
elif isinstance(content, File):
key.set_contents_from_file(content)
else:
key.set_contents_from_string(content)
key.make_public()
return name
def delete(self, name):
self.bucket.delete_key(name)
def exists(self, name):
return Key(self.bucket, name).exists()
def listdir(self, path):
return [key.name for key in self.bucket.list()]
def path(self, name):
raise NotImplementedError
def size(self, name):
return self.bucket.get_key(name).size
def url(self, name):
return Key(self.bucket, name).generate_url(100000)
def get_available_name(self, name):
return name
Not sure by looking at your code. But, sounds like there might be connections that aren't getting closed? At any rate, why not work with:
https://bitbucket.org/david/django-storages/wiki/S3Storage
?
django-storages has worked well for me.
Related
I try to build a decorator for methods of instances (not classes) that flexibly puts code blocks in front and/or behind the method (and not affect other instances). Up to my code below works:
def Decorate(func, before = None, after = None):
def wrap(*args, **kwargs):
if before: before() # code block insert
result = func(*args, **kwargs)
if after: after() # code block insert
return result
return wrap
class Test():
def __init__(self, name):
self.name = name
def put(self, prefix):
print(prefix, self.name)
a = Test('me')
def Before():
print('before')
def After():
print('after')
a.put = Decorate(a.put, Before, After)
a.put('it is')
How can I extend the code blocks accessing/using variables and methods of the instance? A code example for this would look like this:
def Before():
print('before')
print(self.name)
self.any_method(any_argument) # just an example!
I already tried several things without success. And I already struggle to access the instance values directly in the wrapper:
def Decorate(func, before = None, after = None):
def wrap(self, *args, **kwargs):
if before: before() # code block insert
print(self.name) # --> even this DOES NOT WORK!
result = func(self, *args, **kwargs)
if after: after() # code block insert
return result
return wrap
Here print(self.name) throws an error: AttributeError: 'str' object has no attribute 'name'. So it looks like that I am far away in using the same comment in one of the code blocks (Before() & After()) below.
One addition: The approach works when I add a method to the instance:
This method is in the class (so for working with strings and exec, but that enables to deliver the name as string or the function itself):
def addMethod(self, method, givenName = ''):
print('add')
if givenName == '':
N = method.__name__
else:
N = givenName
self._methods.append(N)
exec('self.' + N + ' = ' + method.__name__ + '.__get__(self)')
The code in the main part looks like this:
def x(self):
print('hello,', self.name)
a.addMethod(x)
a.x()
Any solution is appreciated and many thanks in advance!
from functools import wraps
def Decorate(func, before = None, after = None):
#wraps(func)
def wrap(*args, **kwargs):
if before: before() # code block insert
result = func(*args, **kwargs)
if after: after() # code block insert
return result
return wrap
def Before():
print('before')
def After():
print('after')
class Test():
def __init__(self, name):
self.name = name
def put(self, prefix):
print(prefix, self.name)
put = Decorate(put, Before, After)
a = Test('me')
a.put("pre")
You could execute your decorator inside your class. In your wrap, pass whatever you get to func by (*args, **kwargs). self is still the first argument implicitly in args.
Edit: Code related concerns from comments
from functools import wraps
def Before(t):
print('before')
print(t.name)
def After(t):
print('after')
print(t.name)
def Decorate(func, before = None, after = None):
#wraps(func)
def wrap(*args, **kwargs):
if before: before(args[0]) # code block insert
result = func(*args, **kwargs)
if after: after(args[0]) # code block insert
return result
return wrap
class Test():
def __init__(self, name):
self.name = name
def put(self, prefix):
print(prefix, self.name)
put = Decorate(put, Before, After)
a = Test('me')
a.put("pre")
I am working on a web-scraping project (monster.com). Everything is working, but I wanted to display the the relevant info on the same line (heading, company, location, date), but I can't I tried end='', but it doesn't work
from locators.jobs_locators import JobsLocators
class JobsParser:
def __init__(self, parent):
self.parent = parent
def __repr__(self):
return f"{self.heading}, at {self.company}, located in {self.location}, posted {self.date}"
#property
def heading(self):
return self.parent.select_one(JobsLocators.HEADING).string
#property
def company(self):
try:
return self.parent.select_one(JobsLocators.COMPANY).string
except:
return self.parent.select_one(JobsLocators.COMPANY1).string
#property
def location(self):
try:
return self.parent.select_one(JobsLocators.LOCATION).string
except:
return self.parent.select_one(JobsLocators.COMPANY1).string
#property
def date(self):
return self.parent.select_one(JobsLocators.DATE).string
There is my code I'm trying to do:
import json
import requests
class Request(object):
def __init__(self, method, path):
self.method = method
## Q1
self.url = ''.join([***MyClass.host***, path])
self.h = {
"Content-Type": "application/json;charset=UTF-8"
}
def __call__(self, f):
def wrapper(*args, **kwargs):
# Q2
getattr(requests, self.method)(url = self.url, headers = h***, data = json.dumps(body)***)
return wrapper
class MyClass(object):
def __init__(self, host):
self.host = host
#Request(method = "post", path = "/add_mission")
def AddMission(self, mission):
body = {
"mission_id": mission
}
#Request(method = "get", path = "/system_info")
def GetInfo(self):
print("I want get info")
There are some questions hope someone can solve my problems:
1. How can my decorator "Request" get the variable "host" from "MyClass"? (In comment ## Q1)
2. How can I pass the function's variable "body" to decorator, is it possible? (In comment ## Q2)
Because I need to access different url(host+path) with [post, get, delete].
I'm not sure isn't decorator match on this case?
Or maybe there are better ways to deal with this case?
The host is available once you call the decorated function, via the invoking object's host attribute, so wait until you actually call the method to build the URL. You'll want to call the underlying function to get back a value, if any, to use as the payload.
import json
import requests
class Request(object):
def __init__(self, method, path):
self.method = method.upper()
self.h = {
"Content-Type": "application/json;charset=UTF-8"
}
self.path = path
def __call__(self, f):
def wrapper(obj, *args, **kwargs):
payload = f(*args, **kwargs)
args = {
'method': self.method,
'url': 'http://{}/{}'.format(obj.host, self.path),
'headers': self.h
}
if payload is not None:
args['json'] = payload
return requests.request(**args)
return wrapper
class MyClass(object):
def __init__(self, host):
self.host = host
#Request(method="post", path="/add_mission")
def AddMission(self, mission):
return {"mission_id": mission}
#Request(method="get", path="/system_info")
def GetInfo(self):
print("I want get info")
Then you can write, for example,
api = MyClass("www.example.com")
add_response = api.AddMission("mars")
info_response = api.GetInfo()
Each decorated function returns the Response object produced by the call to requests.request.
You might consider subclasses of Request for each method:
class GetRequest(Request):
def __init__(self, path):
super().__init__("GET", path)
class PostRequest(Request):
def __init__(self, path):
super().__init__("POST", path)
# etc
The code now has all types of requests covered, plus the possibility of having authentication on each request.
Code on github
Pros:
Easy to read, less code, therefore easier to maintain.
Easy to add new methods or to update the existing ones.
Cons:
Can't get it to work on pytest.
Less "standard" API might cause hard to debug errors.
Less "standard" API might need changes to the wrapper that can be tricky.
Big performance loss because of all the nested function calls (methods/decorators are functions)
import ujson
import requests
class Request(object):
def __init__(self, method, path, h={ "Content-Type": "application/json;charset=UTF-8" }):
self.path = path
#print(f"PATH {path}")
self.method = method.upper()
self.h = h
def __call__(self, f):
def wrapper(obj, *args, **kwargs):
payload = f(obj, *args, **kwargs)
#print(f"PAYLOAD {payload}")
if (payload is not None) and (self.method == "GET" or self.method == "DELETE"):
dic_key = list(payload.keys())[0]
self.path = "{}/?{}={}".format(self.path, dic_key, payload[dic_key])
payload = None
args = {
'method': self.method,
'url': 'http://{}/{}'.format(obj.host, self.path),
'headers': self.h
}
if payload is not None:
args['json'] = payload
return requests.request(**args)
return wrapper
class GetRequest(Request):
def __init__(self, path, h=None):
if h is None:
super().__init__("GET", path)
else:
super().__init__("GET", path, h)
class PostRequest(Request):
def __init__(self, path, h=None):
if h is None:
super().__init__("POST", path)
else:
super().__init__("POST", path, h)
class PutRequest(Request):
def __init__(self, path, h=None):
if h is None:
super().__init__("PUT", path)
else:
super().__init__("PUT", path, h)
class DeleteRequest(Request):
def __init__(self, path, h=None):
if h is None:
super().__init__("DELETE", path)
else:
super().__init__("DELETE", path, h)
myToken = 'Secret_key_1234567890'
class DecoRequests(object):
def __init__(self, host):
self.host = host
#GetRequest(path="userlist", h={'access_token':myToken})
def list_employees(self):
pass
#GetRequest(path="users", h={'access_token':myToken})
def list_employee(self, username):
return {"username":username}
#PostRequest(path="users", h={'access_token':myToken})
def add_employee(self, employee_data):
return employee_data
#PutRequest(path="users", h={'access_token':myToken})
def update_employee(self, employee_data):
return employee_data
#DeleteRequest(path="users", h={'access_token':myToken})
def rm_employee(self, username):
return {"username":username}
I have a class Record :
class Record(object):
def __init__(self, id=None, url=None, time= None, user=None, input_dict=None):
if input_dict:
self.from_dict(input_dict)
else:
self.id = id
self.url = url
self.time = time
self.user = user
def from_dict_internal(self, input_dict):
#creates a record using data from a dictionary
self.id = input_dict.get('id')
self.url = input_dict.get('url')
self.time = input_dict.get('time')
self.user = input_dict.get('user')
def from_dict(self, input_dict):
return self.from_dict_internal()
I want to make a derived class Page with 1 extra attribute - referrer. I am not sure how to go about it as I have tried this:
class Page(Record):
def __init__(self):
Record.__init__(self)
self.referrer = referrer
def from_dict(self, input_dict):
self.from_dict_internal()
self.referrer = input_dict.get('referrer')
However Pycharm keeps underlining the referrer in the constructor of page with a reason of unresolved reference
You are getting an error because referrer is not declared anywhere or passed into the init method. Also you should pass in the same data that your Record() class uses.
def __init__(self, id=None, url=None, time= None, user=None, input_dict=None, referrer=None):
Record.__init__(self, id, url, time, user, input_dict)
self.referrer = referrer
....
In my project I'm using code from http://djangosnippets.org/snippets/562/ and I'm encountering some problems.
When I use my own ChainableQuerySetManager and try to create an object through a related object I get a recursion error.
ChainableQuerySetManager source:
class ChainableQuerySetManager(models.Manager):
def __init__(self, qs_class=models.query.QuerySet):
super(ChainableQuerySetManager,self).__init__()
self.queryset_class = qs_class
def get_query_set(self):
return self.queryset_class(self.model)
def __getattr__(self, attr, *args):
try:
return getattr(self.__class__, attr, *args)
except AttributeError:
return getattr(self.get_query_set(), attr, *args)
Extended query set:
class ExtendedQuerySet(models.query.QuerySet):
def get_or_None(self, *args, **kwargs):
result = None
try:
result = self.get(*args, **kwargs)
except ObjectDoesNotExist:
pass
return result
And test models:
class ParentObject(models.Model):
value = models.IntegerField(default=0)
def count_someobjects(self):
return self.someobjects.count()
def create_someobject_throw_related(self):
return self.someobjects.create()
def create_someobject(self):
return SomeObject.objects.create(parent=self)
class SomeObject(models.Model):
parent = models.ForeignKey(ParentObject, related_name='someobjects')
value = models.IntegerField(default=1)
objects = ChainableQuerySetManager(ExtendedQuerySet)
Test Case looks like:
class ExtendedQuerySetTests(TestCase):
def setUp(self):
self.parent = ParentObject.objects.create()
def test_create_someobject_in_parent(self):
someobject = self.parent.create_someobject_throw_related()
I would appreciate your help.
Full source can be found at https://github.com/RANUX/django-simptools
I recently had a similar issue. Try replacing self.__class__ with ChainableQuerySetManager in your query manager. I never quite sorted out exactly why this was the issue, but it did solve things for me.