from textblob import TextBlob
line = """ अशा किंमतीवर खरोखरच चांगला कुकर आहे ok are you sure how are you आधुनिक तंत्रज्ञानासह हे सुलभ आणि सुरक्षित are you आहे இது அற்புதமான தரம் மற்றும் சூப்பர் தயாரிப்பு."""
def split_line(in_line):
line_sp = line.split(" ")
line_two = [" ".join(line_sp[i:i + 3]) for i in range(0, len(line_sp), 3)]
return line_two
#print(split_line(line))
try:
for i in split_line(line):
blob = TextBlob(i)
print (blob.translate(to = 'en'))
except:
print ("same language found not translated")
This is language Translation code some times text blob throw error so I used try/except block so my code stops when print error message but I want to continue this loop after catching exception
Just put the try except block inside the for (if the error is raised by TextBlob(i)):
for i in split_line(line):
try:
blob = TextBlob(i)
print (blob.translate(to = 'en'))
except:
print ("same language found not translated")
This will make the for run until it ends even if an error has been raised in it.
is there any solution in python which lets a function execute after the previous one was finished?
Here is one of the ideas I'm using now. But it is not solving the problem when files are larger and the program needs more time.
def copy_to_jumphost(self):
try:
if self.connect():
stdin, stdout, stderr = self.client.exec_command('sshpass -p %s scp -r %s#%s:%s/' % (self.password,self.username,self.hostname,self.log_path) + self.lista[self.file_number].rstrip() + ' ' + '/home/%s/' % (self.username) + self.lista[self.file_number].rstrip())
except (AttributeError, TypeError) as e:
print("Error occurred:", e)
try:
if self.connect():
if self.copy_to_jumphost():
ftp_client = self.client.open_sftp()
ftp_client.get(filepath, self.localpath)
print("Success! \nFile coppied to %s" %(self.localpath))
else:
time.sleep(5)
ftp_client = self.client.open_sftp()
ftp_client.get(filepath, self.localpath)
print("Success but needed some time! \nFile coppied to %s" %(self.localpath))
except (AttributeError, TypeError) as e:
print("Error occurred:", e)
Perfect situation for me will be if in else statement there is a solution to wait for finishing the copy_to_jumphost() function, because time.sleep(5) will fail if I will need to copy larger files.
I'm using multiprocessing module for files processing in parallel, which works perfectly fine almost every time.
Also I've written that in try , except block to catch any exception.
I've come across a situation where except block doesn't catch the exception.
Since the code is huge I'm just putting relevant block which is giving problem.
def reader(que, ip, start, end, filename):
""" Reader function checks each line of the file
and if the line contains any of the ip addresses which are
being scanned, then it writes to its buffer.
If the line field doesn't match date string it skips the line.
"""
logging.info("Processing : %s" % os.path.basename(filename))
ip_pat = re.compile("(\d+\.\d+\.\d+\.\d+\:\d+)")
chunk = 10000000 # taking chunk of 10MB data
buff = ""
with bz2.BZ2File(filename,"rb", chunk) as fh: # open the compressed file
for line in fh:
output = []
fields = line.split()
try:
ts = fields[1].strip() + "/" +fields[0]+"/"+fields[3].split("-")[0]+" "+fields[2]
times = da.datetime.strptime(ts,"%d/%b/%Y %H:%M:%S")
if times < start:
continue
if times > end:
break
ips = re.findall(ip_pat,line)
if len(ips) < 3:
continue
if ips[0].split(":")[0] == ip:
output.append(times.strftime("%d/%m/%Y %H:%M:%S"))
status = "SESSION_OPEN" if "SESSION_OPEN" in line or "CREATE" in line else "SESSION_CLOSE"
protocol = "TCP" if "TCP" in line else "UDP"
output.append(status)
output.append(protocol)
ips[1], ips[2] = ips[2], ips[1]
output.extend(ips)
res = "|".join(output)
buff += res + "\n"
except IndexError, ValueError:
continue
logging.info("Processed : %s of size [ %d ]" % (os.path.basename(filename), os.path.getsize(filename)))
if buff:
que.put((ip,buff))
return buff
And this is what is received as error.
File "/usr/lib64/python2.7/multiprocessing/pool.py", line 554, in get
raise self._value
ValueError: time data '2/Dec/20 10:59:59' does not match format '%d/%b/%Y %H:%M:%S'
What I don't understand is why the exception is not caught, I've mentioned ValueError in except block.
What's the best way to get through this problem.
Provide the multiple exceptions as a tuple:
except (IndexError, ValueError):
continue
The relevant doc is https://docs.python.org/2/tutorial/errors.html#handling-exceptions
Snippet from the page:
Note that the parentheses around this tuple are required, because except ValueError, e: was the syntax used for what is normally written as except ValueError as e: in modern Python (described below). The old syntax is still supported for backwards compatibility. This means except RuntimeError, TypeError is not equivalent to except (RuntimeError, TypeError): but to except RuntimeError as TypeError: which is not what you want.
I am trying to download map image in python with urllib module. But it always failed.
I'm tried to use urllib.urlopen() with some parameter variants
tried in urllib.urlretrieve()
But it doesn't work. And, When I see the source code of image url, I didn't find image file. Here is image: https://maps.googleapis.com/maps/api/staticmap?center=31.0456,121.3997&zoom=12&size=320x385&sensor=false
Source code:
#-------------------------- PARSE IP ADDRESS -------------------------------
import re
import urllib
try:
mysite = urllib.urlopen('http://ip-api.com/line')
except urllib.HTTPError, e:
print "Cannot retrieve URL: HTTP Error Code", e.code
except urllib.URLError, e:
print "Cannot retrieve URL: " + e.reason[1]
list_of_params = mysite.read()
print list_of_params
ip_arr = list_of_params.splitlines()
#--------------------- HERE IS FIND MAP IMAGE --------------------------------------
try:
map_page = urllib.urlopen('http://ip-api.com')
except urllib.HTTPError, e:
print "Cannot retrieve URL: HTTP Error Code", e.code
except urllib.URLError, e:
print "Cannot retrieve URL: " + e.reason[1]
#f = open("data.html", "w")
#f.write(str(mysite.read()))
#f.close()
#looking for this in page
pattern = re.findall(re.compile("url\(\'(https://maps\.googleapis\.com/maps/api/staticmap\?center=.*)\'"), page_get_map.read())
map_img_url = pattern[0].replace('&', '&')
#------------------- DOWNLOAD MAP IMAGE And SAVE IT ------------------------
#file_name = map_img_url.rsplit('/',1)[1]
try:
get_map_img = urllib.urlretrieve(map_img_url, "staticmap.png")
except urllib.HTTPError, e:
print "Cannot retrieve URL: HTTP Error Code", e.code
except urllib.URLError, e:
print "Cannot retrieve URL: " + e.reason[1]
i = open("pict.png", "w")
i.write(get_map_img.read())
i.close()
print "End of file"
import requests
f=open('static.png','wb')
f.write(requests.get('https://maps.googleapis.com/maps/api/staticmap?center=31.0456,121.3997&zoom=12&size=320x385&sensor=false').content)
f.close()
Why are you parsing the map URL? Construct it yourself:
import json, urllib
query = '' # IP to get coordinates of, leave empty for current IP
geo = urllib.urlopen('http://ip-api.com/json/%s?fields=240' % query)
result = json.load(geo)
if result['zip']:
zoom = 13
elif result['city']:
zoom = 12
else:
zoom = 6
map_img_url = "https://maps.googleapis.com/maps/api/staticmap?center=%s,%s&zoom=%i&size=320x385&sensor=false" % (result['lat'], result['lon'], zoom)
get_map_img = urllib.urlretrieve(map_img_url, "staticmap.png")
My facebook app lets the user upload an image, select what the image is (eyes, nose, mouth or other body part) and then can combine by selecting three random images by category and that's works fine and the code looks ok and readable though not very advanced:
class CyberFazeHandler(BaseHandler):
def get_random_image(self, category):
fileinfos = FileInfo.all().filter("category =", category)
return fileinfos[random.randint(0, fileinfos.count()-1)]
def get(self):
eyes_image = self.get_random_image(category="eyes")
nose_image = self.get_random_image(category="nose")
mouth_image = self.get_random_image(category="mouth")
eyes_data = None
try:
eyes_data = blobstore.fetch_data(eyes_image.blob.key(), 0, 50000)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find eyes data for file '+str(eyes_image.key().id())+' (' + unicode(e) + u')')
eyes_img = None
try:
eyes_img = images.Image(image_data=eyes_data)
...now I just fetch 3 random images and then combine then in the template:
<img src="{{eyes_url}}"><br>
<img src="{{nose_url}}"><br>
<img src="{{mouth_url}}">
Could this be improved by sending a composite image combining the three images into one? With the advantage that everything on the image will load at the same time and that it will be saved already if a randomization comes up next time the result is already saved. What do you think?
Thank you (the application is apps.facebook.com/cyberfaze you may inspect I did for fun and learning)
The entire class is
class CyberFazeHandler(BaseHandler):
def get_random_image(self, category):
fileinfos = FileInfo.all().filter("category =", category)
return fileinfos[random.randint(0, fileinfos.count()-1)] #optimize
def get(self):
eyes_image = self.get_random_image(category="eyes")
nose_image = self.get_random_image(category="nose")
mouth_image = self.get_random_image(category="mouth")
eyes_data = None
try:
eyes_data = blobstore.fetch_data(eyes_image.blob.key(), 0, 50000)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find eyes data for file '+str(eyes_image.key().id())+' (' + unicode(e) + u')')
eyes_img = None
try:
eyes_img = images.Image(image_data=eyes_data)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find eyes img for file '+str(eyes_image.key().id())+' (' + unicode(e) + u')')
nose_data = None
try:
nose_data = blobstore.fetch_data(nose_image.blob.key(), 0, 50000)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find nose data for file '+str(nose_image.key().id())+' (' + unicode(e) + u')')
nose_img = None
try:
nose_img = images.Image(image_data=nose_data)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find nose img for file '+str(nose_image.key().id())+' (' + unicode(e) + u')')
mouth_data = None
try:
mouth_data = blobstore.fetch_data(mouth_image.blob.key(), 0, 50000)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find mouth data for file '+str(eyes_image.key().id())+' (' + unicode(e) + u')')
mouth_img = None
try:
mouth_img = images.Image(image_data=mouth_data)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find mouth img for file '+str(mouth_image.key().id())+' (' + unicode(e) + u')')
minimum = min(int(eyes_img.width), int(nose_img.width), int(mouth_img.width))
eyes_url = images.get_serving_url(str(eyes_image.blob.key()), size=minimum)
nose_url = images.get_serving_url(str(nose_image.blob.key()), size=minimum)
mouth_url = images.get_serving_url(str(mouth_image.blob.key()), size=minimum)
self.render(u'cyberfaze', minimum=minimum, eyes_image=eyes_image, eyes_url=eyes_url, nose_image=nose_image, nose_url=nose_url, mouth_image=mouth_image, mouth_url=mouth_url, form_url = blobstore.create_upload_url('/upload'),)
After rewrite it's working like said:
class CyberFazeHandler(BaseHandler):
def get_random_image(self, category):
q = FileInfo.all()
q.filter('category =', category)
q.filter('randomvalue >=', random.random())
return q.get()
def get_random_image_legacy(self, category):
fileinfos = FileInfo.all().filter('category =', category)
return fileinfos[random.randint(0, fileinfos.count() - 1)]
def get(self):
eyes_image = self.get_random_image(category='eyes')
if not eyes_image:
logging.debug("getting eyes failed, trying legacy method")
eyes_image = self.get_random_image_legacy(category='eyes')
nose_image = self.get_random_image(category='nose')
if not nose_image:
nose_image = self.get_random_image_legacy(category='nose')
mouth_image = self.get_random_image(category='mouth')
if not mouth_image:
mouth_image = self.get_random_image_legacy(category='mouth')
eyes_data = None
try:
eyes_data = blobstore.fetch_data(eyes_image.blob.key(), 0,
50000)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find eyes data for file '
+ str(eyes_image.key().id()) + ' ('
+ unicode(e) + u')')
eyes_img = None
try:
eyes_img = images.Image(image_data=eyes_data)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find eyes img for file '
+ str(eyes_image.key().id()) + ' ('
+ unicode(e) + u')')
nose_data = None
try:
nose_data = blobstore.fetch_data(nose_image.blob.key(), 0,
50000)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find nose data for file '
+ str(nose_image.key().id()) + ' ('
+ unicode(e) + u')')
nose_img = None
try:
nose_img = images.Image(image_data=nose_data)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find nose img for file '
+ str(nose_image.key().id()) + ' ('
+ unicode(e) + u')')
mouth_data = None
try:
mouth_data = blobstore.fetch_data(mouth_image.blob.key(),
0, 50000)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find mouth data for file '
+ str(eyes_image.key().id()) + ' ('
+ unicode(e) + u')')
mouth_img = None
try:
mouth_img = images.Image(image_data=mouth_data)
except Exception, e:
self.set_message(type=u'error',
content=u'Could not find mouth img for file '
+ str(mouth_image.key().id()) + ' ('
+ unicode(e) + u')')
minimum = min(int(eyes_img.width), int(nose_img.width),
int(mouth_img.width))
eyes_url = images.get_serving_url(str(eyes_image.blob.key()),
size=minimum)
nose_url = images.get_serving_url(str(nose_image.blob.key()),
size=minimum)
mouth_url = images.get_serving_url(str(mouth_image.blob.key()),
size=minimum)
self.render(
u'cyberfaze',
minimum=minimum,
eyes_image=eyes_image,
eyes_url=eyes_url,
nose_image=nose_image,
nose_url=nose_url,
mouth_image=mouth_image,
mouth_url=mouth_url,
form_url=blobstore.create_upload_url('/upload'),
)
Which is more efficient depends on how it'll be used. If the user will be loading a lot of these mashups, it makes more sense to send them as separate images, because there will be fewer images for the browser to cache (a+b+c images instead of a*b*c).
Your code has a much more egregious performance issue, however:
def get_random_image(self, category):
fileinfos = FileInfo.all().filter("category =", category)
return fileinfos[random.randint(0, fileinfos.count()-1)]
Every time you call this function, it will perform a count operation, which is O(n) with the number of FileInfo entities, then perform an offset query, which is O(n) with the offset. This is extremely slow and inefficient, and will get more so as you increase the number of images.
If you expect the set of images to be small (less than a few thousand) and fairly constant, simply store them in code, which will be faster than any other option. If the set is larger, or changes at runtime, assign a random value between 0 and 1 to each entity, and use a query like this to retrieve a randomly selected one:
q = FileInfo.all()
q.filter('category =', category)
q.filter('random >=', random.random())
return q.get()