We Keep Coding

How to query arXiv for a specific year?

I'm using the code shown below in order to retrieve papers from arXiv. I want to retrieve papers that have words "machine" and "learning" in the title. The number of papers is large, therefore I want to implement a slicing by year (published).
How can I request records of 2020 and 2019 in search_query? Please notice that I'm not interested in post-filtering.
import urllib.request
import time
import feedparser
# Base api query url
base_url = 'http://export.arxiv.org/api/query?';
# Search parameters
search_query = urllib.parse.quote("ti:machine learning")
start = 0
total_results = 5000
results_per_iteration = 1000
wait_time = 3
papers = []
print('Searching arXiv for %s' % search_query)
for i in range(start,total_results,results_per_iteration):
print("Results %i - %i" % (i,i+results_per_iteration))
query = 'search_query=%s&start=%i&max_results=%i' % (search_query,
i,
results_per_iteration)
# perform a GET request using the base_url and query
response = urllib.request.urlopen(base_url+query).read()
# parse the response using feedparser
feed = feedparser.parse(response)
# Run through each entry, and print out information
for entry in feed.entries:
#print('arxiv-id: %s' % entry.id.split('/abs/')[-1])
#print('Title: %s' % entry.title)
#feedparser v4.1 only grabs the first author
#print('First Author: %s' % entry.author)
paper = {}
paper["date"] = entry.published
paper["title"] = entry.title
paper["first_author"] = entry.author
paper["summary"] = entry.summary
papers.append(paper)
# Sleep a bit before calling the API again
print('Bulk: %i' % 1)
time.sleep(wait_time)

According to the arXiv documentation, there is no published or date field available.
What you can do is to sort the results by date (by adding &sortBy=submittedDate&sortOrder=descending to your query parameters) and stop making requests when you reach 2018.
Basically your code should be modified like this:
import urllib.request
import time
import feedparser
# Base api query url
base_url = 'http://export.arxiv.org/api/query?';
# Search parameters
search_query = urllib.parse.quote("ti:machine learning")
i = 0
results_per_iteration = 1000
wait_time = 3
papers = []
year = ""
print('Searching arXiv for %s' % search_query)
while (year != "2018"): #stop requesting when papers date reach 2018
print("Results %i - %i" % (i,i+results_per_iteration))
query = 'search_query=%s&start=%i&max_results=%i&sortBy=submittedDate&sortOrder=descending' % (search_query,
i,
results_per_iteration)
# perform a GET request using the base_url and query
response = urllib.request.urlopen(base_url+query).read()
# parse the response using feedparser
feed = feedparser.parse(response)
# Run through each entry, and print out information
for entry in feed.entries:
#print('arxiv-id: %s' % entry.id.split('/abs/')[-1])
#print('Title: %s' % entry.title)
#feedparser v4.1 only grabs the first author
#print('First Author: %s' % entry.author)
paper = {}
paper["date"] = entry.published
year = paper["date"][0:4]
paper["title"] = entry.title
paper["first_author"] = entry.author
paper["summary"] = entry.summary
papers.append(paper)
# Sleep a bit before calling the API again
print('Bulk: %i' % 1)
i += results_per_iteration
time.sleep(wait_time)
for the "post-filtering" approach, once enough results are collected, I'd do something like this:
papers2019 = [item for item in papers if item["date"][0:4] == "2019"]

Evernote Python API - Hitting rate limits

I've written a short piece of code that will append the tag names of my notes to the title, then remove all associated tags. When I try to run this on production, I hit the rate limit real quickly. Can someone help me optimise this piece of code? Or should I request for a special rate limit with Evernote?
Also, I get an error when a note has no tags. Any way to efficiently get the number of tags from a note so I don't get the error?
from evernote.api.client import EvernoteClient
from evernote.edam.notestore import NoteStore
dev_token = "dev_token"
client = EvernoteClient(token=dev_token, sandbox = False)
userStore = client.get_user_store()
user = userStore.getUser()
print
print user.username
print
noteStore = client.get_note_store()
notebooks = noteStore.listNotebooks()
for n in notebooks:
print "Notebook = " + n.name + " GUID = " + n.guid
filter = NoteStore.NoteFilter()
filter.ascending = False
filter.notebookGuid=n.guid
spec = NoteStore.NotesMetadataResultSpec()
spec.includeTitle = True
spec.includeNotebookGuid = True
spec.includeTagGuids = True
notesMetadataList = noteStore.findNotesMetadata(filter, 0, 25, spec)
for noteMetadata in notesMetadataList.notes:
print "%s :: %s" % (noteMetadata.title, noteMetadata.guid)
newNoteTitle = noteMetadata.title + " -- "
for tagGuid in noteMetadata.tagGuids:
tag = noteStore.getTag(tagGuid)
tagName = tag.name
print tagName
newNoteTitle = newNoteTitle + " " + tagName
print "newNoteTitle = " + newNoteTitle
noteMetadata.title = newNoteTitle
noteMetadata.tagGuids = []
noteMetadata = noteStore.updateNote(noteMetadata)
print noteMetadata.title

Here's how I deal with rate limiting, by wrapping the EvernoteClient in a rate limiting proxy (based on http://code.activestate.com/recipes/496741-object-proxying/)
from time import sleep
from evernote.api.client import EvernoteClient
from evernote.edam.error.ttypes import (EDAMSystemException, EDAMErrorCode)
def evernote_wait_try_again(f):
"""
Wait until mandated wait and try again
http://dev.evernote.com/doc/articles/rate_limits.php
"""
def f2(*args, **kwargs):
try:
return f(*args, **kwargs)
except EDAMSystemException as e:
if e.errorCode == EDAMErrorCode.RATE_LIMIT_REACHED:
print("rate limit: {0} s. wait".format(e.rateLimitDuration))
sleep(e.rateLimitDuration)
print("wait over")
return f(*args, **kwargs)
return f2
class RateLimitingEvernoteProxy(object):
__slots__ = ["_obj"]
def __init__(self, obj):
object.__setattr__(self, "_obj", obj)
def __getattribute__(self, name):
return evernote_wait_try_again(
getattr(object.__getattribute__(self, "_obj"), name))
_client = EvernoteClient(token=auth_token, sandbox=sandbox)
client = RateLimitingEvernoteProxy(_client)

Flask Python: Unable to get parameters using request.args

I want to request some parameters from a external web app. I create an API with flask and query data from MySQL. I able to query the data if I gave a fix input but not when using request.args. I try both request.args['name'] and request.args.get('name') but return the output of Exception path.
Below is my current code. I comment out the fix input I used.
from flask import Flask,jsonify,abort,make_response,request,render_template
import MySQLdb
import MySQLdb.cursors
#app.route('/KLSE/search', methods=['GET'])
def KLSEsearch():
db = MySQLdb.connect(host='vinus.mysql.pythonanywhere-services.com',user='vinus',passwd='Vindex2016',db='vinus$default',cursorclass=MySQLdb.cursors.DictCursor)
curs = db.cursor()
#name ='P'
#macd = 'H'
#volumeMin = '_'
#volumeMax = '_'
#stoch ='H1'
#bollinger ='H'
#rsi ='H1'
#atr ='LV'
#trade = 'HOLD'
#limit = 3
#offSet = 1
name = request.args.get('name')
volumeMin = request.args['volumeMin']
volumeMax = request.args['volumeMax']
macd = request.args['macd']
stoch = request.args['stoch']
bollinger = request.args['bollinger']
rsi = request.args['rsi']
atr = request.args['atr']
trade = request.args['trade']
limit = request.args['limit']
offSet = request.args['offSet']
query0 = "SELECT * FROM KLSE WHERE Stock LIKE '%s' AND"
#query1 = "(Vindex BETWEEN (IF(%s='_',-5000,%s)) AND (IF(%s='_',5000,%s))) AND "
query2 = "(Volume_changes_pc BETWEEN (IF (%s='_',-5000,%s)) AND (IF(%s='_',5000,%s))) AND "
query3 = "MACD LIKE %s AND "
query4 = "STOCH LIKE %s AND "
query5 = "BOLLINGER LIKE %s AND "
query6 = "RSI LIKE %s AND "
query7 = "ATR LIKE %s AND "
query8 = "TRADE LIKE %s LIMIT %s OFFSET %s"
query = query0+query2+query3+query4+query5+query6+query7+query8
input = name+"%",volumeMin,volumeMin,volumeMax,volumeMax,macd,stoch,bollinger,rsi,atr,trade,limit,offSet
try:
curs.execute(query,(input))
g = curs.fetchall()
except Exception:
return 'Error: unable to fetch items'
#return "hihi"
return jsonify({'Stock': g})
The output with fix value as below. I think it shows the query to MySQL is correct.
http://vinus.pythonanywhere.com/KLSE/search1
For the user input value, which use the args,
http://vinus.pythonanywhere.com/KLSE/search?atr=%&bollinger=%&macd=%&name=M&rsi=%&stoch=%&volumeMax=&volumeMin=&trade=HOLD&limit=5&offSet=1
What is the right way, get the parameters? volumeMin,volumeMax,limit and offSet are in float and integers.

You have to serialize your data first.
def serialize():
return {
"id" : g.id,
"volumeMin" : g.name,
"volumeMax" : g.address,
"macd" : g.city,
"stoch" : g.state,
"zipCode" : g.zipCode,
"bollinger" : g.bollinger,
}
#app.route("/KLSE/search/.json")
def stock_json():
query = your.db.query()
return jsonify(Stock=[i.serialize for i in query])

How to check if an Image has geo-info on flickr - python

Here is my code so far:
import flickrapi
import osmapi
import geopy
from geopy.geocoders import Nominatim
import overpy
import requests
import xmltodict
from time import sleep
api_key = "xxxxxxxxxxxxxxxxxxxx"
secret_api_key = "xxxxxxx"
flickr = flickrapi.FlickrAPI(api_key, secret_api_key)
def obtainImages3():
group_list = flickr.groups.search (api_key=api_key, text = 'Paris', per_page = 10)
for group in group_list[0]:
group_images = flickr.groups.pools.getPhotos (api_key=api_key, group_id = group.attrib['nsid'], extras = 'geo, tags, url_s')
for image in group_images[0]:
url = str(image.attrib['url_s'])
tags = (image.attrib['tags']).encode('utf-8')
#ONLY RUN THIS CODE IF IMAGE HAS GEO INFO
photo_location = flickr.photos_geo_getLocation(photo_id=image.attrib['id'])
lat = float(photo_location[0][0].attrib['latitude'])
lon = float(photo_location[0][0].attrib['longitude'])
geolocator = Nominatim()
location = geolocator.reverse("{}, {}".format(lat, lon))
dict = location.raw
osmid = dict.get('osm_id', 'default_value_if_null_here')
osmtype = dict.get('osm_type', 'default_value_if_null_here')
osmaddress = dict.get('display_name', 'default_value_if_null_here')
sleep(1)
if(osmtype == 'node'):
node_info = requests.get("http://api.openstreetmap.org/api/0.6/node/"+ osmid)
#print node_info.content
d = xmltodict.parse(node_info.content)
amenity_tags = [tag for tag in d['osm']['node']['tag'] if tag['#k'] == 'amenity']
amenity_name = [tag for tag in d['osm']['node']['tag'] if tag['#k'] == 'name']
if len(amenity_tags) != 0:
amenity_type = amenity_tags[0]['#v']
amenity_type_name = amenity_name[0]['#v']
print amenity_type
print amenity_type_name
print url
obtainImages3()
I want to run the code following the comment if and only if geo-info is available on the image obtained. In my group_images api call, I am asking for geo info to be returned if it it present.
I have tried for example if image.attrib['geo'] != None, and have tried a try/ exception block but all are returning errors, especially key errors.
Can anyone suggest an easy way to figure out if the image has geo info, so that the code below the comment will only run if it is available?

wordpresslib attaching images to posts

looking at the example provided by wordpresslib, its very straight forward on how to upload images to the media library. However, the attachment of images looks like it was never finished. Has anyone successfully attached the images?
#!/usr/bin/env python
"""
Small example script that publish post with JPEG image
"""
# import library
import wordpresslib
print 'Example of posting.'
print
url = raw_input('Wordpress URL (xmlrpc.php will be added):')
user = raw_input('Username:')
password = raw_input('Password:')
# prepare client object
wp = wordpresslib.WordPressClient(url+"xmlrpc.php", user, password)
# select blog id
wp.selectBlog(0)
# upload image for post
# imageSrc = wp.newMediaObject('python.jpg')
# FIXME if imageSrc:
# create post object
post = wordpresslib.WordPressPost()
post.title = 'Test post'
post.description = '''
Python is the best programming language in the earth !
No image BROKEN FIXME <img src="" />
'''
#post.categories = (wp.getCategoryIdFromName('Python'),)
# Add tags
post.tags = ["python", "snake"]
# do not publish post
idNewPost = wp.newPost(post, False)
print
print 'Posting successfull! (Post has not been published though)'
WordPressPost class:
class WordPressPost:
"""Represents post item
"""
def __init__(self):
self.id = 0
self.title = ''
self.date = None
self.permaLink = ''
self.description = ''
self.textMore = ''
self.excerpt = ''
self.link = ''
self.categories = []
self.user = ''
self.allowPings = False
self.allowComments = False
self.tags = []
self.customFields = []
def addCustomField(self, key, value):
kv = {'key':key, 'value':value}
self.customFields.append(kv)

Wordpress saves images as website.com/wp-content/uploads/YEAR/MONTH/FILENAME
Adding a simple image tag with the above format in to post.description display the image on the post.
where YEAR is the current year with a 4 digit format (ex. 2015)
and MONTH is the current month with a leading zero (ex. 01,02,... 12)
and FILENAME is the file name submitted via imageSrc = wp.newMediaObject('python.jpg')
Example file name: website.com/wp-content/uploads/2015/06/image.jpg
Here is how I posted my image:
import time
import wordpresslib
import Image
from datetime import datetime
time = datetime.now()
h = str(time.strftime('%H'))
m = str(time.strftime('%M'))
s = str(time.strftime('%S'))
mo = str(time.strftime('%m'))
yr = str(time.strftime('%Y'))
url = 'WORDPRESSURL.xmlrpc.php'
wp = wordpresslib.WordPressClient(url,'USERNAME','PASSWORD')
wp.selectBlog(0)
imageSrc = wp.newMediaObject('testimage'+h+m+s'.jpg') #Used this format so that if i post images with the same name its unlikely they will override eachother
img = 'http://WORDPRESSURL/wp-content/uploads/'+yr+'/'+mo+'/testimage'+h+m+s+'.jpg'
post=wordpresslib.WordPressPost()
post.title='title'
post.description='<img src="'+img+'"/>'
idPost=wp.newPost(post,true)