I made an API call to a restful service that by default comes with 100 paginated pages.
I have been working with my response and even built search functions into the app but I cannot figure out how to use the pagination.
Would someone please explain how I should move through the pagination of the response?
From the api
The pagination info is included in the Link header. It is important
to follow these Link header values instead of constructing your own
URLs.
And this is the current view I am working with. I have a plus and minus button in my search controls. I was hoping to allow people to move forward or backwards through the pagination with these controls. Is this not just a url I need to hit to move forward?
When I add rel="next" to the end of the url it just reloads the same results.
here is my view for reference.
def graphs(request):
mtg_url = 'https://api.deckbrew.com/mtg/cards'
if request.user.is_authenticated():
if request.method == 'POST':
data = []
mtg_colors = request.POST.get('colors')
mtg_type = request.POST.get('type')
mtg_supertypes = request.POST.get('supertypes')
mtg_subtypes = request.POST.get('subtypes')
mtg_name = request.POST.get('mtgName')
type_search = '?color='+mtg_colors+'&type='+mtg_type+'&supertypes='+mtg_supertypes+'&subtypes='+mtg_subtypes
nameSearch = '/typeahead?q='+mtg_name
if mtg_name == "":
r = requests.get(mtg_url+type_search)
print(r)
else:
r = requests.get(mtg_url+nameSearch)
jsonList = r.json()
for cards in jsonList:
data.append(cards)
print(data)
return render(request, 'graphs/graphs.html', {'data': data})
else:
data = []
r = requests.get(mtg_url)
jsonList = r.json()
for cards in jsonList:
data.append(cards)
return render(request, 'graphs/graphs.html', {'data': data})
else:
return redirect('index')
Response in JSON
[
{
"name": "_____",
"id": "_____",
"url": "https://api.deckbrew.com/mtg/cards/_____",
"store_url": "http://store.tcgplayer.com/magic/unhinged/_____?partner=DECKBREW",
"types": [
"creature"
],
"subtypes": [
"shapeshifter"
],
"colors": [
"blue"
],
"cmc": 2,
"cost": "{1}{U}",
"text": "{1}: This card's name becomes the name of your choice. Play this ability anywhere, anytime.",
"power": "1",
"toughness": "1",
"formats": {},
"editions": [
{
"set": "Unhinged",
"set_id": "UNH",
"rarity": "uncommon",
"artist": "Ron Spears",
"multiverse_id": 74252,
"flavor": "{1}: This card's flavor text becomes the flavor text of your choice. (This ability doesn't work because it's flavor text, not rules text (but neither does this reminder text, so you figure it out).)",
"number": "23",
"layout": "normal",
"price": {
"low": 0,
"median": 0,
"high": 0
},
"url": "https://api.deckbrew.com/mtg/cards?multiverseid=74252",
"image_url": "https://image.deckbrew.com/mtg/multiverseid/74252.jpg",
"set_url": "https://api.deckbrew.com/mtg/sets/UNH",
"store_url": "http://store.tcgplayer.com/magic/unhinged/_____?partner=DECKBREW"
}
]
},
Related
I am trying to get my reply to the Google assistant as a string to search it through a Google spreadsheet and return additional information about it. Specifically, I have a Google spreadsheet with two columns, one being for the name of a thing, and another for my rating of it. I'm using ngrok and Flask to get my script's response online. The two parts I'm struggling with are...
(1) Getting my requested thing into the program.
(2) Switching the url of the webhook and/or changing the webhook so that the system can output the final result.
I am a beginner so I'm probably misusing a few terms here but anyways my code is below. I put a bunch of underscores in the places where I thought sensitive information was.
action.json >>
"actions": [
{
"description": "Welcome Intent",
"name": "MAIN",
"fulfillment": {
"conversationName": "jdii ratings"
},
"intent": {
"name": "actions.intent.MAIN",
"trigger": {
"queryPatterns": [
"What did I rate $Test:text",
"Look up $Test:text on my ratings spreadsheet"
]
},
"parameters": [
{
"name": "text",
"type": "Test"
}
]
}
}
],
"conversations": {
"jdii ratings": {
"name": "jdii ratings",
"url": "<_________>",
"fulfillmentApiVersion": 2
}
}
}
program.py >>
import random
import types
import gspread
from flask import Flask
from google.cloud import dialogflow_v2
from oauth2client.service_account import ServiceAccountCredentials
app = Flask(__name__)
wanted_thingy = "Squirt"
def get_requested_thingy_rating(requested_thingy_name):
scopes = [
'https://www.googleapis.com/auth/spreadsheets',
'https://www.googleapis.com/auth/drive'
]
credentials = ServiceAccountCredentials.from_json_keyfile_name("_______.json", scopes)
file = gspread.authorize(credentials)
sheet = file.open("JDII Ratings API")
requested_thingy_rating_final = 0.0
if not isinstance(requested_thingy_name, types.NoneType):
sheet = sheet.sheet1
thingy_list = sheet.col_values(1)
ratings_list = sheet.col_values(2)
requested_thingy_rating = 0.0
if requested_thingy_name in thingy_list:
thingy_pos = thingy_list.index(requested_thingy_name)
requested_thingy_rating = ratings_list[thingy_pos]
requested_thingy_rating_final = str(requested_thingy_rating)
# split_string = ready_string.split('\'')
# requested_thingy_rating_final = split_string[1]
print(requested_thingy_rating_final)
return requested_thingy_rating_final
#app.route("/start", methods=['POST', 'GET', 'PUT'])
def main():
response = {
"expectUserResponse": True,
"expectedInputs": [
{
"inputPrompt": {
"richInitialPrompt": {
"items": [
{
"simpleResponse": {
"textToSpeech": "What would you like to look up?",
"displayText": "What would you like to look up?"
}
}
]
}
},
"possibleIntents": [
{
"intent": "actions.intent.TEXT"
}
]
}
]
}
response_text = json.dumps(response)
return response_text
#app.route("/webhook", methods=['POST', 'GET', 'PUT'])
def main2():
global wanted_thingy
wanted_thingy_final = str(wanted_thingy)
wanted_thingy_rating = get_requested_thingy_rating(wanted_thingy_final)
print(wanted_thingy)
string = f"You rated {wanted_thingy} a {wanted_thingy_rating} out of ten."
response2 = {
"expectUserResponse": False,
"finalResponse": {
"richResponse": {
"items": [
{
"simpleResponse": {
'ssml': f'<speak>{string}</speak>'
}
}
]
}
}
}
response_text = json.dumps(response2)
return response_text
app.run()
I am working with a JSON request.get that returns a list. I will like to save each individual object in the response to my models so I did this:
in views.py:
def save_ram_api(request):
r = requests.get('https://ramb.com/ciss-api/v1/')
# data = json.loads(r)
data = r.json()
for x in data:
title = x["title"]
ramyo_donotuse = x["ramyo"]
date = x["date"]
thumbnail = x["thumbnail"]
home_team_name = x["side1"]["name"]
away_team_name = x["side2"]["name"]
competition_name = x["tournament"]["name"]
ramAdd = ramSample.objects.create(title=title, ramyo_donotuse=ramyo_donotuse, date=date, thumbnail=thumbnail, home_team_name=home_team_name, away_team_name=away_team_name, competition_name=competition_name)
ramAdd.save()
return HttpResponse("Successfully submitted!")
This works fine except that it would only save the last objects on the list.
the JSON response list (as a random 60 objects at any time) would look something like:
[
{
"title": "AY - BasketMouth",
"ramyo": "AY de comedian"
"side1": {
"name": "Comedy Central",
"url": "https:\/\/www.rabithole.com\/laugh\/dave-chappel\/"
},
"side2": {
"name": "Basket Mouth",
"url": "https:\/\/www.rabithole.com\/laugh\/chris-rockie\/"
},
"tournament": {
"name": "Night of a thousand laugh",
"id": 15,
"url": "https:\/\/www.rabithole.com\/laugh\/chris-rockie\/"
},
"points": [
{
"nature": "Gentle",
"phrase": "Just stay"
},
{
"nature": "Sarcastic",
"phrase": "Help me"
}
]
},
{
"title": "Dave - Chris",
"ramyo": "Dave Chapelle"
"side1": {
"name": "Comedy Central",
"url": "https:\/\/www.rabithole.com\/laugh\/dave-chappel\/"
},
"side2": {
"name": "Chris Rockie",
"url": "https:\/\/www.rabithole.com\/laugh\/chris-rockie\/"
},
"tournament": {
"name": "Tickle me",
"id": 15,
"url": "https:\/\/www.rabithole.com\/laugh\/chris-rockie\/"
},
"points": [
{
"nature": "Rogue",
"phrase": "Just stay"
}
]
}
]
In this case my views.py will only save the last dictionary on the list, ignoring the other 59.
My question would be:
How do I get the views.py to save the entire objects on the list?
Notice that the "points" is also a list that contains one or more dictionaries, any help how to save this as well?
Your code is saving only the last object in the list because you are creating and saving the object outside of the loop. Try this,
def save_ram_api(request):
r = requests.get('https://ramb.com/ciss-api/v1/')
# data = json.loads(r)
data = r.json()
for x in data:
title = x["title"]
ramyo_donotuse = x["ramyo"]
date = x["date"]
thumbnail = x["thumbnail"]
home_team_name = x["side1"]["name"]
away_team_name = x["side2"]["name"]
competition_name = x["tournament"]["name"]
ramAdd = ramSample.objects.create(title=title, ramyo_donotuse=ramyo_donotuse, date=date, thumbnail=thumbnail, home_team_name=home_team_name, away_team_name=away_team_name, competition_name=competition_name)
ramAdd.save()
return HttpResponse("Successfully submitted!")
How do I get the views.py to save the entire objects on the list?
Notice that the "points" is also a list that contains one or more
dictionaries, any help how to save this as well?
Regarding your those questions
If you are using PostgreSQL as a database then you can use Django's built is JSONField and ArrayField for PostgreSQL database.
And if your database is not PostgreSQL you can use jsonfield library.
I have a form in my template that when submitted, provides search filters for my database query. The query otherwise returns all documents. I am using flask-paginate to navigate the results. My bug is, if the user is on page 2 when they submit the form, the page reloads page 2 with the results of the new query, even if the number of documents returned does not equate to more than 1 page. In this case, the page that loads is just blank and there are no pagination links to navigate back to page 1. So far, I have semi solved this by changing the value of flask-paginate's 'page' variable to equal 1 when the form is submitted. It now seems to load the results from page 1, when I print(page) it correctly prints '1'. However, the query string in the url still says ?page=2. I am therefore looking for a cleaner solution? or a way to clear the query string?
#app.route("/thingstodo/<city>", methods=["GET", "POST"])
def suggestion_list(city):
form = FilterResultsForm()
cities = mongo.db.cities
page, per_page, offset = get_page_args(
page_parameter="page", per_page_parameter="per_page"
)
print(page)
if form.validate_on_submit():
page = 1
per_page = 3
offset = (page - 1) * per_page
query = ""
if form.validate_on_submit() or "filters" in session:
filters = (
form.category.data if form.category.data else session["filters"]
)
session["filters"] = filters
array = []
for filter in filters:
newdict = {}
newdict["thingsToDo.category"] = filter
array.append(newdict)
query = cities.aggregate(
[
{"$unwind": "$thingsToDo"},
{"$match": {"location": city, "$or": array}},
{
"$lookup": {
"from": "users",
"localField": "thingsToDo.author",
"foreignField": "username",
"as": "user_profile",
}
},
{"$unwind": "$user_profile"},
{
"$project": {
"suggestion": "$thingsToDo.suggestion",
"cost": "$thingsToDo.cost",
"category": "$thingsToDo.category",
"url": "$thingsToDo.url",
"comment": "$thingsToDo.comment",
"author": "$user_profile.username",
"profile": "$user_profile.picture",
}
},
]
)
else:
query = cities.aggregate(
[
{"$match": {"location": city}},
{"$unwind": "$thingsToDo"},
{
"$lookup": {
"from": "users",
"localField": "thingsToDo.author",
"foreignField": "username",
"as": "user_profile",
}
},
{"$unwind": "$user_profile"},
{
"$project": {
"suggestion": "$thingsToDo.suggestion",
"cost": "$thingsToDo.cost",
"category": "$thingsToDo.category",
"url": "$thingsToDo.url",
"comment": "$thingsToDo.comment",
"author": "$user_profile.username",
"profile": "$user_profile.picture",
}
},
]
)
results = list(query)
total = len(results)
suggestions = results[offset: offset + per_page]
pagination = Pagination(
page=page, per_page=per_page, total=total, css_framework="bootstrap4"
)
return render_template(
"thingstodo.html",
city=city,
things=suggestions,
page=page,
per_page=per_page,
pagination=pagination,
form=form,
title="Things to do",
)
I have created a Facebook Messenger bot which works fine. I have used the Button Template and Image template, and both work perfectly. But when I try the Generic Template, I get no response. I have simply copy pasted the code from here, by performing the appropriate modifications.
I don't know how to debug. Facebook Messenger gives no output on the messaging box. I am currently running the app via Heroku.
Here is my code:
def send_message(token, recipient):
r = requests.post("https://graph.facebook.com/v2.6/me/messages",
params={"access_token": token},
data=json.dumps({
"recipient":{
"id":recipient
},
"message":{
"attachment":{
"type":"template",
"payload":{
"template_type":"generic",
"elements":[
{
"title":"Welcome to Peter\'s Hats",
"image_url":"http://www.godominion.com/content/images/feature-img-small-appliance-electronics.png",
"subtitle":"We\'ve got the right hat for everyone.",
"default_action": {
"type": "web_url",
"url": "https://peterssendreceiveapp.ngrok.io/view?item=103",
"messenger_extensions": true,
"webview_height_ratio": "tall",
"fallback_url": "https://peterssendreceiveapp.ngrok.io/"
},
"buttons":[
{
"type":"web_url",
"url":"https://petersfancybrownhats.com",
"title":"View Website"
}
]
}
]
}
}
}
}),
headers={'Content-type': 'application/json'})
if r.status_code != requests.codes.ok:
print r.text
I would appreciate any help.
Thank you.
EDIT 1: SOLUTION
I got rid of the issue by commenting out:
"messenger_extensions": true,
and
"fallback_url": "https://peterssendreceiveapp.ngrok.io/"},
I'm sure this is not the correct method. But as I am creating a bot, without actual links, this works.
On the second button, "url":"https://petersfancybrownhats.com" is broken.
Try like this
firstly make a function
def function():
extra_data = {
"attachment": {
"type": "template",
"payload": {
"template_type": "generic",
"elements": [
{
"title": "Any Title",
"image_url": "https://mbtskoudsalg.com/images/road-clipart-journey-3.png",
"subtitle": "Subtitle.",
"buttons": [
{
"type": "web_url",
"title": "View",
"url": "**MAKE SURE TO WHITELIST THIS URL**", # URL
"messenger_extensions": "true",
"webview_height_ratio": "full"
}
]
}
]
}
}
}
# w_message = "Hi there! How may I help you?"
fb_message_template(extra_data["attachment"], "****RECIEVER ID****")
Make another function
import requests
# // Importing User Defined Modules // #
from get_environ_var import get_environ_var
# // Global vars // #
ACCESS_TOKEN = "FB_ACCESS_TOKEN"
def fb_message_template(extra_data, sender_id):
"""This function sends template message to facebook"""
data = {
'recipient': {'id': sender_id},
'message': {
"attachment": extra_data
}
}
qs = 'access_token=' + ACCESS_TOKEN
resp = requests.post('https://graph.facebook.com/v2.6/me/messages?' + qs, json=data)
print(resp.content)
I just want to grab public information from my accounts on two social media sites. (Instagram and Twitter) My code returns info for twitter, and I know the xpath is correct for instagram but for some reason i'm not getting data for it. I know the XPATH's could be more specific but I can fix that later. Both my accounts are public.
1) I thought maybe it didn't like the python header, so I tried changing it and I still get nothing. That line is commented out but its still there.
2) I heard something about an API on github, this lengthy code is very intimidating and way above my level of understanding. I don't know more than half of what i'm reading on there.
from lxml import html
import requests
import webbrowser
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}
#page = requests.get('https://www.instagram.com/<my account>/', headers=headers)
page = requests.get('https://www.instagram.com/<my account>/')
tree = html.fromstring(page.text)
pageTwo = requests.get('http://www.twitter.com/<my account>')
treeTwo = html.fromstring(pageTwo.text)
instaFollowers = tree.xpath("//span[#data-reactid='.0.1.0.0:0.1.3.1.0']/span[2]/text()")
instaFollowing = tree.xpath("//span[#data-reactid='.0.1.0.0:0.1.3.2.0']/span[2]/text()")
twitFollowers = treeTwo.xpath("//a[#data-nav='followers']/span[#class='ProfileNav-value']/text()")
twitFollowing = treeTwo.xpath("//a[#data-nav='following']/span[#class='ProfileNav-value']/text()")
print ''
print '--------------------'
print 'Social Media Checker'
print '--------------------'
print ''
print 'Instagram: ' + str(instaFollowers) + ' / ' + str(instaFollowing)
print ''
print 'Twitter: ' + str(twitFollowers) + ' / ' + str(twitFollowing)
As mentioned, Instragram's page source does not reflect its rendered source as a Javascript function is called to pass content from JSON data to browser. Hence, what Python scrapes in page source does not show exactly what browser renders to screen. Welcome to the new world of dynamic web programming! Consider using Instagram's API or other web parser that can retrieve html generated content (not just page source).
With that said, if you simply need the IG account data you can still use Python's lxml to XPath the JSON content in <script> tag (specifically sixth occurrence but adjust to your needed page). Below example parses Google's Instagram JSON data:
import lxml.etree as et
import urllib.request as rq
rqpage = rq.urlopen('https://instagram.com/google')
txtpage = rqpage.read()
tree = et.HTML(txtpage)
jsondata = tree.xpath("//script[#type='text/javascript' and position()=6]/text()")
for i in jsondata:
print(i)
OUTPUT
window._sharedData = {"qs":"{\"shift\":10,\"header
\":\"n3bTdmHGHDgxvZYPN0KDFHqbkxd6zpTl\",\"edges\":100,\"blob
\":\"AQCq42rOTCnKOZcOxFn06L1J6_W8wY6ntAS1bX88VBClAjQD9PyJdefCzOwfSAbUdsBwHKb1QSndurPtjyN-
rHMOrZ_6ubE_Xpu908cyron9Zczkj4QMkAYUHIgnmmftuXG8rrFzq_Oq3BoXpQgovI9hefha-
6SAs1RLJMwMArrbMlFMLAwyd1TZhArcxQkk9bgRGT4MZK4Tk2VNt1YOKDN1pO3NJneFlUxdUJTdDX
zj3eY-stT7DnxF_GM_j6xwk1o\",\"iterations\":7,\"size\":42}","static_root":"
\/\/instagramstatic-a.akamaihd.net\/bluebar\/5829dff","entry_data":
{"ProfilePage":[{"__query_string":"?","__path":"\/google\/","__get_params":
{},"user":{"username":"google","has_blocked_viewer":false,"follows":
{"count":10},"requested_by_viewer":false,"followed_by":
{"count":977186},"country_block":null,"has_requested_viewer":false,"followed_
by_viewer":false,"follows_viewer":false,"profile_pic_url":"https:
\/\/instagram.ford1-1.fna.fbcdn.net\/hphotos-xfp1\/t51.2885-19\/s150x150
\/11910217_933356470069152_115044571_a.jpg","is_private":false,"full_name":
"Google","media":{"count":180,"page_info":
{"has_previous_page":false,"start_cursor":"1126896719808871555","end_cursor":
"1092117490206686720","has_next_page":true},"nodes":[{"code":"-
jipiawryD","dimensions":{"width":640,"height":640},"owner":
{"id":"1067259270"},"comments":{"count":105},"caption":"Today's the day!
Your searches are served. Happy Thanksgiving \ud83c\udf57\ud83c\udf70
#GoogleTrends","likes":
{"count":11410},"date":1448556579.0,"thumbnail_src":"https:\/
\/instagram.ford1-1.fna.fbcdn.net\/hphotos-xat1\/t51.2885-15\/e15\
/11848856_482502108621097_589421586_n.jpg","is_video":true,"id":"112689671980
8871555","display_src":"https:\/\/instagram.ford1-1.fna.fbcdn.net\/hphotos-
xat1\/t51.2885-15
...
JSON Pretty Print (extracting the window._sharedData variable from above)
See below where user (followers, following, etc.) data shows at beginning:
{
"qs": "{\"shift\":10,\"header\":\"n3bTdmHGHDgxvZYPN0KDFHqbkxd6zpTl\",\"edges\":100,\"blob\":\"AQCq42rOTCnKOZcOxFn06L1J6_W8wY6ntAS1bX88VBClAjQD9PyJdefCzOwfSAbUdsBwHKb1QSndurPtjyN-rHMOrZ_6ubE_Xpu908cyron9Zczkj4QMkAYUHIgnmmftuXG8rrFzq_Oq3BoXpQgovI9hefha-6SAs1RLJMwMArrbMlFMLAwyd1TZhArcxQkk9bgRGT4MZK4Tk2VNt1YOKDN1pO3NJneFlUxdUJTdDXzj3eY-stT7DnxF_GM_j6xwk1o\",\"iterations\":7,\"size\":42}",
"static_root": "\/\/instagramstatic-a.akamaihd.net\/bluebar\/5829dff",
"entry_data": {
"ProfilePage": [
{
"__query_string": "?",
"__path": "\/google\/",
"__get_params": {
},
"user": {
"username": "google",
"has_blocked_viewer": false,
"follows": {
"count": 10
},
"requested_by_viewer": false,
"followed_by": {
"count": 977186
},
"country_block": null,
"has_requested_viewer": false,
"followed_by_viewer": false,
"follows_viewer": false,
"profile_pic_url": "https:\/\/instagram.ford1-1.fna.fbcdn.net\/hphotos-xfp1\/t51.2885-19\/s150x150\/11910217_933356470069152_115044571_a.jpg",
"is_private": false,
"full_name": "Google",
"media": {
"count": 180,
"page_info": {
"has_previous_page": false,
"start_cursor": "1126896719808871555",
"end_cursor": "1092117490206686720",
"has_next_page": true
},
"nodes": [
{
"code": "-jipiawryD",
"dimensions": {
"width": 640,
"height": 640
},
"owner": {
"id": "1067259270"
},
"comments": {
"count": 105
},
"caption": "Today's the day! Your searches are served. Happy Thanksgiving \ud83c\udf57\ud83c\udf70 #GoogleTrends",
"likes": {
"count": 11410
},
"date": 1448556579,
"thumbnail_src": "https:\/\/instagram.ford1-1.fna.fbcdn.net\/hphotos-xat1\/t51.2885-15\/e15\/11848856_482502108621097_589421586_n.jpg",
"is_video": true,
"id": "1126896719808871555",
"display_src": "https:\/\/instagram.ford1-1.fna.fbcdn.net\/hphotos-xat1\/t51.2885-15\/e15\/11848856_482502108621097_589421586_n.jpg"
},
{
"code": "-hwbf2wr0O",
"dimensions": {
"width": 640,
"height": 640
},
"owner": {
"id": "1067259270"
},
"comments": {
"count": 95
},
"caption": "Thanksgiving dinner is waiting. But first, the airport. \u2708\ufe0f #GoogleApp",
"likes": {
"count": 12621
},
...
IF anyone is interested in this sort of thing still, using selenium solved my problems.
http://pastebin.com/5eHeDt3r
Is there a faster way ?
In case you want to find information about yourself and others without hassling with code, try this piece of software. Apart from automatic scraping, it analyzes and visualizes the received information on a PDF report from such social networks: Facebook, Twitter, Instagram and from the Google Search engine.
P.S. I am the main developer and maintainer of this project.