I am working on updating a Confluence page via python and the Confluence API.
I have found a function to write the data to the my page, unfortunately it creates a new page with my data and the old page becomes an archived version.
I have been searching the reference material and cannot see the reason I am getting a new page instead of appending the data to the end of the page. The only solution I can think of is to copy the body and append the new data to it, then create the new page ... but I am thinking there should be a way to append.
The write function code I found / am leveraging is as follows :
def write_data(auth, html, pageid, title = None):
info = get_page_info(auth, pageid)
print ("after get page info")
ver = int(info['version']['number']) + 1
ancestors = get_page_ancestors(auth, pageid)
anc = ancestors[-1]
del anc['_links']
del anc['_expandable']
del anc['extensions']
if title is not None:
info['title'] = title
data = {
'id' : str(pageid),
'type' : 'page',
'title' : info['title'],
'version' : {'number' : ver},
'ancestors' : [anc],
'body' : {
'storage' :
{
'representation' : 'storage',
'value' : str(html),
}
}
}
data = json.dumps(data)
pprint (data)
url = '{base}/{pageid}'.format(base = BASE_URL, pageid = pageid)
r = requests.put(
url,
data = data,
auth = auth,
headers = { 'Content-Type' : 'application/json' }
)
r.raise_for_status()
I am starting to think copying / appending to the body is the option, but hope someone else has encountered this issue.
Not an elegant solution, but I went with the copy the old body and append option.
Basically, wrote a simple function to return the existing body :
def get_page_content(auth, pageid):
url = '{base}/{pageid}?expand=body.storage'.format(
base = BASE_URL,
pageid = pageid)
r = requests.get(url, auth = auth)
r.raise_for_status()
return (r.json()['body']['storage']['value'])
In this example, just appending (+=) a new string to the existing body.
def write_data(auth, html, pageid, title = None):
info = get_page_info(auth, pageid)
page_content = get_page_content(auth, pageid)
ver = int(info['version']['number']) + 1
ancestors = get_page_ancestors(auth, pageid)
anc = ancestors[-1]
del anc['_links']
del anc['_expandable']
del anc['extensions']
page_content += "\n\n"
page_content += html
if title is not None:
info['title'] = title
data = {
'id' : str(pageid),
'type' : 'page',
'title' : info['title'],
'version' : {'number' : ver},
'ancestors' : [anc],
'body' : {
'storage' :
{
'representation' : 'storage',
'value' : str(page_content),
}
}
}
data = json.dumps(data)
url = '{base}/{pageid}'.format(base = BASE_URL, pageid = pageid)
r = requests.put(
url,
data = data,
auth = auth,
headers = { 'Content-Type' : 'application/json' }
)
r.raise_for_status()
print "Wrote '%s' version %d" % (info['title'], ver)
print "URL: %s%d" % (VIEW_URL, pageid)
Should note that as this is a post to a confluence body, the text being passed is html. '\n' for newline does not work, you need to pass '<br \>' etc ...
If anyone has a more elegant solution, would welcome the suggestions.
Dan.
Related
I'm making a project which will add songs from your youtube playlist to spotify playlist. everything works fine except the add_item() method. I'm getting 404 : not found response from the requests object.
Yes, i checked if the song actually exists. It does and even printed the id of the song.So it exists.
I'm following the official documentation
here is my code -
import json
from requests_oauthlib import OAuth2Session
import requests
import base64
import urllib.parse as urlparse
from urllib.parse import parse_qs
client_id = #id
client_secret = #secret
redirect_uri = 'https://www.spotify.com'
scope = "playlist-modify-public user-read-email user-read-private playlist-modify-private"
req = f'{client_id}:{client_secret}'
encoded = base64.b64encode(req.encode())
class sp :
def __init__(self) :
self.get_token()
self.get_id()
self.uri_list = []
def get_token(self) :
url = 'https://accounts.spotify.com/authorize'
oauth = OAuth2Session(client_id, redirect_uri = redirect_uri, scope = scope)
authorization_url, state = oauth.authorization_url(url)
print(authorization_url)
authorization_response = input('paste here : ')
parsed = urlparse.urlparse(authorization_response)
authorization_code = parse_qs(parsed.query)['code'][0]
# to get auth token
headers = {
'Authorization' : f'Basic {encoded.decode()}'
}
data = {
'grant_type' : 'authorization_code',
'redirect_uri' : redirect_uri,
'code' : authorization_code
}
access = requests.post('https://accounts.spotify.com/api/token', data = data, headers = headers)
response = json.loads(access.text)
self.access_token = response['access_token']
def get_id(self) :
headers = {
'Authorization' : f'Bearer {self.access_token}'
}
user_info = requests.get('https://api.spotify.com/v1/me', headers = headers)
user_info.raise_for_status()
user_info = json.loads(user_info.text)
self.user_id = user_info['id']
def search(self) :
search_url = 'https://api.spotify.com/v1/search'
headers = {
'Authorization': f'Bearer {self.access_token}'
}
params = {
'q' : 'track:Memories artist:Maroon 5',
'type' : 'track',
'limit' : 1,
}
search_response = requests.get(search_url, headers = headers, params = params)
search_response.raise_for_status()
json_response = search_response.json()
song_uri = json_response['tracks']['items'][0]['uri']
self.uri_list.append(song_uri)
def create_playlist(self) :
create_playlist_url = f'https://api.spotify.com/v1/users/{self.user_id}/playlists'
headers = {
'Authorization' : f'Bearer {self.access_token}',
'Content-Type' : 'application/json'
}
data = json.dumps({
'name' : 'new playlist'
})
response = requests.post(create_playlist_url, headers = headers, data = data)
print(response)
self.playlist_id = response.json()['uri']
def add_items(self) :
add_items_url = f'https://api.spotify.com/v1/playlists/{self.playlist_id}/tracks'
headers = {
'Authorization' : f'Bearer {self.access_token}',
'Content-Type' : 'application/json'
}
print(self.uri_list)
data = {
'uris' : json.dumps(self.uri_list)
}
res = requests.post(add_items_url, headers = headers, data = data)
print(res)
user = sp()
user.create_playlist()
user.search()
user.add_items()
Any help is appreciated. Thanks
You have this line of code for playlist creation:
self.playlist_id = response.json()['uri']
and then in items addition logic you have:
add_items_url = f'https://api.spotify.com/v1/playlists/{self.playlist_id}/tracks'
are you sure that you want to use playlist uri as playlist id?
Could you update your question with more info:
response.json() value after playlist is created
print add_items_url after f-string was declared
UPDATE
https://developer.spotify.com/documentation/web-api/reference/playlists/create-playlist/ as I can see here - the response after creation of the playlist include id field
So you should just change this line
self.playlist_id = response.json()['uri']
to
self.playlist_id = response.json()['id']
I am trying to parse data from this website.
In Network section of inspect element i found this link https://busfor.pl/api/v1/searches that is used for a POST request that returns JSON i am interested in.
But for making this POST request there is request Payload with some dictionary.
I assumed it like normal formdata that we use to make FormRequest in scrapy but it returns 403 error.
I have already tried the following.
url = "https://busfor.pl/api/v1/searches"
formdata = {"from_id" : d_id
,"to_id" : a_id
,"on" : '2019-10-10'
,"passengers" : 1
,"details" : []
}
yield scrapy.FormRequest(url, callback=self.parse, formdata=formdata)
This returns 403 Error
I also tried this by referring to one of the StackOverflow post.
url = "https://busfor.pl/api/v1/searches"
payload = [{"from_id" : d_id
,"to_id" : a_id
,"on" : '2019-10-10'
,"passengers" : 1
,"details" : []
}]
yield scrapy.Request(url, self.parse, method = "POST", body = json.dumps(payload))
But even this returns the same error.
Can someone help me. to figure out how to parse the required data using Scrapy.
The way to send POST requests with json data is the later, but you are passing a wrong json to the site, it expects a dictionary, not a list of dictionaries.
So instead of:
payload = [{"from_id" : d_id
,"to_id" : a_id
,"on" : '2019-10-10'
,"passengers" : 1
,"details" : []
}]
You should use:
payload = {"from_id" : d_id
,"to_id" : a_id
,"on" : '2019-10-10'
,"passengers" : 1
,"details" : []
}
Another thing you didn't notice are the headers passed to the POST request, sometimes the site uses IDs and hashes to control access to their API, in this case I found two values that appear to be needed, X-CSRF-Token and X-NewRelic-ID. Luckily for us these two values are available on the search page.
Here is a working spider, the search result is available at the method self.parse_search.
import json
import scrapy
class BusForSpider(scrapy.Spider):
name = 'busfor'
start_urls = ['https://busfor.pl/autobusy/Sopot/Gda%C5%84sk?from_id=62113&on=2019-10-09&passengers=1&search=true&to_id=3559']
search_url = 'https://busfor.pl/api/v1/searches'
def parse(self, response):
payload = {"from_id" : '62113',
"to_id" : '3559',
"on" : '2019-10-10',
"passengers" : 1,
"details" : []}
csrf_token = response.xpath('//meta[#name="csrf-token"]/#content').get()
newrelic_id = response.xpath('//script/text()').re_first(r'xpid:"(.*?)"')
headers = {
'X-CSRF-Token': csrf_token,
'X-NewRelic-ID': newrelic_id,
'Content-Type': 'application/json; charset=UTF-8',
}
yield scrapy.Request(self.search_url, callback=self.parse_search, method="POST", body=json.dumps(payload), headers=headers)
def parse_search(self, response):
data = json.loads(response.text)
I have a function that tries to write a web page into conflunece. Instead of posting the contents of a file, it writes the name of the file to the page.
I am using the python requests module to write to the web page.
On the web page I see this:
../output_files/aws_instance_list/html/aws-master-list-06-05-2019.html
As the only contents of the page.
This is the code that I'm using to write to the page:
htmlfile = '../output_files/aws_instance_list/html/aws-master-list-06-05-2019.html'
pageid = 138317098
auth = ('username','password')
write_data(auth, htmlfile, pageid, 'AWS EC2 Instance List')
def write_data(auth, htmlfile, pageid, title = None):
info = get_page_info(auth, pageid)
ver = int(info['version']['number']) + 1
ancestors = get_page_ancestors(auth, pageid)
anc = ancestors[-1]
del anc['_links']
del anc['_expandable']
del anc['extensions']
if title is not None:
info['title'] = title
data = {
'id' : str(pageid),
'type' : 'page',
'title' : info['title'],
'version' : {'number' : ver},
'ancestors' : [anc],
'body' : {
'storage' :
{
'representation' : 'storage',
'value' : str(htmlfile),
}
}
}
data = json.dumps(data)
url = '{base}/{pageid}'.format(base = BASE_URL, pageid = pageid)
r = requests.put(
url,
data = data,
auth = auth,
headers = { 'Content-Type' : 'application/json' }
)
r.raise_for_status()
print("Wrote '%s' version %d" % (info['title'], ver))
print("URL: %s%d" % (VIEW_URL, pageid))
I've looked at the contents of the 'htmlfile' and can see that it contains valid HTML.
How can I write the contents of the file to the page instead of the name of the file?
You need to read the contents of the file.
'value' : open(htmlfile).read(),
I am trying to create a merge request using the GitLab Merge Request API with python and the python requests package. This is a snippet of my code
import requests, json
MR = 'http://www.gitlab.com/api/v4/projects/317/merge_requests'
id = '317'
gitlabAccessToken = 'MySecretAccessToken'
sourceBranch = 'issue110'
targetBranch = 'master'
title = 'title'
description = 'description'
header = { 'PRIVATE-TOKEN' : gitlabAccessToken,
'id' : id,
'title' : title,
'source_branch' : sourceBranch,
'target_branch' : targetBranch
}
reply = requests.post(MR, headers = header)
status = json.loads(reply.text)
but I keep getting the following message in the reply
{'error': 'title is missing, source_branch is missing, target_branch is missing'}
What should I change in my request to make it work?
Apart from PRIVATE-TOKEN, all the parameters should be passed as form-encoded parameters, meaning:
header = {'PRIVATE-TOKEN' : gitlabAccessToken}
params = {
'id' : id,
'title' : title,
'source_branch' : sourceBranch,
'target_branch' : targetBranch
}
reply = requests.post(MR, data=params, headers=header)
I am trying to retrieve events from the Outlook Calendar using the API. I only want to retrieve events after today. I'm using the following code (which is basically a cut and paste from Microsoft's tutorial):
def make_api_call(method, url, token, user_email, payload = None, parameters = None):
# Send these headers with all API calls
headers = { 'User-Agent' : 'python_tutorial/1.0',
'Authorization' : 'Bearer {0}'.format(token),
'Accept' : 'application/json',
'X-AnchorMailbox' : user_email }
# Use these headers to instrument calls. Makes it easier
# to correlate requests and responses in case of problems
# and is a recommended best practice.
request_id = str(uuid.uuid4())
instrumentation = { 'client-request-id' : request_id,
'return-client-request-id' : 'true' }
headers.update(instrumentation)
response = None
if (method.upper() == 'GET'):
response = requests.get(url, headers = headers, params = parameters)
print response.url
elif (method.upper() == 'DELETE'):
response = requests.delete(url, headers = headers, params = parameters)
elif (method.upper() == 'PATCH'):
headers.update({ 'Content-Type' : 'application/json' })
response = requests.patch(url, headers = headers, data = json.dumps(payload), params = parameters)
elif (method.upper() == 'POST'):
headers.update({ 'Content-Type' : 'application/json' })
response = requests.post(url, headers = headers, data = json.dumps(payload), params = parameters)
return response
This function gets called from the following:
get_events_url = outlook_api_endpoint.format('/Me/Events')
query_parameters = {'$select': 'Subject,Start,End',
'$orderby': 'Start/DateTime ASC',
'startDateTime' : datetime.datetime.now().isoformat(),}
r = make_api_call('GET', get_events_url, access_token, user_email, parameters = query_parameters)
It simply returns the first 10 entries in the calendar rather than entries from today onwards. How do I get back entries for specific dates?
I've found a solution that works for me. First I've changed events to calendarview, I've capitalised StartDateTime and added EndDateTime so it looks like this:
get_events_url = outlook_api_endpoint.format('/Me/calendarview')
query_parameters = {'$select': 'Subject,Start,End',
'$orderby': 'Start/DateTime ASC',
'StartDateTime' : datetime.datetime.now().isoformat(),
'EndDateTime' : datetime.datetime(2100,12,31),}
r = make_api_call('GET', get_events_url, access_token, user_email, parameters = query_parameters)
It works so I'm happy