We Keep Coding

Trouble with the google api client. It does not want to read a variable as a list in the body section

Why am I getting an error on the batch update line?
from googleapiclient.discovery import build
from google.oauth2 import service_account
import pandas as pd
# Authenticate into Target worksheet
# Provide the right methods from google and the right scopes (app services)
SERVICE_ACCOUNT_FILE = 'keys.json'
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
creds = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE, scopes=SCOPES)
# The ID for the target spreadsheet.
TARGET_WORKBOOK = 'xxx'
service = build('sheets', 'v4', credentials=creds)
sheet_id = 2083229665
# Build the service object for the Google Sheets API
service = build('sheets', 'v4', credentials=creds)
# Load the DataFrame
data = [['Name', 'Age'], ['Alice', 25], ['Bob', 30],['steve','55'], ['gayle','54']]
df = pd.DataFrame(data)
datalist = df.values.tolist()
# Define the range of the data
range_ = 'ExpenseLog!A1:' + chr(ord('A') + len(df.columns) - 1) + str(len(df))
# Define the request body Execute the request
requests = []
for i, row in enumerate(datalist):
# Format the row values into the proper structure
values = [{'userEnteredValue': {'stringValue': cell}} for cell in row]
# Create the updateCells request for the current row
request = {
'updateCells': {
'range': {
'sheetId' : sheet_id,
'startRowIndex': i,
'endRowIndex': i + 1,
'startColumnIndex': 0,
'endColumnIndex': len(row)
},
'rows': [
{
'values': values
}
],
'fields': 'userEnteredValue'
}
}
# Append the request to the list of requests
requests.append(request)
# Perform the batch update
result = service.spreadsheets().batchUpdate(spreadsheetId=TARGET_WORKBOOK,
body={'requests': requests}).execute()
I've seen several attempts at describing this but they all seem to think that having requests as a list in the body section will work, but it doesn't seem to be.
Any thoughts?
Thanks
100's of print statements later and everything seems to have the values, but they are not getting read when used in the last line of the code.

KeyError "id" Python: Why isn't the key in the dictionary?

I wanted to know why this error occurs. I know the key is not in the dictionary but how come? Line 94 is labeled in the code below.
Traceback (most recent call last): File "", line 154, in
cp.add_song_to_playlist()
File "", line 127, in add_song_to_playlist
playlist_id = self.create_playlist()
File "", line 94, in create_playlist
return response_json["id"] KeyError: 'id'
import json
import os
import google_auth_oauthlib.flow
import googleapiclient.discovery
import googleapiclient.errors
import requests
import youtube_dl
from exceptions import ResponseException
from secrets import spotify_token, spotify_user_id
class CreatePlaylist:
def __init__(self):
self.youtube_client = self.get_youtube_client()
self.all_song_info = {}
def get_youtube_client(self):
""" Log Into Youtube, Copied from Youtube Data API """
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
client_secrets_file = "client_secret.json"
# Get credentials and create an API client
scopes = ["https://www.googleapis.com/auth/youtube.readonly"]
flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file(
client_secrets_file, scopes)
credentials = flow.run_console()
# from the Youtube DATA API
youtube_client = googleapiclient.discovery.build(
api_service_name, api_version, credentials=credentials)
return youtube_client
def get_liked_videos(self):
"""Grab Our Liked Videos & Create A Dictionary Of Important Song Information"""
request = self.youtube_client.videos().list(
part="snippet,contentDetails,statistics",
myRating="like"
)
response = request.execute()
# collect each video and get important information
for item in response["items"]:
video_title = item["snippet"]["title"]
youtube_url = "https://www.youtube.com/watch?v={}".format(
item["id"])
# use youtube_dl to collect the song name & artist name
video = youtube_dl.YoutubeDL({}).extract_info(
youtube_url, download=False)
song_name = video["track"]
artist = video["artist"]
if song_name is not None and artist is not None:
# save all important info and skip any missing song and artist
self.all_song_info[video_title] = {
"youtube_url": youtube_url,
"song_name": song_name,
"artist": artist,
# add the uri, easy to get song to put into playlist
"spotify_uri": self.get_spotify_uri(song_name, artist)
}
def create_playlist(self):
"""Create A New Playlist"""
request_body = json.dumps({
"name": "Youtube Liked Vids",
"description": "All Liked Youtube Videos",
"public": True
})
query = "https://api.spotify.com/v1/users/{}/playlists".format(
spotify_user_id)
response = requests.post(
query,
data=request_body,
headers={
"Content-Type": "application/json",
"Authorization": "Bearer {}".format(spotify_token)
}
)
response_json = response.json()
# playlist id
try:
response_json = response.json()
except Exception:
return None
return response_json.get('id', None)
def get_spotify_uri(self, song_name, artist):
"""Search For the Song"""
query = "https://api.spotify.com/v1/search?query=track%3A{}+artist%3A{}&type=track&offset=0&limit=20".format(
song_name,
artist
)
response = requests.get(
query,
headers={
"Content-Type": "application/json",
"Authorization": "Bearer {}".format(spotify_token)
}
)
response_json = response.json()
songs = response_json["tracks"]["items"]
# only use the first song
uri = songs[0]["uri"]
return uri
def add_song_to_playlist(self):
"""Add all liked songs into a new Spotify playlist"""
# populate dictionary with our liked songs
self.get_liked_videos()
# collect all of uri
uris = [info["spotify_uri"]
for song, info in self.all_song_info.items()]
# create a new playlist
playlist_id = self.create_playlist()
# add all songs into new playlist
request_data = json.dumps(uris)
query = "https://api.spotify.com/v1/playlists/{}/tracks".format(
playlist_id)
response = requests.post(
query,
data=request_data,
headers={
"Content-Type": "application/json",
"Authorization": "Bearer {}".format(spotify_token)
}
)
# check for valid response status
if response.status_code != 200:
raise ResponseException(response.status_code)
response_json = response.json()
return response_json
if __name__ == '__main__':
cp = CreatePlaylist()
cp.add_song_to_playlist()

This exception occurs when you try to get undefined keys value with [] from dict. To gently handle this try to use .get('id', None).
try:
response_json = response.json()
except Exception:
return None
return response_json.get('id', None)
it will return None if there is no value for the key 'id'.
You can replace None with any fallback value.

Upload PDF from Python as attachment to Salesforce Object

I am trying to upload a pdf generated in Python as an attachment to a salesforce object using the simple_salesforce Python package. I have tried several different ways to accomplish this, but have had no luck so far. Here is the code
import base64
import json
from simple_salesforce import Salesforce
instance = ''
sessionId = sf.session_id
def pdf_encode(pdf_filename):
body = open(pdf_filename, 'rb') #open binary file in read mode
body = body.read()
body = base64.encodebytes(body)
body = pdf_encode('PDF_Report.pdf')
response = requests.post('https://%s.salesforce.com/services/data/v29.0/sobjects/Attachment/' % instance,
headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer %s' % sessionId },
data = json.dumps({
'ParentId': parent_id,
'Name': 'test.txt',
'body': body
})
)
I get this error.
TypeError: Object of type bytes is not JSON serializable
I have also tried to use
body = base64.encodebytes(body).decode('ascii')
in my code, but I can't get that to work either. I get the error
UnicodeError: encoding with 'idna' codec failed (UnicodeError: label empty or too long)
Any suggestions on how to upload a PDF in Python 3 into Salesforce as an attachment using simple_salesforce?

I was working on this and found a few resources to upload files. I created one for myself using that.
Below is the code that you can use for Python and have the file uploaded on Salesforce.
import requests
import base64
import json
params = {
"grant_type": "password",
"client_id": "Your_Client_Id",
"client_secret": "Your_Client_Secret",
"username": "YOUR_EMAIL#procureanalytics.com.pcsandbox", # The email you use to login
"password": "YOUR_PASSWORD+YOUR_SECURITY_TOKEN" # Concat your password and your security token
}
r = requests.post("https://test.salesforce.com/services/oauth2/token", params=params)
# if you connect to a Sandbox, use test.salesforce.com instead
access_token = r.json().get("access_token")
instance_url = r.json().get("instance_url")
print("Access Token:", access_token)
print("Instance URL", instance_url)
#######################################################################################
# Helper function
#######################################################################################
def sf_api_call(action, parameters = {}, method = 'get', data = {}):
"""
Helper function to make calls to Salesforce REST API.
Parameters: action (the URL), URL params, method (get, post or patch), data for POST/PATCH.
"""
headers = {
'Content-type': 'application/json',
'Accept-Encoding': 'gzip',
'Authorization': 'Bearer %s' % access_token
}
if method == 'get':
r = requests.request(method, instance_url+action, headers=headers, params=parameters, timeout=30)
elif method in ['post', 'patch']:
r = requests.request(method, instance_url+action, headers=headers, json=data, params=parameters, timeout=10)
else:
# other methods not implemented in this example
raise ValueError('Method should be get or post or patch.')
print('Debug: API %s call: %s' % (method, r.url) )
if r.status_code < 300:
if method=='patch':
return None
else:
return r.json()
else:
raise Exception('API error when calling %s : %s' % (r.url, r.content))
# Test connection
print(json.dumps(sf_api_call('/services/data/v40.0/query/', {
'q': 'SELECT Account.Name, Name, CloseDate from Opportunity where IsClosed = False order by CloseDate ASC LIMIT 1'
}), indent=2))
#######################################################################################
# File Upload from directory
#######################################################################################
# 1) Create a ContentVersion
path = "Folder_name\Sample_pdf.pdf"
with open(path, "rb") as f:
encoded_string = base64.b64encode(f.read()).decode("utf-8")
ContentVersion = sf_api_call('/services/data/v40.0/sobjects/ContentVersion', method="post", data={
'Title': 'Sample_pdf file',
'PathOnClient': path,
'VersionData': encoded_string,
})
ContentVersion_id = ContentVersion.get('id')
# 2) Get the ContentDocument id
ContentVersion = sf_api_call('/services/data/v40.0/sobjects/ContentVersion/%s' % ContentVersion_id)
ContentDocument_id = ContentVersion.get('ContentDocumentId')
# 3) Create a ContentDocumentLink
Id = "Abcd123" # This Id can be anything: Account_Id or Lead_Id or Opportunity_Id
ContentDocumentLink = sf_api_call('/services/data/v40.0/sobjects/ContentDocumentLink', method = 'post', data={
'ContentDocumentId': ContentDocument_id,
'LinkedEntityId': Id,
'ShareType': 'V'
})
How to use
Step 1:
Key in your email address and password here. Please note that the password here is a string of 'your password' and your 'security token'.
# Import libraries
import requests
import base64
import json
params = {
"grant_type": "password",
"client_id": "Your_Client_Id",
"client_secret": "Your_Client_Secret",
"username": "YOUR_EMAIL#procureanalytics.com.pcsandbox", # The email you use to login
"password": "YOUR_PASSWORD+YOUR_SECURITY_TOKEN" # Concat your password and your security token
}
r = requests.post("https://test.salesforce.com/services/oauth2/token", params=params)
# if you connect to a Sandbox, use test.salesforce.com instead
access_token = r.json().get("access_token")
instance_url = r.json().get("instance_url")
print("Access Token:", access_token)
print("Instance URL", instance_url)
You can get your security token on Salesforce through Account >> Settings >> Reset My Security Token.
You will receive an email from salesforce with your security token.
Step 2:
Choose appropriate link for request.post
For Sandbox environment:
r = requests.post("https://test.salesforce.com/services/oauth2/token", params=params)
For Production enviroment:
r = requests.post("https://login.salesforce.com/services/oauth2/token", params=params)
After your initial connection is ready, the output on the second cell should look something like this:
Access Token: !2864b793dbce2ad32c1ba7d71009ec84.b793dbce2ad32c1ba7d71009ec84
Instance URL https://your_company_name--pcsandbox.my.salesforce.com
Step 3:
Under the 'File upload from a directory' cell (Cell #5), specify your file path.
In my case, this is
# 1) Create a ContentVersion
path = "Folder_name\Sample_pdf.pdf"
with open(path, "rb") as f:
encoded_string = base64.b64encode(f.read()).decode("utf-8")
Step 4:
Under the same cell, mention the Id in which you would like to upload your file.
The sample code below is uploading a file on Accounts object for an account with Id: Abcd123
# 3) Create a ContentDocumentLink
Id = "Abcd123" # This Id can be anything: Account_Id or Lead_Id or Opportunity_Id
ContentDocumentLink = sf_api_call('/services/data/v40.0/sobjects/ContentDocumentLink', method = 'post', data={
'ContentDocumentId': ContentDocument_id,
'LinkedEntityId': Id,
'ShareType': 'V'
})

How to download files from Google Vault export immediately after creating it with Python API?

Using Python API, I have created an export. How do I download the .zip file in the export using the same authorized service? When creating the export, I can see the bucketName and objectNames of the cloudStorageSink, however I cannot find any documentation on how to download them to my host using the existing service that created the export
#!/usr/bin/env python
from __future__ import print_function
import datetime
import json
import time
from googleapiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
# If modifying these scopes, delete the file token.json.
SCOPES = 'https://www.googleapis.com/auth/ediscovery'
def list_exports(service, matter_id):
return service.matters().exports().list(matterId=matter_id).execute()
def get_export_by_id(service, matter_id, export_id):
return service.matters().exports().get(matterId=matter_id, exportId=export_id).execute()
def get_service():
'''
Look for an active credential token, if one does not exist, use credentials.json
and ask user for permission to access. Store new token, return the service object
'''
store = file.Storage('token.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets('credentials.json', SCOPES)
creds = tools.run_flow(flow, store)
service = build('vault', 'v1', http=creds.authorize(Http()))
return service
def create_drive_export(service, matter_id, export_name, num_days):
"""
once we have a matter_id , we can create an export under it with the relevant files we are looking for.
"""
# set times for beginning and end of query:
today = datetime.datetime.now()
print("creating a drive export at {}".format(today))
start_time = today - datetime.timedelta(days=num_days)
drive_query_options = {'includeTeamDrives': True}
user_list = ['me#gmail.com']
drive_query = {
'corpus': 'DRIVE',
'dataScope': 'ALL_DATA',
'searchMethod': 'ACCOUNT',
'accountInfo': {
'emails': user_list
},
'driveOptions': drive_query_options,
# end time is more recent date, start time is older date
'endTime': '{}-{}-{}T00:00:00Z'.format(today.year, today.month, today.day),
'startTime': '{}-{}-{}T00:00:00Z'.format(start_time.year, start_time.month, start_time.day),
'timeZone': 'Etc/GMT'
}
wanted_export = {
'name': export_name,
'query': drive_query,
'exportOptions': {
'driveOptions': {}
}
}
return service.matters().exports().create(matterId=matter_id, body=wanted_export).execute()
def get_export(service, matter_id, export_id):
return service.matters().exports().get(matterId=matter_id, exportId=export_id).execute()
def main():
service = get_service()
matter_id = '<known_matter_id>'
timestamp = datetime.datetime.now().strftime("%Y%m%d.%H%M%s")
export = create_drive_export(service, matter_id, "code_gen_export.{}".format(timestamp), 1)
# check every 5 seconds until export is done being created:
while export['status'] == 'IN_PROGRESS':
export = get_export(service, matter_id, export['id'])
print('...')
time.sleep(5)
# print(json.dumps(export, indent=2))
print(json.dumps(export['cloudStorageSink']['files'], indent=2))
if __name__ == '__main__':
main()
and running the above code produces:
creating a drive export at 2018-09-20 17:12:38.026402
...
...
...
...
...
...
[
{
"md5Hash": "hash_value",
"bucketName": "bucket_string",
"objectName": "object1_string/code_gen_export.20180920.17121537481558-custodian-docid.csv",
"size": "1684"
},
{
"md5Hash": "hash_value",
"bucketName": "bucket_string",
"objectName": "object2_string/code_gen_export.20180920.17121537481558-metadata.xml",
"size": "10600"
},
{
"md5Hash": "hash_value",
"bucketName": "bucket_string",
"objectName": "object3_string/code_gen_export.20180920.17121537481558_0.zip",
"size": "21599222"
}
]
Can I download the .zip file using the service object I created in get_service()?

After a long struggle with the above, I found the right approach with the aid of one of Googles' API support agents.
Notice that you will need to create a new service using:
build('storage', 'v1', credentials=credentials)
where cradintials is:
service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE,
scopes=SCOPES,
subject='user#domain.com'
)
(it may be that the same argument you used for your credentials: http=creds.authorize(Http()) will work as weel - I did not try that)
In addition you will need to use a byte stream library such as: io and import googleapiclient.http as well.
The full code:
import io
from google.oauth2 import service_account
from googleapiclient.discovery import build
import googleapiclient.http
SCOPES = ['https://www.googleapis.com/auth/devstorage.full_control']
SERVICE_ACCOUNT_FILE = 'yourServiceAccountFile.json'
bucket_name = 'yourBucketName'
object_name = 'yourObjectName.zip'
credentials = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE,
scopes=SCOPES,
subject='user#domain.com'
)
service = build('storage', 'v1', credentials=credentials)
req = service.objects().get_media(bucket=bucket_name, object=object_name)
out_file = io.BytesIO()
downloader = googleapiclient.http.MediaIoBaseDownload(out_file, req)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download {}%.".format(int(status.progress() * 100)))
file_name = '/Users/myUser/Downloads/new_file.zip'
open(file_name, "w").write(out_file.getvalue())

The above answer is great, but causes issue with large files, as BytesIO holds the data in memory. In a low RAM environment a 2GB can kill your download. Suggest using FileIO instead.
Change the following bit of code:
out_file = io.BytesIO()
downloader = googleapiclient.http.MediaIoBaseDownload(out_file, req)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download {}%.".format(int(status.progress() * 100)))
file_name = '/Users/myUser/Downloads/new_file.zip'
open(file_name, "w").write(out_file.getvalue())
To:
file_name = '/myfilepath/myfilename.ext'
with io.FileIO(file_name, mode='wb') as out_file:
downloader = googleapiclient.http.MediaIoBaseDownload(out_file, req)
done = False
while not done:
status, done = downloader.next_chunk()

How to use Google sheet API V4 insert note in cell by python

I don't know how to add note in a cell by Google Sheet API:
https://developers.google.com/sheets/api/guides/values#writing_multiple_ranges
I read info of Google Sheet API: https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets#CellData
I also read this question but it's still not working
Is it possible to use the Google Spreadsheet API to add a comment in a cell?
I don't know how to set note in the value.
I need someone help ~!
Thanks a lot.
from __future__ import print_function
import httplib2
import os
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None
# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/sheets.googleapis.com-python-quickstart.json
# Scope type --> https://developers.google.com/sheets/api/guides/authorizing
SCOPES = 'https://www.googleapis.com/auth/spreadsheets'
CLIENT_SECRET_FILE = 'client_secret.json' # credentials file name
APPLICATION_NAME = 'auto_update_caspar'
def get_credentials():
"""Gets valid user credentials from storage.
If nothing has been stored, or if the stored credentials are invalid,
the OAuth2 flow is completed to obtain the new credentials.
Returns:
Credentials, the obtained credential.
"""
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir,
'auto_update_client_caspar.json')
store = Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
if flags:
credentials = tools.run_flow(flow, store, flags)
else: # Needed only for compatibility with Python 2.6
credentials = tools.run(flow, store)
print('Storing credentials to ' + credential_path)
return credentials
def main():
"""Shows basic usage of the Sheets API.
Creates a Sheets API service object and prints the names and majors of
students in a sample spreadsheet:
https://docs.google.com/spreadsheets/d/1k7OmDU_QUrCPVmsEpWFRCj-4BOu6PcUb7-SQlA7T_8I/edit
"""
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
discoveryUrl = ('https://sheets.googleapis.com/$discovery/rest?'
'version=v4')
service = discovery.build('sheets', 'v4', http=http,
discoveryServiceUrl=discoveryUrl)
spreadsheetId = '1lEJeSNe5T3rNXEEMjY4D04QkoW-ngOeiFo40_S4H4FI'
rangeName = 'stress_temp!A3'
result = service.spreadsheets().values().get(
spreadsheetId=spreadsheetId, range=rangeName).execute()
print (result)
values = result.get('values', [])
if not values:
print('No data found.')
else:
print('Name, Major:')
for row in values:
# Print columns A and E, which correspond to indices 0 and 4.
print('%s' % row[0])
value_input_option = 'RAW'
range_name = 'stress_temp!A3'
values = [['aa']]
data = [
{
'range': range_name,
'values': values
},
]`enter code here`
body = {
'valueInputOption': value_input_option,
'data': data
}
result = service.spreadsheets().values().batchUpdate(
spreadsheetId=spreadsheetId, body=body).execute()
if __name__ == '__main__':
main()

In Python, I use this piece of code and it work.
from googleapiclient.discovery import build
from google.oauth2 import service_account
SCOPES = ["https://www.googleapis.com/auth/drive", "https://www.googleapis.com/auth/drive.file",
"https://www.googleapis.com/auth/spreadsheets"]
SERVICE_ACCOUNT_FILE = 'credentials.json'
credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPES)
service = build('sheets', 'v4', credentials=credentials)
spreadsheetId = "O188999ITC"
sheetid_src = "123a"
notes = {
"updateCells": {
"range": {
"sheetId": sheetid_src,
"startRowIndex": 1,
"endRowIndex: 1,
"startColumnIndex": 1,
"endColumnIndex": 1
},
"rows": [
{
"values": [
{
"note": "my note"
}
]
}
],
"fields": "note"
}
}
body = {"requests":[notes]}
result = service.spreadsheets().values().batchUpdate(
spreadsheetId=spreadsheetId, body=body).execute()
Note are available in API with cellData, so you have to update values of differents cells with the keyword "Notes".
You can find more information here :
https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/cells#celldata

I don't see any documentation regarding your issue. You may refer with this SO thread however it discussed how to get a note using getNote(). In your case, you need to use Google Apps Script and use the methods:
setNote(note) - Sets the note to the given value.
setNotes(notes) - Sets a rectangular grid of notes (must match dimensions of this range).
You may also check into this related reported issue. Hope this helps!