We Keep Coding

Google Python SDK : google.auth.exceptions.RefreshError: ('No access token in response.',

I am trying to make basic calls for gsuite, ala listing groups. I have poked around quite a bit and have seen several posts suggesting this is a scope issue. However, I have included as many scope paths as I can find that even looked remotely relevant. I still get an exception in the library because there is no "access_token" in the response. There IS an id_token, but nothing labelled "access_token", so the lib throws an exception.
Also, I am pretty sure the account has access to the "group settings API", and many other things.
Banging my head on the desk...
google-api-core==2.10.1
google-api-python-client==2.64.0
google-auth==2.12.0
google-auth-httplib2==0.1.0
google-auth-oauthlib==0.5.3
googleapis-common-protos==1.56.4
from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from google.oauth2 import service_account
SCOPES = ['https://www.googleapis.com/auth/sqlservice.admin',
'https://www.googleapis.com/auth/admin.directory.user',
"https://www.googleapis.com/auth/apps.groups.settings",
'https://www.googleapis.com/auth/admin',
'https://www.googleapis.com/auth/apps.groups.settings',
'https://www.googleapis.com/auth/indexing',
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/contacts',
'https://www.googleapis.com/auth/contacts.other.readonly'
'https://www.googleapis.com/auth/contacts.readonly',
'https://www.googleapis.com/auth/directory.readonly',
'https://www.googleapis.com/auth/user.addresses.read',
'https://www.googleapis.com/auth/user.birthday.read',
'https://www.googleapis.com/auth/user.emails.read',
'https://www.googleapis.com/auth/user.gender.read',
'https://www.googleapis.com/auth/user.organization.read',
'https://www.googleapis.com/auth/user.phonenumbers.read',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/userinfo.profile',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/admin',
'https://www.googleapis.com/auth/service.management.readonly',
'https://www.googleapis.com/auth/monitoring',
'https://www.googleapis.com/auth/admin.reports.audit.readonly',
'https://www.googleapis.com/auth/admin.reports.usage.readonly',
'https://www.googleapis.com/auth/admin.datatransfer',
'https://www.googleapis.com/auth/admin.datatransfer.readonly',
'https://www.googleapis.com/auth/admin.directory.customer',
'https://www.googleapis.com/auth/admin.directory.customer.readonly',
'https://www.googleapis.com/auth/admin.directory.domain',
'https://www.googleapis.com/auth/admin.directory.domain.readonly',
'https://www.googleapis.com/auth/admin.directory.group',
'https://www.googleapis.com/auth/admin.directory.group.member',
'https://www.googleapis.com/auth/admin.directory.group.member.readonly',
'https://www.googleapis.com/auth/admin.directory.group.readonly',
'https://www.googleapis.com/auth/admin.directory.orgunit',
'https://www.googleapis.com/auth/admin.directory.orgunit.readonly',
'https://www.googleapis.com/auth/admin.directory.resource.calendar',
'https://www.googleapis.com/auth/admin.directory.resource.calendar.readonly',
'https://www.googleapis.com/auth/admin.directory.rolemanagement',
'https://www.googleapis.com/auth/admin.directory.rolemanagement.readonly',
'https://www.googleapis.com/auth/admin.directory.user',
'https://www.googleapis.com/auth/admin.directory.user.alias',
'https://www.googleapis.com/auth/admin.directory.user.alias.readonly',
'https://www.googleapis.com/auth/admin.directory.user.readonly',
'https://www.googleapis.com/auth/admin.directory.user.security',
'https://www.googleapis.com/auth/admin.directory.userschema',
'https://www.googleapis.com/auth/admin.directory.userschema.readonly',
'https://www.googleapis.com/auth/groups',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/cloud-platform.read-only',
'https://www.googleapis.com/auth/apps.groups.migration']
SERVICE_ACCOUNT_FILE = 'privkey.json'
def main():
credentials = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE, scopes=SCOPES)
service = build('admin', 'directory_v1', credentials=credentials)
# this is where the library throws the exception
foobar = service.groups().list().execute()
if __name__ == '__main__':
main()
This is a partial stack trace:
Traceback (most recent call last):
File "/XXXXXXXXXXXXXX/.local/lib/python3.7/site-packages/google/oauth2/_client.py", line 294, in jwt_grant
access_token = response_data["access_token"]
KeyError: 'access_token'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "../gsuite_auth_test6.py", line 93, in <module>
main()
File "../gsuite_auth_test6.py", line 90, in main
foobar = service.groups().list().execute()
File "/XXXXXXXXXXXXXX/.local/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/XXXXXXXXXX/.local/lib/python3.7/site-packages/googleapiclient/http.py", line 932, in execute
headers=self.headers,
File "/XXXXXXXXXXXXX/.local/lib/python3.7/site-packages/googleapiclient/http.py", line 191, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/XXXXXXXXXXXXXX/.local/lib/python3.7/site-packages/google_auth_httplib2.py", line 209, in request
self.credentials.before_request(self._request, method, uri, request_headers)
File "/XXXXXXXXXXXXXX/.local/lib/python3.7/site-packages/google/auth/credentials.py", line 133, in before_request
self.refresh(request)
File "/XXXXXXXXXXXXXX/.local/lib/python3.7/site-packages/google/oauth2/service_account.py", line 411, in refresh
request, self._token_uri, assertion
File "/XXXXXXXXXXXXXX/.local/lib/python3.7/site-packages/google/oauth2/_client.py", line 299, in jwt_grant
six.raise_from(new_exc, caught_exc)
File "<string>", line 3, in raise_from
google.auth.exceptions.RefreshError: ('No access token in response.', {'id_token': '....'

Your research is right in that this can be caused by wrong scopes but there are more issues here that I will try to address.
I have included as many scope paths as I can find that even looked remotely relevant.
This is a problem. You cannot just throw all scopes to the wall and see what sticks. First you need to have enabled the relevant APIs on your GCP project. In this case you don't actually need the Groups Settings API, instead you need the Directory API, also known as Admin SDK API:
Admin SDK lets administrators of enterprise domains to view and manage resources like user, groups etc. It also provides audit and usage reports of domain.
Next, to know the scopes you can refer to Google's documentation. The groups.list method has the scopes that you need to use to list all Groups in the domain. In this case you can just use https://www.googleapis.com/auth/admin.directory.group.
Also, in the groups.list documentation I linked above you can see that you need to specify either the parameter customer for all groups within the Google Workspace account or domain for just one of its subdomains. You can also just use the my_customer alias to search everything within the account. The Python documentation here has its equivalent for the Python library. All of this means that your line to call groups.list should specify the customer like this:
foobar = service.groups().list(customer="my_customer").execute()
Finally, you seem to be using a service account's credentials. This means that you need to first enable domain-wide delegation for your service account so it can impersonate your user accounts, next, you have to impersonate an admin that already has the privileges to manage groups in the Admin Console. You can refer to the Google OAuth docs to check out how to do this, and here's also a Python sample. Pretty much you can just create a copy of the credentials while specifying the admin account, like this:
delegated_credentials=credentials.with_subject("admin#yourdomain.com")
Here's a sample based on your code that worked for me after taking all of the above into account:
from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from google.oauth2 import service_account
SCOPES = ['https://www.googleapis.com/auth/admin.directory.group']
SERVICE_ACCOUNT_FILE = 'privkey.json'
def main():
credentials = service_account.Credentials.from_service_account_file(
SERVICE_ACCOUNT_FILE, scopes=SCOPES)
#delegate the admin credentials to the service account
delegated_credentials=credentials.with_subject("admin#yourdomain.com")
service = build('admin', 'directory_v1', credentials=delegated_credentials)
# make sure to add the customer parameter per the documentation requirements
foobar = service.groups().list(customer="my_customer").execute()
print(foobar)
if __name__ == '__main__':
main()
As you can see, aside from removing the unnecessary scopes I just changed two lines. First make sure that your GCP and service account configurations are in order and then don't forget to delegate access and specify the customer variable.

Google Drive API for Python: how to create credential?

I was writing a Python script to automate uploading some files to Google Drive. Since I'm still a newbie Python programmer and this is an exercise as much as anything else, I started following the Google Quickstart and decided to use their quickstart.py as a basis on which to base my own script. In the part where it talks about how to create credentials for your Python script, it refers to the "Create credentials" link, at https://developers.google.com/workspace/guides/create-credentials
I follow the link, get into one of my Google Cloud projects, and try to set up the OAuth consent screen, using an "Internal" project, as they tell you... but I can't. Google says:
“Because you’re not a Google Workspace user, you can only make your
app available to external (general audience) users. ”
So I try to create an "External" project, and then proceed to create a new client ID, using a Desktop application. Then I download the JSON credentials and put them in the same folder as my Python script, as "credentials.json". I then execute the Python script in order to authenticate it: the browser opens, I log into my Google account, give it my permissions... and then the browser hangs, because it's redirecting to a localhost URL and obviously my little Python script isn't listening in my computer at all.
I believe they must have changed this recently, because a year ago I started following the same Python tutorial and could create credentials without problems, but the Google Drive API docs haven't been updated yet. So... how do I create credentials for a Python script now?
EDIT: adding here the source code for my script. As I said, it's very similar to Google's "quickstart.py":
from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.errors import HttpError
# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/drive.metadata', 'https://www.googleapis.com/auth/drive']
def main():
"""Shows basic usage of the Drive v3 API.
Prints the names and ids of the first 10 files the user has access to.
"""
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token_myappname.pickle'):
with open('token_myappname.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token_myappname.pickle', 'wb') as token:
pickle.dump(creds, token)
service = build('drive', 'v3', credentials=creds)
# Call the Drive v3 API
results = service.files().list(
pageSize=10, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
if not items:
print('No files found.')
else:
#print(items[0])
print('Files:')
for item in items:
#print (item)
print(u'{0} {1} {2}'.format(item['name'], item['owners'], item['parents']))

I propose you to use a service account to access to the Drive.
For that, you need to share the drive (or the folder) with the service account email. And then use this code
from googleapiclient.discovery import build
import google.auth
SCOPES = ['https://www.googleapis.com/auth/drive.metadata', 'https://www.googleapis.com/auth/drive']
def main():
credentials, project_id = google.auth.default(scopes=SCOPES)
service = build('drive', 'v3', credentials=credentials)
# Call the Drive v3 API
results = service.files().list(
q=f"'1YJ6gMgACOqVVbcgKviJKtVa5ITgsI1yP' in parents",
pageSize=10, fields="nextPageToken, files(id, name, owners, parents)").execute()
items = results.get('files', [])
if not items:
print('No files found.')
else:
#print(items[0])
print('Files:')
for item in items:
#print (item)
print(u'{0} {1} {2}'.format(item['name'], item['owners'], item['parents']))
If you run your code on Google Cloud, in a compute engine instance for example, you need to customize the VM with the service account that you authorized in your drive. (Don't use the compute engine default service account, else you will need extra configuration on your VM)
If you run your script outside GCP, you need to generate a service account key file and to store it on your local server. Then, create an environment variable GOOGLE_APPLICATION_CREDENTIALS that reference the full path of the stored key file.

Aside from the other solution posted by Guillaume Blaquiere, I also found another one on my own, which I wanted to post here in case it's helpful. All I had to do is to... erm, actually read the code I was copying and pasting, in particular this line:
creds = flow.run_local_server(port=0)
I checked Google's documentation outside of the Quickstart and found in the following: https://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html
It turns out, the example code was opening a local port in my computer to listen to the request, and it wasn't working probably due to the "port 0" part, or some other network problem.
So the workaround I found was to use a different auth method found in the docs:
creds = flow.run_console()
In this case, you paste manually in the command line the auth code given to you by Google. I just tried it, and have my credentials happily stored in my local pickle file.

Google OAuth client is using the wrong project_id from the json file- Python

My Python (3.6.7) code uses oauth2client to access Google Photos APIs. It successfully authenticates, but when it tries to access the Google Photos albums, it seems to be using the username as the project_id.
from __future__ import print_function
from apiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
# Setup the Photo v1 API
SCOPES = 'https://www.googleapis.com/auth/photoslibrary.readonly'
store = file.Storage('credentials.json')
creds = store.get()
if not creds or creds.invalid:
flow = client.flow_from_clientsecrets('scripts/client_id.json', SCOPES)
creds = tools.run_flow(flow, store)
service = build('photoslibrary', 'v1', http=creds.authorize(Http()))
# Call the Photo v1 API
results = service.albums().list(
pageSize=10, fields="nextPageToken,albums(id,title)").execute()
items = results.get('albums', [])
if not items:
print('No albums found.')
else:
print('Albums:')
for item in items:
print('{0} ({1})'.format(item['title'].encode('utf8'), item['id']))
When executing the above code, it prompts me the auth page. When I successfully authenticate, it shows me the following error:
HttpError 403 when requesting {URL} returned "Photos Library API has not been used in project 123456 before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/photoslibrary.googleapis.com/overview?project=123456 then retry. If you enabled this API recently, wait a few minutes for the action to propagate to our systems and retry.">
Interestingly, the number in bold 123456 (obviously changed) is actually the first part of the client_id found in the client_id.json
But the project_id looks something like this: test1-235515
So what I got from this error is that the oauth2client client is passing the client_id instead of the project_id. So even though I have enabled the Photos API, it will never access it correctly.
Please help with this error. How can I manually change the project_id?

The project ID is different from the project number. You will be able to see both in your Google Cloud Console configuration. See this documentation for more on how to identify your projects [1].
A single Google Cloud project can have many different OAuth client IDs configured. See this documentation for information about creating OAuth client credentials [2]. You should be only have to make sure that the client you created belongs to the project for which you have enabled APIs. Going to the URL provided in the error message should take you to the right configuration page.
[1] https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects
[2] https://support.google.com/cloud/answer/6158849?hl=en

Google API quickstart.py error KeyError: '_module'

Using Gmail API. My client secret file is downloaded and working for Ruby. When I try the quickstart.py (python) version I get this error
File "quickstart.py", line 70, in <module>
main()
File "quickstart.py", line 55, in main
credentials = get_credentials()
File "quickstart.py", line 38, in get_credentials
credentials = store.get()
File "/Library/Python/2.7/site-packages/oauth2client/client.py", line 374, in get
return self.locked_get()
File "/Library/Python/2.7/site-packages/oauth2client/file.py", line 79, in locked_get
credentials = Credentials.new_from_json(content)
File "/Library/Python/2.7/site-packages/oauth2client/client.py", line 281, in new_from_json
module = data['_module']
KeyError: '_module'
I have not changed the file at all, just added the client_secret.json to that working directory and also install the google-api-python-client. My python code came from here: https://developers.google.com/gmail/api/quickstart/python

Try replacing creds = store.get() with creds = None temporarily. If this works, you can refactor your code to always start with flow-based credentials instantiation. This worked for me. It seems Google samples are out of sync with their oauth2client.

oauth2client is trying to load credentials from a json file with an incorrect structure.
Maybe the Ruby client uses a different file format but I'd be surprised. Are you sure you didn't save client_secret.json as ~/.credentials/gmail-quickstart.json accidentally?
Regardless, removing ~/.credentials/gmail-quickstart.json and re-authenticating will generate a new credentials file with the correct structure.

I am learning Python myself and had a similar problem, but with the Calendar API example. It turned out that it was a typo with regards to the SCOPE.
## Typo - Invalid definition
SCOPES = 'https://ww.googleapies.com/auth/calendar.readonly'
## Correct Value for SCOPE
SCOPES = 'https://www.googleapis.com/auth/calendar'
Also, Matt's answer help point me in the right direction. The gmail-quickstart.json is not the same thing as the client_secret.json. The client_secret.json allows you to make a request for an OAuth2 token. While the gmail-quickstart.json contains the issued token and meta-data associated with it. The gmail-quickstart.json isn't created until you successfully login.
One last thought, in order to log in successfully, the quickstart.py app launched an instance of my web-browser (Firefox) and went to the Google login screen. In order for Firefox to run properly, I had to set my DISPLAY variable properly first.
$ export DISPLAY=:0
$ xhost +
access control disabled, clients can connect from any host

In this GitHub issue: error "KeyError: '_module'" when running gdrive_upload.py
sputnik-dev answared on 10 Jan 2016:
If someone have the same issue : auth_token.txt and client_secret.json are not the same! Don't link the client_secret.json from google API console. The file will be automatically created by the script.
Wrong way: gauth.SaveCredentialsFile("client_secret.json")
Right way: gauth.SaveCredentialsFile("<any random name>.json")
Extra:
PyDrive code that automate google drive api authetication. Use the browser just one time to authenticate and never more. It saves your credential data on mycreds.json :)
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
gauth = GoogleAuth()
gauth.LoadCredentialsFile("mycreds.json")
if gauth.credentials is None:
gauth.LocalWebserverAuth()
elif gauth.access_token_expired:
gauth.Refresh()
else:
gauth.Authorize()
gauth.SaveCredentialsFile("mycreds.json")

I solved this by moving the client_secret.json to the same directory as the py file that is trying to read it (quickstart.py), mine was on the Desktop while i had saved the json to Documents. I saved the json to the Desktop too and boy, It flew!.
I dont know why it doesnt work when they are in different directories, defining a custom credential_path doesn't help.

I followed several recommendations listed here:
moving the client_secret.json to the same directory as the py file that is trying to read it, as recommended by simic0de.
Working with skirill idea, instead of completely eliminating the 'credentials=store.get()', I decided to handle the exception, so it would work with flow-based credentials instantiation.
The final code is:
try:
credentials = store.get()
except:
print('Working with flow-based credentials instantiation')
You can write your own code in the except line. This approach will allow the store.get() command to work when conditions are met.

Google Api Auth Http Module Error

I'm working on my first app ever to use Google Api for Calendar. I've read the Google examples at: https://developers.google.com/google-apps/calendar/instantiate
The first time I ran the program below it was successful. I allowed my app to access my Google account and the application made a calendar.dat file with the auth info in my app-directory. After I renamed the filed the code was in the auth stopped working. I have already deleted the file entirely and recreated it from scratch, but the error persists.
I do still get the Google authentication page and can still confirm access, after which I get a message that the authentication flow was completed.
This is the code (standard Google example which I fill in with my app details):
import gflags
import httplib2
from apiclient.discovery import build
from oauth2client.file import Storage
from oauth2client.client import OAuth2WebServerFlow
from oauth2client.tools import run
FLAGS = gflags.FLAGS
# Set up a Flow object to be used if we need to authenticate. This
# sample uses OAuth 2.0, and we set up the OAuth2WebServerFlow with
# the information it needs to authenticate. Note that it is called
# the Web Server Flow, but it can also handle the flow for native
# applications
# The client_id and client_secret are copied from the API Access tab on
# the Google APIs Console
FLOW = OAuth2WebServerFlow(
client_id='YOUR_CLIENT_ID',
client_secret='YOUR_CLIENT_SECRET',
scope='https://www.googleapis.com/auth/calendar',
user_agent='YOUR_APPLICATION_NAME/YOUR_APPLICATION_VERSION')
# To disable the local server feature, uncomment the following line:
# FLAGS.auth_local_webserver = False
# If the Credentials don't exist or are invalid, run through the native client
# flow. The Storage object will ensure that if successful the good
# Credentials will get written back to a file.
storage = Storage('calendar.dat')
credentials = storage.get()
if credentials is None or credentials.invalid == True:
credentials = run(FLOW, storage)
# Create an httplib2.Http object to handle our HTTP requests and authorize it
# with our good Credentials.
http = httplib2.Http()
http = credentials.authorize(http)
# Build a service object for interacting with the API. Visit
# the Google APIs Console
# to get a developerKey for your own application.
service = build(serviceName='calendar', version='v3', http=http,
developerKey='YOUR_DEVELOPER_KEY')
And this is the output:
Your browser has been opened to visit:
https://accounts.google.com/o/oauth2/auth? (auth url shortened)
If your browser is on a different machine then exit and re-run this
application with the command-line parameter
--noauth_local_webserver
Traceback (most recent call last):
File "C:\Users\Desktop\Google Drive\Code\Python\Rooster\calendar.py", line 2, in <module>
import httplib2
File "C:\Python27\lib\site-packages\httplib2-0.7.6-py2.7.egg\httplib2\__init__.py", line 42, in <module>
import calendar
File "C:\Users\Desktop\Google Drive\Code\Python\Rooster\calendar.py", line 33, in <module>
credentials = run(FLOW, storage)
File "C:\Python27\lib\site-packages\google_api_python_client-1.0-py2.7.egg\oauth2client\util.py", line 120, in positional_wrapper
return wrapped(*args, **kwargs)
File "C:\Python27\lib\site-packages\google_api_python_client-1.0-py2.7.egg\oauth2client\tools.py", line 169, in run
credential = flow.step2_exchange(code, http=http)
File "C:\Python27\lib\site-packages\google_api_python_client-1.0-py2.7.egg\oauth2client\util.py", line 120, in positional_wrapper
return wrapped(*args, **kwargs)
File "C:\Python27\lib\site-packages\google_api_python_client-1.0-py2.7.egg\oauth2client\client.py", line 1128, in step2_exchange
http = httplib2.Http()
AttributeError: 'module' object has no attribute 'Http'

The problem is that in your run directory you have the file named calendar.py. When Google's httplib2 wants to import a standard calendar module it gets the local one instead. In the local one it executes it to perform the import. But because httplib2 is not yet fully imported the calendar.py code is not working properly.
Just rename the calendar.py to something like myCalendar.py.