What I'm traing to do is to get all the members of a single group, and then download all the emails headers they have in gmail. To do this I started with google gmail api and members api, but i get stuck. I realice that I can't do it even if I have admin role.
What i found is that the right way to do it (I think...) is to use a service account for impersonate user. What i done:
Create a new project.
Create service account credentials.
Add Domain-wide Delegation to the service account.
Enable scopes for this proyect.
The code to create the service:
from __future__ import print_function
from googleapiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
from google.oauth2 import service_account
import requests
import json
class Service:
SCOPES = ['https://mail.google.com/',
'https://sites.google.com/feeds',
'https://www.google.com/m8/feeds',
'https://www.googleapis.com/auth/activity',
'https://www.googleapis.com/auth/calendar',
'https://www.googleapis.com/auth/drive',
'https://www.googleapis.com/auth/gmail.settings.basic',
'https://www.googleapis.com/auth/gmail.settings.sharing',
'https://www.googleapis.com/auth/plus.login',
'https://www.googleapis.com/auth/plus.me',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/userinfo.profile',
'https://www.googleapis.com/auth/admin.directory.group',
'https://www.googleapis.com/auth/gmail.metadata'
]
secret_file = 'client-secret.json'
service = ''
service_name = ''
service_version = ''
delegated_credentials = ''
credentials = ''
def __init__(self, servicename, serviceversion):
self.service_name = servicename
self.service_verion = serviceversion
self.newService()
def newService(self):
self.credentials = service_account.Credentials.from_service_account_file(self.secret_file, scopes=self.SCOPES)
self.credentials = self.credentials.with_subject('admin#domain.com')
self.service = build(self.service_name, self.service_verion, credentials=self.credentials)
return self.service
if __name__ == '__main__':
main()
Members class:
from __future__ import print_function
from googleapiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
from service import Service
import requests
import json
class Members:
service = ''
group_name = ''
members = []
def __init__(self, groupname):
self.group_name = groupname
self.service = Service('admin', 'directory_v1')
def get_members(self):
print('Buscando los miembros del grupo')
results = self.service.service.members().list(groupKey=self.group_name).execute()
for member in results['members']:
print('Descargando datos de: ', member['email'])
self.members.append(member['id'])
return self.members
if __name__ == '__main__':
main()
app.py:
from __future__ import print_function
from googleapiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
from members import Members
from user import User
from service import Service
import requests
def main():
member = Members("group#ldomain.com")
members_list = member.get_members()
for member in members_list:
print(member)
user = User(member)
messages = user.get_user_inbox()
user.get_message_info(messages)
if __name__ == '__main__':
main()
The error I have:
"Client is unauthorized to retrieve access tokens using this method,
or client not authorized for any of the scopes requested."
Related
In the python google api documentation regarding mocks it is stated:
As you develop and test your application, it is a good idea to save actual API responses in files like books-discovery.json or books-android.json for use in testing.
Where do I get these json from? In particular the response to mock the build() command.
EDIT:
The class I want to test, calendar.py:
from google.oauth2 import service_account
from googleapiclient.discovery import build
class Calendar:
def __init__(self, credentials_file, calendar_id) -> None:
credentials = service_account.Credentials.from_service_account_file(
credentials_file, scopes=["https://www.googleapis.com/auth/calendar"]
)
self.service = build("calendar", "v3", credentials=credentials)
self._id = calendar_id
How to retrive the json response of build to save it in calendar-discovery.json?
Modified class to simplify mocking
import google_auth_httplib2
from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.http import build_http
class Calendar:
def __init__(self, credentials_file, calendar_id) -> None:
credentials = service_account.Credentials.from_service_account_file(
credentials_file, scopes=["https://www.googleapis.com/auth/calendar"]
)
http = google_auth_httplib2.AuthorizedHttp(credentials, http=build_http())
self.service = build("calendar", "v3", http=http)
self._id = calendar_id
The test:
def test_calendar_initialization(mocker):
mock = mocker.patch("meal_planner.calendar.build_http")
mock.return_value = HttpMock("calendar-discovery.json", {"status": 200})
calendar = Calendar(credentials_file="credential.json", calendar_id="id")
List all the discovery APIs: https://discovery.googleapis.com/discovery/v1/apis
Search for "name": "calendar"
Look for "discoveryRestUrl": "https://calendar-json.googleapis.com/$discovery/rest?version=v3"
Get the calendar discovery document
curl https://calendar-json.googleapis.com/$discovery/rest?version=v3 > calendar-discovery.json`
Reference:
https://developers.google.com/discovery
I am using authlib with a pretty standard OAuth2 flow.
import pickle
import json
import os.path
from requests.models import Response
from authlib.integrations.flask_client import OAuth, OAuthError
class BaseApi(object):
def __init__(self, oauth_client, config_directory, token_file=''):
self.oauth_client = oauth_client
if token_file:
self.token_file = token_file
else:
self.token_file = os.path.join(config_directory, 'token.pickle')
#property
def token(self):
try:
print ("Token: %s" % self.__token)
return self.__token
except AttributeError:
if os.path.exists(self.token_file):
with open(self.token_file, 'rb') as f:
self.__token = pickle.load(f)
print ("Pickled Token: %s" % self.token)
return self.__token
#token.setter
def token(self, new_token):
self.__token = new_token
# The authorizaiton flow sends us to the OAuth provider
# with a redirect back to our app
def login(self, state=None):
return self.oauth_client.authorize_redirect(self.auth_callback, state=state)
# Our authorized endpoint.
def authorized(self,request):
# Get the access token!
token = self.oauth_client.authorize_access_token(client_id=self.oauth_client.client_id,
client_secret=self.oauth_client.client_secret)
#if resp is None or resp.get('access_token') is None:
# return 'Access denied: error=%s resp=%s' % (
# request.args,
# resp
# )
self.token = token
This is all subclassed in a different module:
from __future__ import print_function
import json
from backend.oauth_base import BaseApi, OAuth, load_credentials_file
from urllib.parse import urlparse
from backend import app, url_for # This is the flask app
from flask import request, redirect, jsonify
import requests
import os
config_directory = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config')
class BasecampApi(BaseApi):
def __init__(self):
oauth = OAuth(app)
credentials = load_credentials_file(os.path.join(config_directory,
'credentials.json'))
oauth.register(
name='basecamp',
client_id=credentials['client_id'],
client_secret=credentials['client_secret'],
api_base_url='https://3.basecampapi.com/',
request_token_url=None,
request_token_params={'type': 'web_server'},
access_token_url='https://launchpad.37signals.com/authorization/token',
access_token_params={'type': 'web_server'},
authorize_url='https://launchpad.37signals.com/authorization/new',
authorize_params={'type': 'web_server'}
)
oauth_client = oauth.create_client('basecamp')
super().__init__(oauth_client, config_directory)
Everything here works, but I'm confused as to why I need to pass client_id and client_secret explicitly in the authorize access token.
token = self.oauth_client.authorize_access_token()
Causes the basecamp API to throw an error complaining about a lack of clien_id (and then client_secret).
This is different from the docs (which I admittedly found confusing). Is this the expected behavior?
The default token_endpoint_auth_method is client_secret_basic, which will send client credentials in HTTP Authorization header. However, according to your description, it seems that the provider needs client_id and client_secret in payload, which should be client_secret_post. Consider to add below client_kwargs into your .register:
client_kwargs = {
'token_endpoint_auth_method': 'client_secret_post',
}
I have this piece of code to extract some metrics about my YouTube channel and create a pandas dataframe from them.
import os
import google.oauth2.credentials
import google_auth_oauthlib.flow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from google_auth_oauthlib.flow import InstalledAppFlow
import json
SCOPES = ['https://www.googleapis.com/auth/yt-analytics.readonly']
API_SERVICE_NAME = 'youtubeAnalytics'
API_VERSION = 'v2'
CLIENT_SECRETS_FILE = 'client_secrets.json'
def get_service():
flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRETS_FILE, SCOPES)
credentials = flow.run_console()
return build(API_SERVICE_NAME, API_VERSION, credentials = credentials)
def execute_api_request(client_library_function, **kwargs):
response = client_library_function(
**kwargs
).execute()
with open('data.json', 'w') as fp:
json.dump(response, fp)
if __name__ == '__main__':
# Disable OAuthlib's HTTPs verification when running locally.
# *DO NOT* leave this option enabled when running in production.
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
youtubeAnalytics = get_service()
execute_api_request(
youtubeAnalytics.reports().query,
ids='channel==MINE',
startDate='2014-01-01',
endDate='2019-02-26',
metrics='averageViewDuration,views,likes,dislikes,subscribersGained,subscribersLost',
dimensions='day',
sort='day',
filters = 'country==US'
)
## Now, convert the json to dataframe
import json
import pandas as pd
with open('data.json') as json_data:
d = json.load(json_data)
colnames = [d['columnHeaders'][i]['name'] for i in range(0,len(d['columnHeaders']))]
Results = pd.DataFrame(d['rows'],columns = colnames)
Results.to_csv("Youtube_data.csv")
By running this code, a windows opens and asks me to login into youtube and then provide me the authorization code. Entering this authorization code finishes the running of above python program. However, you should repeat this authorization process each time you are running this program.
Is there anyway to bypass this repeated authorization such that this process can be automated?
You need to use the oauth2client.file.Storage class to store and retrieves the credentials object as (badly) explained here: https://developers.google.com/api-client-library/python/guide/aaa_oauth
You will need to modify your get_service function with something like this:
from oauth2client import client, file
def get_service():
flow = client.flow_from_clientsecrets(CLIENT_SECRETS_FILE, SCOPES)
storage = file.Storage(API_SERVICE_NAME + '.dat')
credentials = storage.get()
http = credentials.authorize(http=httplib2.Http())
service = build(API_SERVICE_NAME, API_VERSION, http=http)
return service
Hope this helps
We are trying to start with Google Analytics Management API by running an example script in Python from the documentation:
https://developers.google.com/analytics/devguides/config/mgmt/v3/user-management
Unfortunately, it is not working and we receive a bug like:
It was sugested to add authorization to that script, so we use the code from:
https://developers.google.com/analytics/devguides/reporting/core/v3/quickstart/service-py
like:
scope = ['https://www.googleapis.com/auth/analytics.manage.users']
key_file_location = '...'
service_account_email= '...'
# Authenticate and construct service.
service = get_service('analytics', 'v3', scope, key_file_location,
service_account_email)
But still the problem remains unsolved.
What is wrong?
Is it the authorization process? But the HelloAnalytisc.py script is working and we can authorize and get data.
Our whole script is here:
"""A simple example of Google Analytics batched user permissions."""
import argparse
from googleapiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
import httplib2
from oauth2client import client
from oauth2client import file
from oauth2client import tools
import numpy as np
import pandas as pd
import json
from googleapiclient.errors import HttpError
from googleapiclient.http import BatchHttpRequest
def call_back(request_id, response, exception):
"""Handle batched request responses."""
print (request_id)
if exception is not None:
if isinstance(exception, HttpError):
message = json.loads(exception.content)['error']['message']
print ('Request %s returned API error : %s : %s ' %
(request_id, exception.resp.status, message))
else:
print (response)
def add_users(users, permissions):
"""Adds users to every view (profile) with the given permissions.
Args:
users: A list of user email addresses.
permissions: A list of user permissions.
Note: this code assumes you have MANAGE_USERS level permissions
to each profile and an authorized Google Analytics service object.
"""
# Get the a full set of account summaries.
account_summaries = analytics.management().accountSummaries().list().execute()
# Loop through each account.
for account in account_summaries.get('items', []):
account_id = account.get('id')
# Loop through each user.
for user in users:
# Create the BatchHttpRequest object.
batch = BatchHttpRequest(callback=call_back)
# Loop through each property.
for property_summary in account.get('webProperties', []):
property_id = property_summary.get('id')
# Loop through each view (profile).
for view in property_summary.get('profiles', []):
view_id = view.get('id')
# Construct the Profile User Link.
link = analytics.management().profileUserLinks().insert(
accountId=account_id,
webPropertyId=property_id,
profileId=view_id,
body={
'permissions': {
'local': permissions
},
'userRef': {
'email': user
}
}
)
batch.add(link)
# Execute the batch request for each user.
batch.execute()
if __name__ == '__main__':
# Define the auth scopes to request.
scope = ['https://www.googleapis.com/auth/analytics.manage.users']
key_file_location = '...'
service_account_email='...'
# Construct a list of users.
emails = ['ona#gmail.com', 'emi#gmail.com', 'sue#gmail.com', 'liz#gmail.com']
# call the add_users function with the list of desired permissions.
add_users(emails, ['READ_AND_ANALYZE'])
I'm trying to upload some custom data into GA with Python. It's the first I'm doing this so I'm not sure about nothing.
I've build the following script based on the example from the doc. When running it I have the following error :
File "import.py", line 50, in <module>
daily_upload = analytics.management().uploads().uploadData(
NameError: name 'analytics' is not defined
Here is my code :
import argparse
from apiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
import httplib2
import urllib2
from oauth2client import client
from oauth2client import file
from oauth2client import tools
def get_service(api_name, api_version, scope, key_file_location,
service_account_email):
"""Get a service that communicates to a Google API.
Args:
api_name: The name of the api to connect to.
api_version: The api version to connect to.
scope: A list auth scopes to authorize for the application.
key_file_location: The path to a valid service account p12 key file.
service_account_email: The service account email address.
Returns:
A service that is connected to the specified API.
"""
credentials = ServiceAccountCredentials.from_p12_keyfile(
service_account_email, key_file_location, scopes=scope)
http = credentials.authorize(httplib2.Http())
# Build the service object.
service = build(api_name, api_version, http=http)
return service
from apiclient.http import MediaFileUpload
try:
media = MediaFileUpload('mycsv.csv',
mimetype='application/octet-stream',
resumable=False)
daily_upload = analytics.management().uploads().uploadData(
accountId='XXXXXX',
webPropertyId='XXXXXXX',
customDataSourceId='XXXXXXXXXX',
media_body=media).execute()
except TypeError, error:
# Handle errors in constructing a query.
print 'There was an error in constructing your query : %s' % error
def main():
# Define the auth scopes to request.
scope = ['https://www.googleapis.com/auth/analytics']
# Use the developer console and replace the values with your
# service account email and relative location of your key file.
service_account_email = 'XXXXXX#XXXXXX'
key_file_location = 'XXXXXXXXXX.p12'
# Authenticate and construct service.
service = get_service('analytics', 'v3', scope, key_file_location,
service_account_email)
profile = get_first_profile_id(service)
print_results(get_results(service, profile))
if __name__ == '__main__':
main()
If my code isn't clear or show some other obvious mistake different from the one I'm questioning please be comprehensive I'm learning !
Edit: I've checked in my API Manager the Analytics API is well enable
Ok. It is a simple block alignement issue. I needed to align this part :
from apiclient.http import MediaFileUpload
try:
media = MediaFileUpload('mycsv.csv',
mimetype='application/octet-stream',
resumable=False)
daily_upload = analytics.management().uploads().uploadData(
accountId='XXXXXX',
webPropertyId='XXXXXXX',
customDataSourceId='XXXXXXXXXX',
media_body=media).execute()
except TypeError, error:
# Handle errors in constructing a query.
print 'There was an error in constructing your query : %s' % error
with the first part !