For a third party API, I'm using oauth2 module, I have following code:
Which works well when I use python run.py but throws IOError permission denied when using apache +mod_wsgi configuration.
Here is the error:
ERROR: Traceback (most recent call last):
File "/home/trex/workspace/scripts/flaskapp.py", line 32, in getTwitter_acc_audiences
_, content = client.request(uri=uri, method="GET")
File "build/bdist.linux-x86_64/egg/oauth2/__init__.py", line 687, in request
connection_type=connection_type)
File "/home/trex/workspace/myenv/lib/python2.7/site-packages/httplib2-0.9.2-py2.7.egg/httplib2/__init__.py", line 1609, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/home/trex/workspace/myenv/lib/python2.7/site-packages/httplib2-0.9.2-py2.7.egg/httplib2/__init__.py", line 1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/home/trex/workspace/myenv/lib/python2.7/site-packages/httplib2-0.9.2-py2.7.egg/httplib2/__init__.py", line 1272, in _conn_request
conn.connect()
File "/home/trex/workspace/myenv/lib/python2.7/site-packages/httplib2-0.9.2-py2.7.egg/httplib2/__init__.py", line 1036, in connect
self.disable_ssl_certificate_validation, self.ca_certs)
File "/home/trex/workspace/myenv/lib/python2.7/site-packages/httplib2-0.9.2-py2.7.egg/httplib2/__init__.py", line 80, in _ssl_wrap_socket
cert_reqs=cert_reqs, ca_certs=ca_certs)
File "/usr/lib/python2.7/ssl.py", line 911, in wrap_socket
ciphers=ciphers)
File "/usr/lib/python2.7/ssl.py", line 520, in __init__
self._context.load_verify_locations(ca_certs)
IOError: [Errno 13] Permission denied
api_manager.py
import oauth2 as oauth
from settings import * #-- All global params
class APIManager(object):
""" """
#init function goes here
def get_api_content(self, consumer_key, consumer_secret, access_token, access_secret):
""" Fetch API response using Oauth2 """
try:
consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret)
token = oauth.Token(key=access_token, secret=access_secret)
client = oauth.Client(consumer, token)
uri = APIURL + APIURI.format(account_id=SOME_ACC_ID)
_, content = client.request(uri=uri, method="GET")
except Exception as e:
logerror("ERROR:", e)
content = {}
return content
flaskapp.py
from flask import Flask, jsonify
from api_manager import APIManager
from settings import *
app = Flask(__name__)
#app.route("/")
def callapi():
try:
apiobj = APIManager()
response = apiobj.get_api_content(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_SECRET)
except Exception, e:
logerror("ERROR:", e)
return jsonify({})
return jsonify(response)
Apache WSGI Configuration:
<VirtualHost _default_:443>
...
# Default other config goes here...
WSGIScriptAlias / /home/trex/workspace/scripts/appwsgi.py
WSGIScriptReloading On
<Directory "/home/trex/workspace/scripts">
<Files "appwsgi.py">
Require all granted
</Files>
</Directory>
</VirtualHost>
Let me know if anything else I can give you the details.
There was an issue with permissions to my virtualenv. Following file had permission issues.
myenv/lib/python2.7/site-packages/httplib2-0.9.2-py2.7.egg/httplib2/cacerts.txt
And this simple fix fixed my issue:
chmod -R 755 /home/trex/workspace/myenv
Related
I'm trying to access blogger API using the official Google python library and a service account with a JSON key file made at the Google Developer console.
I'm using Linux Debian.
google-api-python-client=2.65.0
The thing is that I'm getting this traceback:
Traceback (most recent call last):
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/discovery.py", line 287, in build
content = _retrieve_discovery_doc(
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/discovery.py", line 422, in _retrieve_discovery_doc
resp, content = req.execute(num_retries=num_retries)
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/http.py", line 923, in execute
resp, content = _retry_request(
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/http.py", line 191, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/path/to/myproject/lib/python3.9/site-packages/httplib2/__init__.py", line 1322, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/path/to/myproject/lib/python3.9/site-packages/httplib2/__init__.py", line 1072, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/path/to/myproject/lib/python3.9/site-packages/httplib2/__init__.py", line 996, in _conn_request
conn.request(method, request_uri, body, headers)
File "/usr/lib/python3.9/http/client.py", line 1279, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1325, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1274, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1034, in _send_output
self.send(msg)
File "/usr/lib/python3.9/http/client.py", line 995, in send
self.sock.sendall(data)
OSError: [Errno 9] Bad file descriptor
Which point to some sort of error at the OS when opening some file, but I can't imagine which is the file that is causing trouble. The credentials seem like are doing all right when recovered from the json file and at the moment of the exception is just sending this:
b'GET /discovery/v1/apis/blogger/v3/rest HTTP/1.1\r\nHost: www.googleapis.com\r\ncontent-length: 0\r\nuser-agent: Python-httplib2/0.10.3 (gzip)\r\naccept-encoding: gzip, deflate\r\n\r\n'
through the internet. This also seems to do fine from my browser.
The code is here below:
import json
import google.auth
from google.oauth2 import service_account
import googleapiclient.discovery
from oauth2client import client
from oauth2client.service_account import ServiceAccountCredentials
SCOPES = ['https://www.googleapis.com/auth/blogger']
def get_key_dict():
SERVICE_ACCOUNT_FILE = '/path/to/myproject/apis/blogger/private_secrets.json'
with open(SERVICE_ACCOUNT_FILE, 'r') as keyfile:
return json.load(keyfile)
key_dict = get_key_dict()
credentials = ServiceAccountCredentials.from_json_keyfile_dict(key_dict, scopes=SCOPES)
# Here it breaks!
service = googleapiclient.discovery.build('blogger', 'v3', credentials=credentials, static_discovery=False,)
try:
users = service.users()
# Retrieve this user's profile information
thisuser = users.get(userId="self").execute()
print("This user's display name is: %s" % thisuser["displayName"])
blogs = service.blogs()
# Retrieve the list of Blogs this user has write privileges on
thisusersblogs = blogs.listByUser(userId="self").execute()
for blog in thisusersblogs["items"]:
print("The blog named '%s' is at: %s" % (blog["name"], blog["url"]))
posts = service.posts()
# List the posts for each blog this user has
for blog in thisusersblogs["items"]:
print("The posts for %s:" % blog["name"])
request = posts.list(blogId=blog["id"])
while request != None:
posts_doc = request.execute()
if "items" in posts_doc and not (posts_doc["items"] is None):
for post in posts_doc["items"]:
print(" %s (%s)" % (post["title"], post["url"]))
request = posts.list_next(request, posts_doc)
except client.AccessTokenRefreshError:
print(
"The credentials have been revoked or expired, please re-run"
"the application to re-authorize"
)
if you check the documentation Authorizing requests and identifying your application you will find that it states that you can use Oauth2 for authorizing a user to the blogger api and api keys. There is no mention of using service account authorization with this api.
To my knowledge as i tried about five years ago this api does not support service account authentication. You will need to use Oauth2.
So I'm running my google endpoint locally with dev_appserver.py.
I use the API explorer to test the application.
The code I'm using to create the Service, so I can call the API is the following:
from apiclient.discovery import build
from oauth2client.client import GoogleCredentials
credentials = GoogleCredentials.get_application_default()
service = build('speech', 'v1beta1', credentials=credentials)
I receive an SSL error (Invalid and/or missing SSL certificate), even though when I access the stated URL via browser it works fine (that is, the green padlock shows up).
I'm not sure what changed, but this was working fine not long ago.
I tried to disable SSL checking, but was unable to.
Full logs below:
INFO 2017-01-02 03:12:02,724 discovery.py:267] URL being requested: GET https://www.googleapis.com/discovery/v1/apis/speech/v1beta1/rest?userIp=0.2.0.3
ERROR 2017-01-02 03:12:03,022 wsgi.py:263]
Traceback (most recent call last):
File "/home/vini/opt/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
File "/home/vini/opt/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
handler, path, err = LoadObject(self._handler)
File "/home/vini/opt/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
obj = __import__(path[0])
File "/mnt/b117/home/vini/udacity/cerci-endpoint/api.py", line 28, in <module>
service = build('speech', 'v1beta1', credentials=credentials)
File "/mnt/b117/home/vini/udacity/cerci-endpoint/lib/oauth2client/_helpers.py", line 133, in positional_wrapper
return wrapped(*args, **kwargs)
File "/mnt/b117/home/vini/udacity/cerci-endpoint/lib/googleapiclient/discovery.py", line 222, in build
cache)
File "/mnt/b117/home/vini/udacity/cerci-endpoint/lib/googleapiclient/discovery.py", line 269, in _retrieve_discovery_doc
resp, content = http.request(actual_url)
File "/mnt/b117/home/vini/udacity/cerci-endpoint/lib/httplib2/__init__.py", line 1609, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/mnt/b117/home/vini/udacity/cerci-endpoint/lib/httplib2/__init__.py", line 1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/mnt/b117/home/vini/udacity/cerci-endpoint/lib/httplib2/__init__.py", line 1307, in _conn_request
response = conn.getresponse()
File "/home/vini/opt/google-cloud-sdk/platform/google_appengine/google/appengine/dist27/gae_override/httplib.py", line 532, in getresponse
raise HTTPException(str(e))
HTTPException: Invalid and/or missing SSL certificate for URL: https://www.googleapis.com/discovery/v1/apis/speech/v1beta1/rest?userIp=0.2.0.3
Any ideas what could be causing this problem?
Do I have to "install" or update the SSL certificates used by python?
According to App Engine issue 13477 it seems that some of the certificates found in urlfetch_cacerts.txt that is included in the App Engine Python SDK / gcloud-sdk expired 2017-01-01.
As a temporary workaround, you can replace the contents of <your-cloud-sdk-path>/platform/google_appengine/lib/cacerts/urlfetch_cacerts.txt with https://curl.haxx.se/ca/cacert.pem
To build on the answer by #danielx for those on macOS, this is what worked for me. The path to the certificates for me was:
/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/cacerts/urlfetch_cacerts.txt
To update it, I used the following steps:
cd /Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/lib/cacerts
mv urlfetch_cacerts.txt urlfetch_cacerts.bup
curl -o urlfetch_cacerts.txt -k https://curl.haxx.se/ca/cacert.pem
If you don't have curl installed, you can manually download the certificates an move them to the folder above.
Don't forget to restart the App Engine dev server if it is already running.
Got this error on local dev environment as recently as Aug 2017. The fix is to update all urlfetch calls and force validation of the certs:
urlfetch.fetch(url=url, validate_certificate=True)
Did not have to touch the gcloud certs (MacOS). See Issuing an HTTPS request.
I'm trying to send an SMS message via Twilio from some Python code running in a Docker container. The container was built from an Ubuntu image with Python 2.7 and the latest twilio-python helper library. When calling the TwilioRestClient messages.create() function, I get the traceback below. Also, I have confirmed via curl that I can reach the url "api.twilio.com" as indicated in the traceback from this container. Any ideas?
Code being used...
import argparse
import requests
import time
import smtplib
from datetime import datetime
from twilio.rest import TwilioRestClient
def sendText( ToNumber, smsMsg):
# put your own credentials here
ACCOUNT_SID = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
AUTH_TOKEN = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
FromNumber = '+nnnnnnnnnnn'
client = TwilioRestClient(ACCOUNT_SID, AUTH_TOKEN)
client.messages.create(to = ToNumber,from_ = FromNumber,body = smsMsg,)
return True
Traceback...
File "./siteminder3.py", line 15, in sendText
message = client.messages.create(to=ToNumber,from_=FromNumber,body=smsMsg)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/messages.py", line 122, in create
return self.create_instance(kwargs)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 365, in create_instance
data=transform_params(body))
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 200, in request
resp = make_twilio_request(method, uri, auth=self.auth, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 152, in make_twilio_request
resp = make_request(method, uri, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 117, in make_request
resp, content = http.request(url, method, headers=headers, body=data)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1609, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1278, in _conn_request
raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
httplib2.ServerNotFoundError: Unable to find the server at api.twilio.com
So I am coding this little simple script that I know will not work, but the script itself is running without errors and making a call to www.googleapis.com, so something is happening there. Without seeing the HTTP response I can't know what is wrong, though. The evidence of it actually doing something is because I am in china, and when I run the interpreter there is a time out until I break the script manually :
OUTPUT WITHOUT PROXY :
File "youtubeconnect.py", line 30, in <module>
execute()
File "youtubeconnect.py", line 27, in execute
youtube = authenticate()
File "youtubeconnect.py", line 15, in authenticate
service = build('youtube', 'v3',http=http)
File "/home/xavier/Code/autotube/venv/lib/python3.5/site-packages/oauth2client-1.5.2-py3.5.egg/oauth2client/util.py", line 140, in positional_wrapper
File "/home/xavier/Code/autotube/venv/lib/python3.5/site-packages/google_api_python_client-1.4.2-py3.5.egg/googleapiclient/discovery.py", line 196, in build
File "/home/xavier/Code/autotube/venv/lib/python3.5/site-packages/google_api_python_client-1.4.2-py3.5.egg/googleapiclient/discovery.py", line 242, in _retrieve_discovery_doc
File "/home/xavier/Code/autotube/venv/lib/python3.5/site-packages/oauth2client-1.5.2-py3.5.egg/oauth2client/client.py", line 596, in new_request
File "/home/xavier/Code/autotube/venv/lib/python3.5/site-packages/httplib2-0.9.2-py3.5.egg/httplib2/__init__.py", line 1314, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/home/xavier/Code/autotube/venv/lib/python3.5/site-packages/httplib2-0.9.2-py3.5.egg/httplib2/__init__.py", line 1064, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/home/xavier/Code/autotube/venv/lib/python3.5/site-packages/httplib2-0.9.2-py3.5.egg/httplib2/__init__.py", line 987, in _conn_request
conn.connect()
File "/usr/lib/python3.5/http/client.py", line 1229, in connect
super().connect()
File "/usr/lib/python3.5/http/client.py", line 826, in connect
(self.host,self.port), self.timeout, self.source_address)
File "/usr/lib/python3.5/socket.py", line 702, in create_connection
sock.connect(sa)
Now, when I run the interpreter through proxychains and tunnel it it works :
ProxyChains-3.1 (http://proxychains.sf.net)
|DNS-request| www.googleapis.com
|S-chain|-<>-127.0.0.1:1080-<><>-4.2.2.2:53-<><>-OK
|DNS-response| www.googleapis.com is 173.194.208.95
|S-chain|-<>-127.0.0.1:1080-<><>-173.194.208.95:443-<><>-OK
I am new to python, how can get the output of the HTTP error that google is sending me back when I try to connect erroneously to the API ? My code :
import httplib2
import os
from oauth2client import tools
from oauth2client.client import AccessTokenCredentials
#from oauth2client.client import AccessTokenRefreshError
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
import urllib
def authenticate():
#scope = "https://www.googleapis.com/auth/youtube.upload"
acc_token = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
flow = AccessTokenCredentials(
acc_token,'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36')
http = flow.authorize(httplib2.Http())
service = build('youtube', 'v3',http=http)
return(service)
def initialize_upload(youtube):
tags = None
vid = youtube.videos().insert
media_body = MediaFileUpload('1977.mp4', mimetype='video/mp4',
chunksize=1024 * 1024, resumable=True)
def execute():
youtube = authenticate()
initialize_upload(youtube)
execute()
Use logging: https://developers.google.com/api-client-library/python/guide/logging
Also, you can use Fiddler or Wireshark to analyze the packets being sent to you.
I am using "bigquery_service = build('bigquery', 'v2', http=http)" and it is executing correctly in my laptop, but when I am executing this query on the server I am getting the following error.
File "/usr/lib64/python2.6/threading.py", line 532, in __bootstrap_inner
self.run()
File "/usr/lib64/python2.6/threading.py", line 484, in run
self.__target(*self.__args, **self.__kwargs)
File "QueryAPI.py", line 117, in localMain
exportDataToGCS(canonicalDate);
File "QueryAPI.py", line 177, in exportDataToGCS
bigquery_service = build('bigquery', 'v2', http=http)
File "/homeBigQuery/src/oauth2client/util.py", line 132, in positional_wrapper
return wrapped(*args, **kwargs)
File "/homeBigQuery/src/googleapiclient/discovery.py", line 198, in build
resp, content = http.request(requested_url)
File "/homeBigQuery/src/oauth2client/util.py", line 132, in positional_wrapper
return wrapped(*args, **kwargs)
File "/homeBigQuery/src/oauth2client/client.py", line 538, in new_request
redirections, connection_type)
File "/homeBigQuery/src/httplib2/__init__.py", line 1570, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/homeBigQuery/src/httplib2/__init__.py", line 1317, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/homeBigQuery/src/httplib2/__init__.py", line 1286, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python2.6/httplib.py", line 980, in getresponse
raise ResponseNotReady()
You could try this 3rd party library for BigQuery. It's a thin wrapper around the Google BigQuery API.
To login to BigQuery:
import bigquery as bq
with open('my_key.pem', 'r') as key:
key_content = key.read()
bq_client = bq.get_client(project_id='foo',
service_account='bar',
private_key=key_content)
To submit a load job:
schema = [{'name':'foo', 'type':'STRING'}]
job = bq_client.import_data_from_uris(source_uris=['gs://foo/bar'],
dataset='foo',
table='bar',
schema=schema,
source_format=bq.client.JOB_FORMAT_CSV,
write_disposition=bq.client.JOB_WRITE_TRUNCATE,
field_delimiter=',',
skip_leading_rows=1)
try:
job_resource = bq_client.wait_for_job(job, timeout=60)
except bq.errors.BigQueryTimeoutException:
logging.critical('BigQuery loading timeout.')
Have you tried removing the "http=http" parameter ?
here is an example of the use of the BQ API:
import httplib2
from apiclient.discovery import build
from oauth2client.client import SignedJwtAssertionCredentials
# REPLACE WITH YOUR Project ID
PROJECT_NUMBER = 'XXXXXXXXXXX'
# REPLACE WITH THE SERVICE ACCOUNT EMAIL FROM GOOGLE DEV CONSOLE
SERVICE_ACCOUNT_EMAIL = 'XXXXX#developer.gserviceaccount.com'
# OBTAIN THE KEY FROM THE GOOGLE APIs CONSOLE
# More instructions here: http://goo.gl/w0YA0
f = file('key.p12', 'rb')
key = f.read()
f.close()
credentials = SignedJwtAssertionCredentials(
SERVICE_ACCOUNT_EMAIL,
key,
scope='https://www.googleapis.com/auth/bigquery')
http = httplib2.Http()
http = credentials.authorize(http)
service = build('bigquery', 'v2')
datasets = service.datasets()
response = datasets.list(projectId=PROJECT_NUMBER).execute(http)
print 'Dataset list:'
for dataset in response['datasets']:
print '%s' % dataset['datasetReference']['datasetId']
Solved this by specifying the proxy server host and port for the server that I was running this on. While initiating an authorized http service object with the help of credentials, I modified the code as follows:
BEFORE:
http = httplib2.Http()
http = credentials.authorize(http)
service = build('bigquery', 'v2', http=http)
FIX:
http = httplib2.Http(proxy_info = httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, 'your-proxy-host-name', your-proxy-port))
http = credentials.authorize(http)
service = build('bigquery', 'v2', http=http)