We Keep Coding

Several error working with blogger/google API using python and service account auth

I'm trying to access blogger API using the official Google python library and a service account with a JSON key file made at the Google Developer console.
I'm using Linux Debian.
google-api-python-client=2.65.0
The thing is that I'm getting this traceback:
Traceback (most recent call last):
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/discovery.py", line 287, in build
content = _retrieve_discovery_doc(
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/discovery.py", line 422, in _retrieve_discovery_doc
resp, content = req.execute(num_retries=num_retries)
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/http.py", line 923, in execute
resp, content = _retry_request(
File "/path/to/myproject/lib/python3.9/site-packages/googleapiclient/http.py", line 191, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/path/to/myproject/lib/python3.9/site-packages/httplib2/__init__.py", line 1322, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/path/to/myproject/lib/python3.9/site-packages/httplib2/__init__.py", line 1072, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/path/to/myproject/lib/python3.9/site-packages/httplib2/__init__.py", line 996, in _conn_request
conn.request(method, request_uri, body, headers)
File "/usr/lib/python3.9/http/client.py", line 1279, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1325, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1274, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1034, in _send_output
self.send(msg)
File "/usr/lib/python3.9/http/client.py", line 995, in send
self.sock.sendall(data)
OSError: [Errno 9] Bad file descriptor
Which point to some sort of error at the OS when opening some file, but I can't imagine which is the file that is causing trouble. The credentials seem like are doing all right when recovered from the json file and at the moment of the exception is just sending this:
b'GET /discovery/v1/apis/blogger/v3/rest HTTP/1.1\r\nHost: www.googleapis.com\r\ncontent-length: 0\r\nuser-agent: Python-httplib2/0.10.3 (gzip)\r\naccept-encoding: gzip, deflate\r\n\r\n'
through the internet. This also seems to do fine from my browser.
The code is here below:
import json
import google.auth
from google.oauth2 import service_account
import googleapiclient.discovery
from oauth2client import client
from oauth2client.service_account import ServiceAccountCredentials
SCOPES = ['https://www.googleapis.com/auth/blogger']
def get_key_dict():
SERVICE_ACCOUNT_FILE = '/path/to/myproject/apis/blogger/private_secrets.json'
with open(SERVICE_ACCOUNT_FILE, 'r') as keyfile:
return json.load(keyfile)
key_dict = get_key_dict()
credentials = ServiceAccountCredentials.from_json_keyfile_dict(key_dict, scopes=SCOPES)
# Here it breaks!
service = googleapiclient.discovery.build('blogger', 'v3', credentials=credentials, static_discovery=False,)
try:
users = service.users()
# Retrieve this user's profile information
thisuser = users.get(userId="self").execute()
print("This user's display name is: %s" % thisuser["displayName"])
blogs = service.blogs()
# Retrieve the list of Blogs this user has write privileges on
thisusersblogs = blogs.listByUser(userId="self").execute()
for blog in thisusersblogs["items"]:
print("The blog named '%s' is at: %s" % (blog["name"], blog["url"]))
posts = service.posts()
# List the posts for each blog this user has
for blog in thisusersblogs["items"]:
print("The posts for %s:" % blog["name"])
request = posts.list(blogId=blog["id"])
while request != None:
posts_doc = request.execute()
if "items" in posts_doc and not (posts_doc["items"] is None):
for post in posts_doc["items"]:
print(" %s (%s)" % (post["title"], post["url"]))
request = posts.list_next(request, posts_doc)
except client.AccessTokenRefreshError:
print(
"The credentials have been revoked or expired, please re-run"
"the application to re-authorize"
)

if you check the documentation Authorizing requests and identifying your application you will find that it states that you can use Oauth2 for authorizing a user to the blogger api and api keys. There is no mention of using service account authorization with this api.
To my knowledge as i tried about five years ago this api does not support service account authentication. You will need to use Oauth2.

PyGithub and Python 3.6

I have the following script that grabs a repository from Github using PYGitHub
import logging
import getpass
import os
from github import Github, Repository as Repository, UnknownObjectException
GITHUB_URL = 'https://github.firstrepublic.com/api/v3'
if __name__ == '__main__':
logging.getLogger().setLevel(logging.DEBUG)
logging.debug('validating GH token')
simpleuser = getpass.getuser().replace('adm_','')
os.path.exists(os.path.join(os.path.expanduser('~' + getpass.getuser()) + '/.ssh/github-' + simpleuser + '.token'))
with open(os.path.join(os.path.expanduser('~' + getpass.getuser()) + '/.ssh/github-' + simpleuser + '.token'), 'r') as token_file:
github_token = token_file.read()
logging.debug(f'Token after file processing: {github_token}')
logging.debug('initializing github')
g = Github(base_url=GITHUB_URL, login_or_token=github_token)
logging.debug("attempting to get repository")
source_repo = g.get_repo('CLOUD/iam')
Works just fine in Python 3.9.1 on my Mac.
In production, we have RHEL7, Python 3.6.8 (can't upgrade it, don't suggest it). This is where it blows up:
(virt) user#lmachine: directory$ python3 test3.py -r ORG/repo_name -d
DEBUG:root:validating GH token
DEBUG:root:Token after file processing: <properly_formed_token>
DEBUG:root:initializing github
DEBUG:root:attempting to get repository
DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): <domain>:443
Traceback (most recent call last):
File "test3.py", line 68, in <module>
source_repo = g.get_repo(args.repo)
File "/home/adm_gciesla/virt/lib/python3.6/site-packages/github/MainClass.py", line 348, in get_repo
"GET", "%s%s" % (url_base, full_name_or_id)
File "/home/user/virt/lib/python3.6/site-packages/github/Requester.py", line 319, in requestJsonAndCheck
verb, url, parameters, headers, input, self.__customConnection(url)
File "/home/user/virt/lib/python3.6/site-packages/github/Requester.py", line 410, in requestJson
return self.__requestEncode(cnx, verb, url, parameters, headers, input, encode)
File "/home/user/virt/lib/python3.6/site-packages/github/Requester.py", line 487, in __requestEncode
cnx, verb, url, requestHeaders, encoded_input
File "/home/user/virt/lib/python3.6/site-packages/github/Requester.py", line 513, in __requestRaw
response = cnx.getresponse()
File "/home/user/virt/lib/python3.6/site-packages/github/Requester.py", line 116, in getresponse
allow_redirects=False,
File "/home/user/virt/lib/python3.6/site-packages/requests/sessions.py", line 543, in get
return self.request('GET', url, **kwargs)
File "/home/user/virt/lib/python3.6/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/home/user/virt/lib/python3.6/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/home/user/virt/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/user/virt/lib/python3.6/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/home/user/virt/lib/python3.6/site-packages/urllib3/connectionpool.py", line 392, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib64/python3.6/http/client.py", line 1254, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib64/python3.6/http/client.py", line 1295, in _send_request
self.putheader(hdr, value)
File "/usr/lib64/python3.6/http/client.py", line 1232, in putheader
raise ValueError('Invalid header value %r' % (values[i],))
ValueError: Invalid header value b'token <properly_formed_token>\n'
The script is a stripped down version of a larger application. I've tried rolling back to earlier versions of PyGitHub, that's really all I have control over in prod. Same error regardless. PyGithub's latest release claims Python >=3.6 should work.
I've really run the gamut of debugging. Seems like reading from environment variables can work sometimes, but the script needs to be able to use whatever credentials are available. Passing in the token as an argument is only for running locally.
Hopefully someone out there has seen something similar.

We just figured it out. Apparently, even though there's no newline in the .token file, there is one after calling file.read()
Changing github_token = token_file.read() to github_token = token_file.read().strip() fixes the problem.

Network is unreachable while using BigQuery API client library for Python

I'm using BigQuery API client library for Python from Google Compute Engine. While making query it throws network unreachable error.
[INFO:2018-01-02 16:16:04,887:oauth2client.transport] Attempting refresh to obtain initial access_token
[INFO:2018-01-02 16:16:04,924:oauth2client.client] Refreshing access_token
Traceback (most recent call last):
File "/test/data/reports/ga_bigquery.py", line 130, in ga_table_string
if bq_dataset.table("ga_sessions_{}".format(str_date)).exists():
File "/test/data/reports/.venv/lib/python2.7/site-packages/gcloud/bigquery/table.py", line 472, in exists
query_params={'fields': 'id'})
File "/test/data/reports/.venv/lib/python2.7/site-packages/gcloud/connection.py", line 343, in api_request
target_object=_target_object)
File "/test/data/reports/.venv/lib/python2.7/site-packages/gcloud/connection.py", line 241, in _make_request
return self._do_request(method, url, headers, data, target_object)
File "/test/data/reports/.venv/lib/python2.7/site-packages/gcloud/connection.py", line 270, in _do_request
body=data)
File "/test/data/reports/.venv/lib/python2.7/site-packages/oauth2client/transport.py", line 153, in new_request
credentials._refresh(orig_request_method)
File "/test/data/reports/.venv/lib/python2.7/site-packages/oauth2client/client.py", line 765, in _refresh
self._do_refresh_request(http_request)
File "/test/data/reports/.venv/lib/python2.7/site-packages/oauth2client/client.py", line 797, in _do_refresh_request
self.token_uri, method='POST', body=body, headers=headers)
File "/test/data/reports/.venv/lib/python2.7/site-packages/httplib2/__init__.py", line 1609, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/test/data/reports/.venv/lib/python2.7/site-packages/httplib2/__init__.py", line 1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/test/data/reports/.venv/lib/python2.7/site-packages/httplib2/__init__.py", line 1272, in _conn_request
conn.connect()
File "/test/data/reports/.venv/lib/python2.7/site-packages/httplib2/__init__.py", line 1075, in connect
raise socket.error, msg
socket.error: [Errno 101] Network is unreachable
What could be the reason?
I run the code with following option :
[root#myserver]# strace -ff -e poll,select,connect,recvfrom,sendto python run.py --date=20180102 >> strace.log
[root#myserver]# cat strace.log | grep unreachable
connect(3, {sa_family=AF_INET6, sin6_port=htons(443), inet_pton(AF_INET6, “2404:6800:4003:c02::54”, &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = -1 ENETUNREACH (Network isunreachable)
connect(4, {sa_family=AF_INET6, sin6_port=htons(443), inet_pton(AF_INET6, “2404:6800:4003:803::200a”, &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = -1 ENETUNREACH (Network is unreachable)
connect(5, {sa_family=AF_INET6, sin6_port=htons(443), inet_pton(AF_INET6, “2404:6800:4003:808::200d”, &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = -1 ENETUNREACH (Network is unreachable)
connect(6, {sa_family=AF_INET6, sin6_port=htons(443), inet_pton(AF_INET6, “2404:6800:4003:c03::5f”, &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = -1 ENETUNREACH (Network isunreachable)
From log it looks like compute instance is blocking outgoing ipv6 request.
Is there any way we can unblcok ipv6 requests in google compute engine ?

First, if you are using the BigQuery API Client Library for Python, you may want to try the BigQuery Cloud Client Library for Python, instead.
One reason for this could be that the API client library uses httplib2 to make requests to BigQuery, which has problems in some network setups such as behind proxies. The Cloud library uses the more standard requests library, so it should be more reliable.
Second, even if you don't switch libraries, you should change what authentication library you are using. Your stack trace shows oauth2client, but oauth2client is deprecated. Use google-auth-httplib2 to use the API client libraries with the google-auth library.

Python Scrape with requests

When I run this script IDLE does not continue. Usually it will give an error. Other scripts run fine so i know its not IDLE. I thought my code was correct but maybe I've missed something. This is not everything i will be scraping from the site, just wanted to see this work first than I can run through everything later.
import csv
import requests
import os
##HOME TEAM
req = requests.get('http://stats.nba.com/stats/leaguedashteamstats?Conference=&DateFrom=10%2F17%2F2017&DateTo=04%2F11%2F2018&Division=&GameScope=&GameSegment=&LastNGames=0&LeagueID=00&Location=Home&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2017-18&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&VsConference=&VsDivision=')
data = req.json()
my_data = []
pk = data['resultSets']
for item in data:
team = item.get['rowSet']
for item in team:
Team_Id = item[0]
Team_Name = item[1]
my_data.append([Team_Id, Team_Name])
headers = ["Team_Id", "Team_Name"]
with open("NBA_Home_Team.csv", "a", newline='') as f:
writer = csv.writer(f)
writer.writerow(headers)
writer.writerows(my_data)
f.close()
##os.system("taskkill /f /im pythonw.exe")

Seems like it hangs because the server doesn't respond. It can be verified by killing the process and checking the stack trace:
Traceback (most recent call last):
req = requests.get('http://stats.nba.com/stats/leaguedashteamstats?Conference=&DateFrom=10%2F17%2F2017&DateTo=04%2F11%2F2018&Division=&GameScope=&GameSegment=&LastNGames=0&LeagueID=00&Location=Home&MeasureType=Base&Month=0
&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2017-18&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&VsCo
nference=&VsDivision=')
File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 72, in get
return request('get', url, params=params, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 502, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 612, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/usr/local/lib/python2.7/dist-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/usr/local/lib/python2.7/dist-packages/urllib3/connectionpool.py", line 379, in _make_request
httplib_response = conn.getresponse(buffering=True)
File "/usr/lib/python2.7/httplib.py", line 1121, in getresponse
response.begin()
File "/usr/lib/python2.7/httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.7/httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "/usr/lib/python2.7/socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize) <-- we're stucking here
KeyboardInterrupt
I tried to open the URL in my browser and it worked fine and I received response within a second. Then I started to tweak the request in the code to mimic a valid browser. My first idea was using a valid User-Agent and I immediately received response with the following code:
data = requests.get(
'http://stats.nba.com/stats/leaguedashteamstats?Conference=&DateFrom=10%2F17%2F2017&DateTo=04%2F11%2F2018&Division=&GameScope=&GameSegment=&LastNGames=0&LeagueID=00&Location=Home&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2017-18&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&VsConference=&VsDivision=',
headers={'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A300 Safari/602.1'},
).json()
Perhaps some kind of defense mechanism against bots causes the unresponsiveness without valid User-Agent.
Other notes on the code snippet:
for item in data:
Use pk instead of data.
team = item.get['rowSet']
Use item['rowSet'] or item.get('rowSet') but do not mix them. item.get is a function so [] cannot be applied.
my_data.append([Team_Id, Team_Name])
The indentation should be the same as the above line

Httplib2 exception raised when calling Twilio from python

I'm trying to send an SMS message via Twilio from some Python code running in a Docker container. The container was built from an Ubuntu image with Python 2.7 and the latest twilio-python helper library. When calling the TwilioRestClient messages.create() function, I get the traceback below. Also, I have confirmed via curl that I can reach the url "api.twilio.com" as indicated in the traceback from this container. Any ideas?
Code being used...
import argparse
import requests
import time
import smtplib
from datetime import datetime
from twilio.rest import TwilioRestClient
def sendText( ToNumber, smsMsg):
# put your own credentials here
ACCOUNT_SID = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
AUTH_TOKEN = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
FromNumber = '+nnnnnnnnnnn'
client = TwilioRestClient(ACCOUNT_SID, AUTH_TOKEN)
client.messages.create(to = ToNumber,from_ = FromNumber,body = smsMsg,)
return True
Traceback...
File "./siteminder3.py", line 15, in sendText
message = client.messages.create(to=ToNumber,from_=FromNumber,body=smsMsg)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/messages.py", line 122, in create
return self.create_instance(kwargs)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 365, in create_instance
data=transform_params(body))
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 200, in request
resp = make_twilio_request(method, uri, auth=self.auth, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 152, in make_twilio_request
resp = make_request(method, uri, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/twilio/rest/resources/base.py", line 117, in make_request
resp, content = http.request(url, method, headers=headers, body=data)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1609, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1278, in _conn_request
raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
httplib2.ServerNotFoundError: Unable to find the server at api.twilio.com