I wrote a script which fetches publicly available data and puts into SQLite DB using peewee. The data is available like this site.com/data/1, site.com/data/2 ... site.com/data/N. So I use a for loop and change the value of N each time.
Problem is after a while script stops working. I mean, it just stops responding (not exits). I have print statement in loop and it stops printing next numbers.
For a smaller loop range it works perfectly and for larger it stops working. I have found that upto range of 80, it just works fine.
When I force close the script, I get following on terminal. It basically something related to requests and http connections.
Traceback (most recent call last):
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 372, in _make_request
httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'
Full traceback:
Traceback (most recent call last):
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 372, in _make_request
httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "script.py", line 71, in <module>
main()
File "script.py", line 56, in main
user_data = get_script_user_data(i)
File "script.py", line 42, in get_script_user_data
r = requests.get(users_data_public_api.format(user_id))
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/api.py", line 69, in get
return request('get', url, params=params, **kwargs)
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/sessions.py", line 465, in request
resp = self.send(prep, **send_kwargs)
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/sessions.py", line 573, in send
r = adapter.send(request, **kwargs)
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/adapters.py", line 370, in send
timeout=timeout
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 544, in urlopen
body=body, headers=headers)
File "/Users/avi/Documents/code/my-app/venv/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 374, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/Cellar/python3/3.4.3/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1171, in getresponse
response.begin()
File "/usr/local/Cellar/python3/3.4.3/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 351, in begin
version, status, reason = self._read_status()
File "/usr/local/Cellar/python3/3.4.3/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 313, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/Cellar/python3/3.4.3/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socket.py", line 374, in readinto
return self._sock.recv_into(b)
KeyboardInterrupt
I thought I am making requests too fast and request is not able to buffer it (just my assumptions). So I added sleep statement for half second. And problem still existed. Then I changed sleep to 1.5 second, worked till loop range was upto 400. But on next iteration it stopped at within range of 50-60.
Here's the full code:
import sqlite3
import datetime
import time
from datetime import date
import requests
from peewee import *
users_data_public_api = "http://api.some-site.com/user/{0}"
db = SqliteDatabase('opendata_users.db')
class OpendataUser(Model):
user_id = IntegerField()
fullname = CharField()
email = CharField()
sex = CharField(null = True)
dob = DateField(null = True)
class Meta:
database = db
def initialize_db():
db.connect()
#db.create_tables([OpendataUser])
def deinit():
db.close()
def get_opendata_user_data(user_id):
""" Returns Opendata user data only if he has all the fields required """
r = requests.get(users_data_public_api.format(user_id))
if r.status_code == requests.codes.ok:
user_data = r.json()['users'][0]
if user_data['email'] != None and 'deleted' not in user_data['email']:
if user_data['fullname'] != None and user_data['dob'] != None:
try:
user_data['dob'] = datetime.datetime.strptime(user_data['dob'], "%d/%m/%Y").date()
except ValueError:
user_data['dob'] = None
return user_data
def main():
for i in range(450, 600):
user_data = get_opendata_user_data(i)
if user_data:
print(i)
od_user = OpendataUser(user_id=user_data['user_id'],
fullname=user_data['fullname'],
email=user_data['email'],
sex=user_data['sex'],
dob=user_data['dob'])
od_user.save()
time.sleep(1.5)
if __name__ == '__main__':
initialize_db()
main()
deinit()
So whats the reason? appreciate any help. (code is not open source, so I have modified here and there)
Related
I'm trying to use Authlib library to access new eBay REST API (as Authorization code grant)
Here is my code;
import json
import os
import webbrowser
from time import time
from authlib.integrations.requests_client import OAuth2Session
from rpi_order_data_sync import settings
def auth(seller):
def token_updater(token, seller=seller):
if not os.path.exists(seller):
open(seller, "w").close()
with open(seller, "w") as token_file:
json.dump(token, token_file)
scope = ["https://api.ebay.com/oauth/api_scope/sell.fulfillment.readonly"]
if not os.path.exists(seller):
ebay = OAuth2Session(
settings.E_APP_ID,
settings.E_CERT_ID,
redirect_uri=settings.E_RU_NAME,
scope=scope,
)
uri, state = ebay.create_authorization_url(
"https://auth.sandbox.ebay.com/oauth2/authorize",
)
print("Please go to {} and authorize access.".format(uri))
try:
webbrowser.open_new_tab(uri)
except webbrowser.Error:
pass
authorization_response = input("Please enter callback URL: ") # nosec
token = ebay.fetch_token(
"https://api.sandbox.ebay.com/identity/v1/oauth2/token",
authorization_response=authorization_response,
)
print(token)
token_updater(token)
return ebay
The problem is eBay's token response has an unconventional token type named "User Access Token" instead of "Bearer". Therefore I get this error;
Traceback (most recent call last):
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/authlib/integrations/requests_client/oauth2_session.py", line 37, in __call__
req.url, req.headers, req.body = self.prepare(
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/authlib/oauth2/auth.py", line 91, in prepare
sign = self.SIGN_METHODS[token_type.lower()]
KeyError: 'user access token'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/bin/rods", line 11, in <module>
load_entry_point('rpi-order-data-sync', 'console_scripts', 'rods')()
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/home/thiras/HDD/freelancer/contentassasin/rpi-order-data-sync/rpi_order_data_sync/main.py", line 132, in sync_ebay_orders
orders = ebay.get(
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/requests/sessions.py", line 543, in get
return self.request('GET', url, **kwargs)
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/authlib/integrations/requests_client/oauth2_session.py", line 113, in request
return super(OAuth2Session, self).request(
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/requests/sessions.py", line 516, in request
prep = self.prepare_request(req)
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/requests/sessions.py", line 449, in prepare_request
p.prepare(
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/requests/models.py", line 318, in prepare
self.prepare_auth(auth, url)
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/requests/models.py", line 549, in prepare_auth
r = auth(self)
File "/home/thiras/.local/share/virtualenvs/rpi-order-data-sync-tA0i1rrc/lib/python3.8/site-packages/authlib/integrations/requests_client/oauth2_session.py", line 41, in __call__
raise UnsupportedTokenTypeError(description=description)
authlib.integrations.base_client.errors.UnsupportedTokenTypeError: unsupported_token_type: Unsupported token_type: 'user access token'
I've noticed Compliance fix for non-standard section at Authlib documentation but couldn't figure out how to do this fix or even possible in this way.
I've found a solution and it also works with requests-oauthlib package. It seems working flawlessly so far. The main struggle was to create a fake request.Response model since request.Response has no setter for .text or .content attributes so modifying them was impossible.
So I've created a FakeResponse class that only mimics .json() method since it was the only method used by Authlib.
class FakeResponse:
""" Fake Class for Request Response class. """
def __init__(self, data):
self.data = data
def json(self):
""" Mocks requests.Response.json(). """
return self.data
After that I've created an access_token_response hook;
def non_compliant_token_type(resp):
data = resp.json()
data["token_type"] = "Bearer"
fake_resp = FakeResponse(data=data)
return fake_resp
Please let me know if you have a better answer or any recommendations to improve it.
I am having problems with Python and Bit Bucket. To display/pull/push do anything really.
I am looking # Two different libs, atlassian-python-api, and stashy, both seem to have problems my code is very simple:
from atlassian import Bitbucket
import getpass
username = input("What is your username: ")
password = getpass.getpass(prompt="Enter your password?: ")
bitbucket = Bitbucket(
url="https://website.com:port/projects/demo_projects/repppos/",
username=username,
password=password)
data = bitbucket.project_list()
both give me this error: using stashy and another library. I heard someone suggest to use Rest API but I have no experience with this?
Traceback (most recent call last):
File "C:/Users/User/PycharmProjects/ProjectName/terrafw_gui/test_no_gui.py", line 12, in <module>
data = bitbucket.project_list()
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\atlassian\bitbucket.py", line 22, in project_list
return (self.get('rest/api/1.0/projects', params=params) or {}).get('values')
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\atlassian\rest_client.py", line 208, in get
trailing=trailing)
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\atlassian\rest_client.py", line 151, in request
files=files
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\requests\sessions.py", line 279, in request
resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\requests\sessions.py", line 374, in send
r = adapter.send(request, **kwargs)
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\requests\adapters.py", line 174, in send
timeout=timeout
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 417, in urlopen
conn = self._get_conn(timeout=pool_timeout)
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 232, in _get_conn
return conn or self._new_conn()
File "C:\Users\User\PycharmProjects\ProjectName\venv\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 547, in _new_conn
strict=self.strict)
TypeError: __init__() got an unexpected keyword argument 'strict'
Process finished with exit code 1
I cannot figure out how come, or why I am getting these error messages without an attempted connection (the error is given immediately without any seconds for timeout).
I need my bot to monitor my raspberry cpu temperature. It checks it every minute and then send an alert if > a threshold. When a message is sent I need it to not send it again for 10 minutes. I've done it but then I get a timeout error when sending the same message 10 minutes after. Can anybody help me? I did not find any help on telepot giyhub page.
This is my code
bot = telepot.Bot(TOKEN)
bot.message_loop(handle)
while 1:
if ((get_cpu_temperature() > 30.0) and alarm()):
data = "Temperature: " + str(get_cpu_temperature()) + " 'C"
bot.sendMessage(users[0],data)
time.sleep(60)
The alarm function just checks if 10 mins are passed.
This is the error:
Traceback (most recent call last):
File "temp_disk_check_live.py", line 74, in <module>
bot.sendMessage(users[0],data)
File "/usr/local/lib/python2.7/dist-packages/telepot/__init__.py", line 456, in sendMessage
return self._api_request('sendMessage', _rectify(p))
File "/usr/local/lib/python2.7/dist-packages/telepot/__init__.py", line 434, in _api_request
return api.request((self._token, method, params, files), **kwargs)
File "/usr/local/lib/python2.7/dist-packages/telepot/api.py", line 130, in request
r = fn(*args, **kwargs) # `fn` must be thread-safe
File "/home/pi/.local/lib/python2.7/site-packages/urllib3/request.py", line 148, in request_encode_body
return self.urlopen(method, url, **extra_kw)
File "/home/pi/.local/lib/python2.7/site-packages/urllib3/poolmanager.py", line 321, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/home/pi/.local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/pi/.local/lib/python2.7/site-packages/urllib3/util/retry.py", line 357, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/pi/.local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/home/pi/.local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 389, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/home/pi/.local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 320, in _raise_timeout
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='api.telegram.org', port=443): Read timed out. (read timeout=30)
Exception in thread Thread-1 (most likely raised during interpreter shutdown):
Traceback (most recent call last):
File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
File "/usr/local/lib/python2.7/dist-packages/telepot/__init__.py", line 391, in run
File "/usr/local/lib/python2.7/dist-packages/telepot/__init__.py", line 310, in k
File "/usr/lib/python2.7/threading.py", line 168, in acquire
<type 'exceptions.TypeError'>: 'NoneType' object is not callable
the handle function is the standard one from telepot examples.
Thanks a lot
You could create a new thread and start a timer like..
def hello():
print("hello, world")
t = Timer(30.0, hello)
t.start() # after 30 seconds, "hello, world" will be printed
So in your code;
def send_message(user,message):
bot.sendMessage(user,message)
t = Timer(600, send_message("Temperature...", user[0])
if cpu_temp > 30: t.start()
How about initialize Bot whenever you really need to send a message?
while 1:
if ((get_cpu_temperature() > 30.0) and alarm()):
data = "Temperature: " + str(get_cpu_temperature()) + " 'C"
telepot.Bot(TOKEN).sendMessage(users[0],data)
time.sleep(60*10) # 10 min
I am trying to step down MongoDB primary using python script.I see the below on my output. Is there a way I can mark the exit code as OK.
Code:
if 'primary' in isMaster:
primary =(isMaster['primary']).split(':')[0]
conn = pymongo.MongoClient('mongodb://'+ primary +':10000 replicaSet=test-ipe1')
stepdown = conn.admin.command("replSetStepDown",100)
if 'ismaster' in isMaster == 'true':
print("I am still the primary")
else:
print("I am no longer the primary")
else:
primary = "No primary currently elected."
Traceback:
Traceback (most recent call last):
File "./repldown.py", line 39, in <module>
stepdown = conn.admin.command("replSetStepDown",60)
File "/usr/local/lib64/python2.6/site-packages/pymongo/database.py", line 391, in command
result = self["$cmd"].find_one(command, **extra_opts)
File "/usr/local/lib64/python2.6/site-packages/pymongo/collection.py", line 604, in find_one
for result in self.find(spec_or_id, *args, **kwargs).limit(-1):
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 904, in next
if len(self.__data) or self._refresh():
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 848, in _refresh
self.__uuid_subtype))
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 782, in __send_message
res = client._send_message_with_response(message, **kwargs)
File "/usr/local/lib64/python2.6/site-packages/pymongo/mongo_client.py", line 1051, in _send_message_with_response
raise AutoReconnect(str(e))
pymongo.errors.AutoReconnect: connection closed
Yes, do:
from pymongo.errors import AutoReconnect
try:
stepdown = conn.admin.command("replSetStepDown",100)
except AutoReconnect:
pass
The primary deliberately closes all connections immediately when it steps down, so the command throws an exception. Just ignore it and continue.
The following code:
import gevent
import gevent.monkey
gevent.monkey.patch_socket()
import requests
import json
base_url = 'https://api.getclever.com'
section_url = base_url + '/v1.1/sections'
#get all sections
sections = requests.get(section_url, auth=('DEMO_KEY', '')).json()
urls = [base_url+data['uri']+'/students' for data in sections['data']]
#get students for each section
threads = [gevent.spawn(requests.get, url, auth=('DEMO_KEY', '')) for url in urls]
gevent.joinall(threads)
students = [thread.value for thread in threads]
#get number of students in each section
num_students = [len(student.json()['data']) for student in students]
print (sum(num_students)/len(num_students))
results in this error:
Traceback (most recent call last):
File "clever.py", line 12, in <module>
sections = requests.get(section_url, auth=('DEMO_KEY', '')).json()
File "/Library/Python/2.7/site-packages/requests/api.py", line 55, in get
return request('get', url, **kwargs)
File "/Library/Python/2.7/site-packages/requests/api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "/Library/Python/2.7/site-packages/requests/sessions.py", line 382, in request
resp = self.send(prep, **send_kwargs)
File "/Library/Python/2.7/site-packages/requests/sessions.py", line 485, in send
r = adapter.send(request, **kwargs)
File "/Library/Python/2.7/site-packages/requests/adapters.py", line 379, in send
raise SSLError(e)
requests.exceptions.SSLError: [Errno 2] _ssl.c:503: The operation did not complete (read)
What am I doing wrong here?
Here's a similar question: [Errno 2] _ssl.c:504: The operation did not complete (read).
When you comment out
gevent.monkey.patch_socket()
or use
gevent.monkey.patch_all()
or use
gevent.monkey.patch_socket()
gevent.monkey.patch_ssl()
then the problem disappears.