I'm trying to pass data of two variables into another variable but I'm getting following error:
Traceback (most recent call last):
File "poll_azure_devops_audit_log.py", line 28, in <module>
x = requests.request('GET', URL, proxies=PROXIES, auth=AZURE_AUTH).json()
File "/opt/splunk/lib/python3.7/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, kwargs)
File "/opt/splunk/lib/python3.7/site-packages/requests/sessions.py", line 519, in request
prep = self.prepare_request(req)
File "/opt/splunk/lib/python3.7/site-packages/requests/sessions.py", line 462, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "/opt/splunk/lib/python3.7/site-packages/requests/models.py", line 317, in prepare
self.prepare_auth(auth, url)
File "/opt/splunk/lib/python3.7/site-packages/requests/models.py", line 548, in prepare_auth
r = auth(self)
TypeError: 'str' object is not callable
How do I have to pass the variables USER & PASSWORD into variable AZURE_AUTH?
import sys
import requests
import json
from urllib.parse import quote
from requests.auth import HTTPBasicAuth
ORGANIZATION = 'xxx'
CONTINUATIONTOKEN = None
API_ENDPOINT = 'https://auditservice.dev.azure.com/'
USER = 'xxx'
PASSWORD = 'xxx'
AZURE_AUTH = HTTPBasicAuth('user123', 'password123')
URL = API_ENDPOINT + ORGANIZATION + '/_apis/audit/auditlog?api-version=6.1-preview&skipAggregation=true'
PROXIES = {
'http' : 'http://xxx:8080',
'https' : 'https://xxx:8080',
}
print(URL)
print(AZURE_AUTH)
print(USER)
print(PASSWORD)
print(PROXIES)
x = requests.request('GET', URL, proxies=PROXIES, auth=AZURE_AUTH).json()
print (x)
I think the problem you think you have is not the problem you actually have. To pass USER and PASSWORD to HTTPBasicAuth is done like this:
AZURE_AUTH = HTTPBasicAuth(USER, PASSWORD)
and you can assign any value you want to your USER and PASSWORD variables like this :
USER = 'user123'
PASSWORD = 'password123'
If you get an error it is comming from somewhere else
Related
I am trying to use Reddit's developer API to build a simple scraper that grabs posts and their replies in a target subreddit and produces JSON with the information.
I am getting a 404 error that I don't understand.
This is my code:
import praw
import json
def scrape(subreddit, limit):
r = praw.Reddit(user_agent='Reddit data organizer 1.0 by /u/reallymemorable', client_id='none of your business', client_secret='none of your business')
submissions = r.subreddit(subreddit).get_hot(limit=limit)
for submission in submissions:
data = {}
data['title'] = submission.title
data['score'] = submission.score
data['url'] = submission.url
data['author'] = str(submission.author)
data['subreddit'] = str(submission.subreddit)
data['num_comments'] = submission.num_comments
data['over_18'] = submission.over_18
data['selftext'] = submission.selftext
data['is_self'] = submission.is_self
data['name'] = submission.name
data['created_utc'] = submission.created_utc
data['permalink'] = submission.permalink
data['domain'] = submission.domain
data['id'] = submission.id
data['kind'] = submission.kind
json.dumps(data)
scrape('https://www.reddit.com/r/funny/', 25)
When I run it, I get this:
reallymemorable#Christians-MBP Desktop % python3 fetch-data-subreddit.py
Traceback (most recent call last):
File "/Users/reallymemorable/Desktop/fetch-data-subreddit.py", line 26, in <module>
scrape('https://www.reddit.com/r/augmentedreality/comments/yv7sn8/ar_maximum_distance/', 25)
File "/Users/reallymemorable/Desktop/fetch-data-subreddit.py", line 6, in scrape
submissions = r.subreddit(subreddit).get_hot(limit=limit)
File "/opt/homebrew/lib/python3.9/site-packages/praw/models/reddit/base.py", line 34, in __getattr__
self._fetch()
File "/opt/homebrew/lib/python3.9/site-packages/praw/models/reddit/subreddit.py", line 583, in _fetch
data = self._fetch_data()
File "/opt/homebrew/lib/python3.9/site-packages/praw/models/reddit/subreddit.py", line 580, in _fetch_data
return self._reddit.request(method="GET", params=params, path=path)
File "/opt/homebrew/lib/python3.9/site-packages/praw/util/deprecate_args.py", line 43, in wrapped
return func(**dict(zip(_old_args, args)), **kwargs)
File "/opt/homebrew/lib/python3.9/site-packages/praw/reddit.py", line 941, in request
return self._core.request(
File "/opt/homebrew/lib/python3.9/site-packages/prawcore/sessions.py", line 330, in request
return self._request_with_retries(
File "/opt/homebrew/lib/python3.9/site-packages/prawcore/sessions.py", line 266, in _request_with_retries
raise self.STATUS_EXCEPTIONS[response.status_code](response)
prawcore.exceptions.NotFound: received 404 HTTP response
r.subreddit(subreddit) - subreddit should just be the name of the subreddit e.g. "funny" and not the full URL.
See the docs here: https://praw.readthedocs.io/en/stable/getting_started/quick_start.html#obtain-a-subreddit
I am trying to use a POST URI in my code to access NCBO's Annotator tool. My current code is a GET request, but I don't know how to format this into a POST request. My data is the text variable.
All the examples which I've seen uses request.get(url) -> request.post(url, data=data), but how can I do this for build_opener() and json.loads()?
Here is my code:
def get_json(url):
#get annotations
opener = urllib.request.build_opener()
opener.addheaders = [('Authorization', 'apikey token=' + API_KEY)]
return json.loads(opener.open(url).read())
text = "random text with a lot of words"
annotations = get_json("http://data.bioontology.org/annotator?text=" + urllib.parse.quote(text))
Updated code:
def get_annotations(text, url):
headers = [('Authorization', 'apikey token=' + API_KEY)]
data = text
response = requests.request("POST",url,headers=headers,data=data)
return response.text.encode('utf-8')
annotations = get_annotations(text, "http://data.bioontology.org/annotator?text=" + urllib.parse.quote(text))
Error:
response = requests.request("POST",url,headers=headers,data=data)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/sessions.py", line 516, in request
prep = self.prepare_request(req)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/sessions.py", line 449, in prepare_request
p.prepare(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/models.py", line 315, in prepare
self.prepare_headers(headers)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/models.py", line 447, in prepare_headers
for header in headers.items():
AttributeError: 'list' object has no attribute 'items'
I will recommend you to import requests module, this module is easy to choose request method.
import requests
def get_json(url):
headers = [('Authorization', 'apikey token=' + API_KEY)] #add headers here
data = #params you want to send
response = requests.request("POST",url,headers=headers,data=data) #you can choose your methods here.
return response.text.encode('utf-8')
FYR simply like this.
I am trying to implement the Amazon Web Scraper mentioned here. However, I get the output mentioned below. The output repeats until it stops with RecursionError: maximum recursion depth exceeded.
I have already tried downgrading eventlet to version 0.17.4 as mentioned here.
Also, the requestsmodule is getting patched as you can see in helpers.py.
helpers.py
import os
import random
from datetime import datetime
from urllib.parse import urlparse
import eventlet
requests = eventlet.import_patched('requests.__init__')
time = eventlet.import_patched('time')
import redis
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import settings
num_requests = 0
redis = redis.StrictRedis(host=settings.redis_host, port=settings.redis_port, db=settings.redis_db)
def make_request(url, return_soup=True):
# global request building and response handling
url = format_url(url)
if "picassoRedirect" in url:
return None # skip the redirect URLs
global num_requests
if num_requests >= settings.max_requests:
raise Exception("Reached the max number of requests: {}".format(settings.max_requests))
proxies = get_proxy()
try:
r = requests.get(url, headers=settings.headers, proxies=proxies)
except RequestException as e:
log("WARNING: Request for {} failed, trying again.".format(url))
num_requests += 1
if r.status_code != 200:
os.system('say "Got non-200 Response"')
log("WARNING: Got a {} status code for URL: {}".format(r.status_code, url))
return None
if return_soup:
return BeautifulSoup(r.text), r.text
return r
def format_url(url):
# make sure URLs aren't relative, and strip unnecssary query args
u = urlparse(url)
scheme = u.scheme or "https"
host = u.netloc or "www.amazon.de"
path = u.path
if not u.query:
query = ""
else:
query = "?"
for piece in u.query.split("&"):
k, v = piece.split("=")
if k in settings.allowed_params:
query += "{k}={v}&".format(**locals())
query = query[:-1]
return "{scheme}://{host}{path}{query}".format(**locals())
def log(msg):
# global logging function
if settings.log_stdout:
try:
print("{}: {}".format(datetime.now(), msg))
except UnicodeEncodeError:
pass # squash logging errors in case of non-ascii text
def get_proxy():
# choose a proxy server to use for this request, if we need one
if not settings.proxies or len(settings.proxies) == 0:
return None
proxy = random.choice(settings.proxies)
proxy_url = "socks5://{user}:{passwd}#{ip}:{port}/".format(
user=settings.proxy_user,
passwd=settings.proxy_pass,
ip=proxy,
port=settings.proxy_port,
)
return {
"http": proxy_url,
"https": proxy_url
}
if __name__ == '__main__':
# test proxy server IP masking
r = make_request('https://api.ipify.org?format=json', return_soup=False)
print(r.text)
output
Traceback (most recent call last):
File "helpers.py", line 112, in <module>
r = make_request('https://api.ipify.org?format=json', return_soup=False)
File "helpers.py", line 36, in make_request
r = requests.get(url, headers=settings.headers, proxies=proxies)
File "/home/ec2-user/env/lib64/python3.7/site-packages/requests/api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "/home/ec2-user/env/lib64/python3.7/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/ec2-user/env/lib64/python3.7/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/home/ec2-user/env/lib64/python3.7/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/home/ec2-user/env/lib64/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/ec2-user/env/lib64/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/home/ec2-user/env/lib64/python3.7/site-packages/urllib3/connectionpool.py", line 376, in _make_request
self._validate_conn(conn)
File "/home/ec2-user/env/lib64/python3.7/site-packages/urllib3/connectionpool.py", line 994, in _validate_conn
conn.connect()
File "/home/ec2-user/env/lib64/python3.7/site-packages/urllib3/connection.py", line 300, in connect
conn = self._new_conn()
File "/home/ec2-user/env/lib64/python3.7/site-packages/urllib3/contrib/socks.py", line 99, in _new_conn
**extra_kw
File "/home/ec2-user/env/lib64/python3.7/site-packages/socks.py", line 199, in create_connection
sock.connect((remote_host, remote_port))
File "/home/ec2-user/env/lib64/python3.7/site-packages/socks.py", line 47, in wrapper
return function(*args, **kwargs)
File "/home/ec2-user/env/lib64/python3.7/site-packages/socks.py", line 774, in connect
super(socksocket, self).settimeout(self._timeout)
File "/home/ec2-user/env/lib64/python3.7/site-packages/eventlet/greenio/base.py", line 395, in settimeout
self.setblocking(True)
What might be the problem here?
Turns out removing eventlet.monkey_patch() and import eventlet solved the problem.
I'm trying to write a script to programmatically login to Google Finance, view my portfolio and then display results on my desktop. I'm using the requests module, currently stuck on the 'login' part.
I keep getting this error requests.cookies.CookieConflictError: There are multiple cookies with name, 'APISID'
Here is the entire script, the error throws on line 48. I'm guessing it has something to do with requests keep-alive and the connection isn't recycling properly?
#!/usr/bin/env python
import getpass
import re
import requests
email = raw_input("Enter your Google username: ")
password = getpass.getpass("Enter your password: ")
session = requests.Session()
# Define URLs
login_page_url = 'https://accounts.google.com/ServiceLogin?passive=true&service=finance'
authenticate_url = 'https://accounts.google.com/ServiceLoginAuth?service=finance'
gf_home_page_url = 'http://www.google.com/finance/portfolio'
login_page_contents = session.get(login_page_url).text
# Find GALX value
galx_match_obj = re.search(r'name="GALX"\s*value="([^"]+)"', login_page_contents, re.IGNORECASE)
galx_value = galx_match_obj.group(1) if galx_match_obj.group(1) is not None else ''
# Find DSH value
dsh_match_obj = re.search(r'id="dsh"\s*value="([^"]+)"', login_page_contents, re.IGNORECASE)
dsh_value = dsh_match_obj.group(1) if dsh_match_obj.group(1) is not None else ''
# Set up login credentials
login_params = {
'Email': email,
'Passwd': password,
'continue': 'http://www.google.com/finance/portfolio',
'followup': 'http://www.google.com/finance/portfolio',
'service': 'finance',
'GALX': galx_value,
'pstMsg': 0,
'dnConn': '',
'checkConnection': '',
'timeStmp': '',
'secTok': '',
'bgresponse': 'js_disabled',
'PersistentCookie': 'yes'
}
print galx_value
print dsh_value
# Login
r = session.post(authenticate_url, params=login_params) # <- Error thrown here
print r.text
exit
Traceback:
Traceback (most recent call last):
File "crawl.py", line 48, in <module>
r = session.post(authenticate_url, params=login_params)
File "/Users/nathan/Development/Scripts/google-finance-crawler/requests/sessions.py", line 358, in post
return self.request('POST', url, data=data, **kwargs)
File "/Users/nathan/Development/Scripts/google-finance-crawler/requests/sessions.py", line 312, in request
resp = self.send(prep, **send_kwargs)
File "/Users/nathan/Development/Scripts/google-finance-crawler/requests/sessions.py", line 426, in send
history = [resp for resp in gen] if allow_redirects else []
File "/Users/nathan/Development/Scripts/google-finance-crawler/requests/sessions.py", line 163, in resolve_redirects
resp.cookies.update(cookiejar)
File "/usr/local/Cellar/python/2.7.3/Frameworks/Python.framework/Versions/2.7/lib/python2.7/_abcoll.py", line 494, in update
self[key] = other[key]
File "/Users/nathan/Development/Scripts/google-finance-crawler/requests/cookies.py", line 246, in __getitem__
return self._find_no_duplicates(name)
File "/Users/nathan/Development/Scripts/google-finance-crawler/requests/cookies.py", line 285, in _find_no_duplicates
raise CookieConflictError('There are multiple cookies with name, %r' % (name))
requests.cookies.CookieConflictError: There are multiple cookies with name, 'APISID'
It's a bug in requests, see issue 1189.
The current proposed fix is to simply delete line 163 of requests/sessions.py:
resp.cookies.update(cookiejar)
I want to write a testcase for sending post data to login page. It does not work. I post my code here and wish you can help me. Thanks.
def setUp(self):
"""set up"""
un = 'abc#gmail.com'
pw = '123'
self.user = User.objects.create_user(un, un)
self.user.is_staff = True
self.user.is_superuser = True
self.user.firstname = "John"
self.user.lastname = "Smith"
self.user.password = '123'
self.user.save()
print '*** password: ', self.user.password
def testPost(self):
"""test POST requests"""
post_data = {
'email': 'abc#gmail.com',
'password': '123',
}
response = self.client.post(reverse('myapp_home', post_data))
print response.status_code
The error output is at below.
ERROR: testPost (submngr.tests.model_tests.model_tests.FormsTestCase)
test POST requests
----------------------------------------------------------------------
Traceback (most recent call last):
File "tests/model_tests/model_tests.py", line 117, in testPost
response = self.client.post('/', post_data)
File "/usr/local/lib/python2.7/dist-packages/django/test/client.py", line 449, in post
response = super(Client, self).post(path, data=data, content_type=content_type, **extra)
File "/usr/local/lib/python2.7/dist-packages/django/test/client.py", line 262, in post
return self.request(**r)
File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 111, in get_response
response = callback(request, *callback_args, **callback_kwargs)
File "views.py", line 84, in homepage
print results[0].check_password(form.cleaned_data['password'])
File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/models.py", line 304, in check_password
return check_password(raw_password, self.password, setter)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/hashers.py", line 42, in check_password
hasher = get_hasher(algorithm)
File "/usr/local/lib/python2.7/dist-packages/django/contrib/auth/hashers.py", line 115, in get_hasher
"setting?" % algorithm)
ValueError: Unknown password hashing algorithm '123'. Did you specify it in the PASSWORD_HASHERS setting?
You have directly stored the user passowrd as a plain string self.user.password = 123 but django stores user passwords using hashing algorithm that is why you are receiving the error. You can set user password by using set_password method of user which will apply hashing algorithm before saving it:
user.set_password('123')
user.save()