Related
Hoping to get some help on making calls to the Coinbase Pro API.
I created a key, noted my phasephrase, key and secret, and am running the below Python script. The response I get is "invalid signature".
On the CBPro documentation site, when I try running it with my credentials on this page, I get a "Sorry, you couldn't be authenticated with those credentails" message.
I've seen some sources that encode to base64 and have success, and others that don't, but neither works for me. What am I don't wrong?
Code:
import requests
import time
import base64
import json
url = "https://api.exchange.coinbase.com/accounts/account_id/transfers?limit=100"
key = "key"
secret = "secret"
passphrase = "pass"
timestamp = str(time.time())
headers = {
"Accept": "application/json",
"cb-access-key": key,
"cb-access-passphrase": passphrase,
"cb-access-sign": encodedData,
"cb-access-timestamp": timestamp
}
response = requests.request("GET", url, headers=headers)
print(response.text)
Signing a request is probably the worst part of the coinbase api.
Here is the documentation for it. some things to note:
signature is only good for 30 seconds so you have to hurry and copy / paste the encoded data and timestamp into the docs form.
the timestamp has to be the same value you use in the encodedData.
the non-pro api signs a little different so make sure you're on the right set of docs. This one I don't think works for pro.
If you're still having trouble authenticating your requests to coinbase here's an example of what i'm using. You might have to change a few things but does the job.
What you're looking for in this example are the HMACAuth & CoinbaseSession classes for your particular need.
# -*- coding: UTF-8 -*-
from base64 import b64encode, b64decode
from collections import namedtuple
from datetime import datetime, timezone
from hashlib import sha256
from hmac import HMAC
from json import loads, JSONDecodeError
from types import SimpleNamespace
from typing import Union, Generator
from requests import Session
from requests.adapters import HTTPAdapter
from requests.auth import AuthBase
from requests.exceptions import HTTPError
from requests.models import PreparedRequest
from requests.utils import to_native_string
from urllib3.util.retry import Retry
EXCHANGE: str = r"https://api.exchange.coinbase.com"
ACCOUNT = namedtuple(
"ACCOUNT",
(
"id",
"currency",
"balance",
"hold",
"available",
"profile_id",
"trading_enabled",
)
)
PRODUCT = namedtuple(
"PRODUCT",
(
"id",
"base_currency",
"quote_currency",
"base_min_size",
"base_max_size",
"quote_increment",
"base_increment",
"display_name",
"min_market_funds",
"max_market_funds",
"margin_enabled",
"fx_stablecoin",
"max_slippage_percentage",
"post_only",
"limit_only",
"cancel_only",
"trading_disabled",
"status",
"status_message",
"auction_mode",
)
)
def encode(value: Union[str, bytes]) -> bytes:
"""Encode the string `value` with UTF-8."""
if isinstance(value, str) is True:
value = value.encode("UTF-8")
return value
def decode(value: Union[bytes, str]) -> str:
"""Decode the bytes-like object `value` with UTF-8."""
if isinstance(value, bytes) is True:
value = value.decode("UTF-8")
return value
def req_time():
"""POSIX timestamp as float. Number of seconds since Unix Epoch in UTC."""
utc = get_utc()
return utc.timestamp()
def get_utc() -> datetime:
"""Construct a datetime object with UTC time zone info."""
return datetime.now(timezone.utc)
class TimeoutHTTPAdapter(HTTPAdapter):
"""Custom HTTP adapter with timeout capability."""
def __init__(self, *args, **kwargs):
self._timeout = kwargs.pop("timeout")
super(TimeoutHTTPAdapter, self).__init__(*args, **kwargs)
def send(self, request, **kwargs):
kwargs.update({"timeout": self._timeout})
return super(TimeoutHTTPAdapter, self).send(request, **kwargs)
class CoinbaseSession(Session):
"""Coinbase Session handle."""
_headers: dict = {
"Accept": "application/json",
"Content-Type": "application/json",
"Accept-Charset": "utf-8",
}
#staticmethod
def http_adapter(retries: int = 3, backoff: int = 1, timeout: int = 30):
return TimeoutHTTPAdapter(
max_retries=Retry(total=retries, backoff_factor=backoff),
timeout=timeout
)
def __init__(self):
super(CoinbaseSession, self).__init__()
self.headers.update(self._headers)
self.auth = HMACAuth()
self.mount("https://", self.http_adapter())
self.mount("http://", self.http_adapter())
class HMACAuth(AuthBase):
"""Requests signing handle."""
#staticmethod
def __pre_hash(timestamp: Union[str, int, float], request: PreparedRequest) -> bytes:
"""
Create the pre-hash string by concatenating the timestamp with
the request method, path_url and body if exists.
"""
message = f"{timestamp}{request.method.upper()}{request.path_url}"
body = request.body
if body is not None:
message = f"{message}{decode(body)}"
return encode(message)
#staticmethod
def __sign(message: bytes) -> bytes:
"""Create a sha256 HMAC and sign the required `message`."""
key = b64decode(encode(API.SECRET)) # be careful were you keep this!
hmac = HMAC(key=key, msg=message, digestmod=sha256).digest()
return b64encode(hmac)
def __call__(self, request: PreparedRequest):
timestamp = req_time()
message = self.__pre_hash(timestamp, request)
cb_access_sign = self.__sign(message)
request.headers.update(
{
to_native_string('CB-ACCESS-KEY'): API.KEY, # be careful where you keep this!
to_native_string('CB-ACCESS-SIGN'): cb_access_sign,
to_native_string('CB-ACCESS-TIMESTAMP'): str(timestamp),
to_native_string('CB-ACCESS-PASSPHRASE'): API.PASSPHRASE, # be careful where you keep this!
}
)
return request
class CoinbaseAPI(object):
"""Coinbase API handle."""
def __init__(self):
self._session = CoinbaseSession()
def request(self, **kwargs):
"""
Send HTTP requests to the Coinbase API.
Raises HTTPError if response is not 200.
"""
print(f"DEBUG: Requesting resource (url = {kwargs.get('url')})")
try:
results = self.__request(**kwargs)
except HTTPError as http_error:
print(f"ERROR: Resource not found!", f"Cause: {http_error}")
else:
print(f"DEBUG: Resource found (url = {kwargs.get('url')})")
return results
def __request(self, **kwargs):
"""
Send HTTP requests to the Coinbase API.
Raises HTTPError if response is not 200.
"""
method = getattr(self._session, kwargs.pop("method"))
response = method(**kwargs)
if response.status_code != 200:
response.raise_for_status()
else:
try:
results = loads(response.text)
except JSONDecodeError as json_error:
print("WARNING: Decoding JSON object failed!", f"Cause: {json_error}")
kwargs.update({"method": method.__name__})
return self.__request(**kwargs)
else:
return results
class Endpoints(object):
"""Coinbase server endpoints."""
_server = None
#staticmethod
def join(*path, **params) -> str:
"""
Construct the resource url by appending all `path`
items to base url and join `params` if any.
"""
url = "/".join(path)
if len(params) > 0:
params = [f"{key}={value}" for key, value in params.items()]
url = f"{url}?{'&'.join(params)}"
return url
class ExchangeEndpoints(Endpoints):
"""Coinbase exchange server endpoints."""
_server = EXCHANGE
def __init__(self):
self.products = self.join(self._server, "products")
self.accounts = self.join(self._server, "accounts")
class Exchange(CoinbaseAPI):
"""Coinbase exchange API client handle."""
_endpoints = ExchangeEndpoints()
def __init__(self):
super(Exchange, self).__init__()
def get_accounts(self) -> Generator:
"""Get a list of trading accounts from the profile of the API key."""
response = self._accounts()
for item in response:
yield ACCOUNT(**item)
def get_account(self, account_id: str) -> ACCOUNT:
"""
Information for a single account. Use this endpoint when you know the account_id.
API key must belong to the same profile as the account.
"""
response = self._accounts(account_id)
return ACCOUNT(**response)
def get_products(self, query: str = None) -> Generator:
"""
Gets a list of available currency pairs for trading.
:param query: `type` query parameter (unknown).
"""
response = self._products(type=query)
for item in response:
yield PRODUCT(**item)
def get_product(self, product_id: str) -> PRODUCT:
"""
Get information on a single product.
:param product_id: The `id` string of the product/currency pair (ex: BTC-USD).
"""
result = self._products(product_id)
return PRODUCT(**result)
def _accounts(self, *path, **params):
"""Access resources from the `accounts` endpoint of the exchange API."""
params = self.clean_params(**params)
_url = self._endpoints.join(
self._endpoints.accounts,
*path,
**params
)
return self.request(method="get", url=_url)
def _products(self, *path, **params):
"""Access resources from the `products` endpoint of the exchange API."""
params = self.clean_params(**params)
_url = self._endpoints.join(
self._endpoints.products,
*path,
**params
)
return self.request(method="get", url=_url)
def clean_params(self, **params) -> dict:
"""Clean `params` by removing None values."""
temp = dict()
for key, value in params.items():
if value is None:
continue
if isinstance(value, dict) is True:
value = self.clean_params(**value)
temp.update({key: value})
return temp
if __name__ == '__main__':
# we're using PBKDF2HMAC (with symmetrically derived encryption key)
# not included in this example
key_vault = KeyVault() # custom class for encrypting and storing secrets to keyring
key_vault.cypher.password(value="your_password", salt="salty_password")
api_credentials = loads(
key_vault.get_password("coinbase", "pro-coinbase-api")
)
# accounts:
ADA: str = "your_crypto_account_id" # ex: 8b9806a4-7395-11ec-9b1a-f02f74d9105d
API = SimpleNamespace(
NAME="pro-coinbase-api",
VERSION="2021-08-27",
KEY=api_credentials.get("key"),
PASSPHRASE=api_credentials.get("passphrase"),
SECRET=api_credentials.get("secret"),
) # not the best example but does the job as long as you don't invite hackers in your PC :)
api = Exchange()
accounts = api.get_accounts()
for account in accounts:
print(account)
account = api.get_account(account_id=ADA)
print(account)
products = api.get_products()
for product in products:
print(product)
product = api.get_product("ATOM-EUR")
print(product)
I have created a module for a Bacnet scan and it will respond with a list of devices and its address as a result. But I'm having trouble implementing a direct method handler in python. When i first tried implementing it myself i got this error. Which could mean I didn't successfully register the direct method callback. I have some references but it was from C# and azure docs is not helping me figure out the right method to register the callback. for IoTHubModuleClient there's a on_method_request_received and a receive_method_request. appreciate any help!
def iothub_client_scan_run():
try:
iot_client = iothub_client_init()
bacnet_scan_listener_thread = threading.Thread(target=device_method_listener, args=(iot_client,))
bacnet_scan_listener_thread.daemon = True
bacnet_scan_listener_thread.start()
while True:
time.sleep(1000)
def device_method_listener(iot_client):
while True:
# Receive the direct method request
method_request = iot_client.receive_method_request()
print (
"\nMethod callback called with:\nmethodName = {method_name}\npayload = {payload}".format(
method_name=method_request.name,
payload=method_request.payload
)
)
if method_request.name == "runBacnetScan":
response = bacnet_scan_device(method_request)
else:
response_payload = {"Response": "Direct method {} not defined".format(method_request.name)}
response_status = 404
# Send a method response indicating the method request was resolved
print('Sending method response')
iot_client.send_method_response(response)
print('Message sent!')
Edit:
Here is my route config
I was able to resolve my issue or at least find the root cause and it was my network configuration under the createOptions. It seems like there's an issue when I'm trying to do NetworkMode: host and connects to the IotModuleClient.connect_from_edge_environment via connect with connection string. I'm still trying to tweak the connection configuration but at least i know its not on the code.
async def method_request_handler(module_client):
while True:
method_request = await module_client.receive_method_request()
print (
"\nMethod callback called with:\nmethodName = {method_name}\npayload = {payload}".format(
method_name=method_request.name,
payload=method_request.payload
)
)
if method_request.name == "method1":
payload = {"result": True, "data": "some data"} # set response payload
status = 200 # set return status code
print("executed method1")
elif method_request.name == "method2":
payload = {"result": True, "data": 1234} # set response payload
status = 200 # set return status code
print("executed method2")
else:
payload = {"result": False, "data": "unknown method"} # set response payload
status = 400 # set return status code
print("executed unknown method: " + method_request.name)
# Send the response
method_response = MethodResponse.create_from_method_request(method_request, status, payload)
await module_client.send_method_response(method_response)
print('Message sent!')
def stdin_listener():
while True:
try:
selection = input("Press Q to quit\n")
if selection == "Q" or selection == "q":
print("Quitting...")
break
except:
time.sleep(10)
# Schedule task for C2D Listener
listeners = asyncio.gather(input1_listener(module_client), twin_patch_listener(module_client), method_request_handler(module_client))
I'm trying to do simple SOAP call with WSDL and authentication, but don't know really how to pass credentials to the call. Is this possible with WSDL, or should I do this some other way?
from SOAPpy import WSDL
WSDLFILE = 'link/to/wsdl/file'
server = WSDL.Proxy(WSDLFILE)
result = server.methodCall()
Right now, I'm getting this error:
File "/usr/lib/pymodules/python2.7/SOAPpy/Client.py", line 263, in call
raise HTTPError(code, msg)
SOAPpy.Errors.HTTPError: <HTTPError 401 Unauthorized>
Quite an old question but since it shows up first when googling "SOAPpy authentication" - I will leave my findings so maybe you won't have to bang your head for 10 hours. Giving back to community.
Googling does not provide much help but this sample here (http://code.activestate.com/recipes/444758-how-to-add-cookiesheaders-to-soappy-calls/) made me to write my own helper (version 0.0 beta)
from SOAPpy import (
WSDL,
HTTPTransport,
Config,
SOAPAddress,
)
import urllib2
import base64
class AuthenticatedTransport(HTTPTransport):
_username = None
_password = None
def call(self, addr, data, namespace, soapaction = None, encoding = None, http_proxy = None, config = Config, timeout = 10):
if not isinstance(addr, SOAPAddress):
addr = SOAPAddress(addr, config)
content_type = 'text/xml'
if encoding != None:
content_type += '; charset="%s"' % encoding
encoded_auth = None
if ( isinstance(AuthenticatedTransport._username, str) != False ):
if ( isinstance(AuthenticatedTransport._password, str) == False ):
AuthenticatedTransport._password = ""
encoded_auth = base64.b64encode('%s:%s' % (self._username, self._password))
this_request = None
if ( encoded_auth is not None ):
this_request = urllib2.Request(
url=addr.proto + "://" + addr.host + addr.path,
data=data,
headers={
"Content-Type":content_type,
"SOAPAction":"%s" % soapaction,
"Authorization":"Basic %s" % encoded_auth
}
)
else:
this_request = urllib2.Request(
url=addr.proto + "://" + addr.host + addr.path,
data=data,
headers={
"Content-Type":content_type,
"SOAPAction":"%s" % soapaction,
}
)
response = urllib2.urlopen(this_request)
data = response.read()
# get the new namespace
if namespace is None:
new_ns = None
else:
new_ns = self.getNS(namespace, data)
# return response payload
return data, new_ns
WSDL.Config.strictNamespaces = 0
username = "your_username"
password = "your_password"
WSDLFile = "https://%s:%s#some_host/your_wsdl.php?wsdl" % (username, password)
namespace = "http://futureware.biz/mantisconnect"
proxy = WSDL.Proxy(WSDLFile, namespace=namespace, transport = AuthenticatedTransport(username,password))
print(proxy.mc_get_version()) # This is my WSDL call, your will be different
For whatever reason it is not enough to use AuthenticatedTransport class, the WSDL url itself also has to contain user name and password. Maybe it's because SOAP wrapper called by WSDL here is creating separate HTTP session (remember reading about it on SOAPpy mailing list).
Hope this helps somebody.
In my project I'm handling all HTTP requests with python requests library.
Now, I need to query the http server using specific DNS - there are two environments, each using its own DNS, and changes are made independently.
So, when the code is running, it should use DNS specific to the environment, and not the DNS specified in my internet connection.
Has anyone tried this using python-requests? I've only found solution for urllib2:
https://stackoverflow.com/questions/4623090/python-set-custom-dns-server-for-urllib-requests
requests uses urllib3, which ultimately uses httplib.HTTPConnection as well, so the techniques from https://stackoverflow.com/questions/4623090/python-set-custom-dns-server-for-urllib-requests (now deleted, it merely linked to Tell urllib2 to use custom DNS) still apply, to a certain extent.
The urllib3.connection module subclasses httplib.HTTPConnection under the same name, having replaced the .connect() method with one that calls self._new_conn. In turn, this delegates to urllib3.util.connection.create_connection(). It is perhaps easiest to patch that function:
from urllib3.util import connection
_orig_create_connection = connection.create_connection
def patched_create_connection(address, *args, **kwargs):
"""Wrap urllib3's create_connection to resolve the name elsewhere"""
# resolve hostname to an ip address; use your own
# resolver here, as otherwise the system resolver will be used.
host, port = address
hostname = your_dns_resolver(host)
return _orig_create_connection((hostname, port), *args, **kwargs)
connection.create_connection = patched_create_connection
and you'd provide your own code to resolve the host portion of the address into an ip address instead of relying on the connection.create_connection() call (which wraps socket.create_connection()) to resolve the hostname for you.
Like all monkeypatching, be careful that the code hasn't significantly changed in later releases; the patch here was created against urllib3 version 1.21.1. but should work for versions as far back as 1.9.
Note that this answer was re-written to work with newer urllib3 releases, which have added a much more convenient patching location. See the edit history for the old method, applicable to version < 1.9, as a patch to the vendored urllib3 version rather than a stand-alone installation.
You should look into the TransportAdapters, including the source code. The documentation on them isn't great, but they give low-level access to a lot of the functionality described in RFC 2818 and RFC 6125. In particular, those documents encourage (require?) client-side code to support application-specific DNS for the purpose of checking certificates' CommonName and SubjectAltName. The keyword argument you need in those calls is "assert_hostname". Here's how to set it with the requests library:
from requests import Session, HTTPError
from requests.adapters import HTTPAdapter, DEFAULT_POOLSIZE, DEFAULT_RETRIES, DEFAULT_POOLBLOCK
class DNSResolverHTTPSAdapter(HTTPAdapter):
def __init__(self, common_name, host, pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE,
max_retries=DEFAULT_RETRIES, pool_block=DEFAULT_POOLBLOCK):
self.__common_name = common_name
self.__host = host
super(DNSResolverHTTPSAdapter, self).__init__(pool_connections=pool_connections, pool_maxsize=pool_maxsize,
max_retries=max_retries, pool_block=pool_block)
def get_connection(self, url, proxies=None):
redirected_url = url.replace(self.__common_name, self.__host)
return super(DNSResolverHTTPSAdapter, self).get_connection(redirected_url, proxies=proxies)
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
pool_kwargs['assert_hostname'] = self.__common_name
super(DNSResolverHTTPSAdapter, self).init_poolmanager(connections, maxsize, block=block, **pool_kwargs)
common_name = 'SuperSecretSarahServer'
host = '192.168.33.51'
port = 666
base_url = 'https://{}:{}/api/'.format(common_name, port)
my_session = Session()
my_session.mount(self.base_url.lower(), DNSResolverHTTPSAdapter(common_name, host))
user_name = 'sarah'
url = '{}users/{}'.format(self.base_url, user_name)
default_response_kwargs = {
'auth': (NAME, PASSWORD),
'headers': {'Content-Type': 'application/json'},
'verify': SSL_OPTIONS['ca_certs'],
'cert': (SSL_OPTIONS['certfile'], SSL_OPTIONS['keyfile'])
}
response = my_session.get(url, **default_response_kwargs)
I use common_name for the name expected to be on the certificate and how your code will reference the desired machine. I use host for a name recognized by the external world - FQDN, IP, DNS entry, ... Of course, the SSL_OPTIONS dictionary (in my example) must list appropriate certificate / key filenames on your machine. (Plus, NAME and PASSWORD should resolve to correct strings.)
A customized HTTPAdapter will do the trick.
Don't forget to set server_hostname to enable SNI.
import requests
class HostHeaderSSLAdapter(requests.adapters.HTTPAdapter):
def resolve(self, hostname):
# a dummy DNS resolver
import random
ips = [
'104.16.89.20', # CloudFlare
'151.101.2.109', # Fastly
]
resolutions = {
'cdn.jsdelivr.net': random.choice(ips),
}
return resolutions.get(hostname)
def send(self, request, **kwargs):
from urllib.parse import urlparse
connection_pool_kwargs = self.poolmanager.connection_pool_kw
result = urlparse(request.url)
resolved_ip = self.resolve(result.hostname)
if result.scheme == 'https' and resolved_ip:
request.url = request.url.replace(
'https://' + result.hostname,
'https://' + resolved_ip,
)
connection_pool_kwargs['server_hostname'] = result.hostname # SNI
connection_pool_kwargs['assert_hostname'] = result.hostname
# overwrite the host header
request.headers['Host'] = result.hostname
else:
# theses headers from a previous request may have been left
connection_pool_kwargs.pop('server_hostname', None)
connection_pool_kwargs.pop('assert_hostname', None)
return super(HostHeaderSSLAdapter, self).send(request, **kwargs)
url = 'https://cdn.jsdelivr.net/npm/bootstrap/LICENSE'
session = requests.Session()
session.mount('https://', HostHeaderSSLAdapter())
r = session.get(url)
print(r.headers)
r = session.get(url)
print(r.headers)
I know this is an old thread but here is my python3 compatible solution using tldextract and dnspython. I've left some commented out code to illustrate how to debug and set additional session parameters.
#!/usr/bin/env python3
import sys
from pprint import pprint as pp
import requests
import dns.resolver # NOTE: dnspython package
import tldextract
class CustomAdapter(requests.adapters.HTTPAdapter):
def __init__(self, nameservers):
self.nameservers = nameservers
super().__init__()
def resolve(self, host, nameservers, record_type):
dns_resolver = dns.resolver.Resolver()
dns_resolver.nameservers = nameservers
answers = dns_resolver.query(host, record_type)
for rdata in answers:
return str(rdata)
def get_connection(self, url, proxies=None):
ext = tldextract.extract(url)
fqdn = ".".join([ ext.subdomain, ext.domain, ext.suffix ])
print("FQDN: {}".format(fqdn))
a_record = self.resolve(fqdn, nameservers, 'A')
print("A record: {}".format(a_record))
resolved_url = url.replace(fqdn, a_record) # NOTE: Replace first occurrence only
print("Resolved URL: {}".format(resolved_url))
return super().get_connection(resolved_url, proxies=proxies)
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: {} <url>".format(sys.argv[0]))
sys.exit(0)
url = sys.argv[1]
nameservers = [
'208.67.222.222', # NOTE: OpenDNS
'8.8.8.8' # NOTE: Google
]
session = requests.Session()
session.mount(url, CustomAdapter(nameservers))
parameters = {
# "headers": {'Content-Type': 'application/json'},
# "timeout" : 45,
# "stream" : True
# "proxies" : {
# "http": "http://your_http_proxy:8080/",
# "https": "http://your_https_proxy:8081/"
# },
# "auth": (name, password),
# ...
}
response = session.get(url, **parameters)
pp(response.__dict__)
And here it the console output :
$ ./run.py http://www.test.com
FQDN: www.test.com
A record: 69.172.200.235
Resolved URL: http://69.172.200.235/
{'_content': b'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3'
b'.org/TR/html4/strict.dtd">\n<html>\n<head>\n<meta http-equiv="C'
b'ontent-Type" content="text/html; charset=iso-8859-1">\n<meta '
b'http-equiv="Content-Script-Type" content="text/javascript">\n'
b'<script type="text/javascript">\nfunction getCookie(c_name) {'
b' // Local function for getting a cookie value\n if (docume'
b'nt.cookie.length > 0) {\n c_start = document.cookie.in'
b'dexOf(c_name + "=");\n if (c_start!=-1) {\n c_st'
b'art=c_start + c_name.length + 1;\n c_end=document.cook'
b'ie.indexOf(";", c_start);\n\n if (c_end==-1) \n '
b' c_end = document.cookie.length;\n\n return unescape('
b'document.cookie.substring(c_start,c_end));\n }\n }\n '
b' return "";\n}\nfunction setCookie(c_name, value, expiredays'
b') { // Local function for setting a value of a cookie\n va'
b'r exdate = new Date();\n exdate.setDate(exdate.getDate()+e'
b'xpiredays);\n document.cookie = c_name + "=" + escape(valu'
b'e) + ((expiredays==null) ? "" : ";expires=" + exdate.toGMTString'
b'()) + ";path=/";\n}\nfunction getHostUri() {\n var loc = doc'
b"ument.location;\n return loc.toString();\n}\nsetCookie('YPF8"
b"827340282Jdskjhfiw_928937459182JAX666', '171.68.244.56', 10)"
b';\ntry { \n location.reload(true); \n} catch (err1) { \n '
b' try { \n location.reload(); \n } catch (err2) { '
b' \n \tlocation.href = getHostUri(); \n } \n}\n</scrip'
b't>\n</head>\n<body>\n<noscript>This site requires JavaScript an'
b'd Cookies to be enabled. Please change your browser settings or '
b'upgrade your browser.</noscript>\n</body>\n</html>\n',
'_content_consumed': True,
'_next': None,
'connection': <requests.adapters.HTTPAdapter object at 0x109130e48>,
'cookies': <RequestsCookieJar[]>,
'elapsed': datetime.timedelta(microseconds=992676),
'encoding': 'ISO-8859-1',
'headers': {'Server': 'nginx/1.14.2', 'Date': 'Wed, 01 May 2019 18:01:58 GMT', 'Content-Type': 'text/html', 'Transfer-Encoding': 'chunked', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=20', 'X-DIS-Request-ID': '2a5057a7c7b8a93dd700856c48fda74a', 'P3P': 'CP="NON DSP COR ADMa OUR IND UNI COM NAV INT"', 'Cache-Control': 'no-cache', 'Content-Encoding': 'gzip'},
'history': [<Response [302]>],
'raw': <urllib3.response.HTTPResponse object at 0x1095b90b8>,
'reason': 'OK',
'request': <PreparedRequest [GET]>,
'status_code': 200,
'url': 'https://www.test.com/'}
Hope this helps.
I have a Flask integration test backed by a 1-node mongodb that randomly fails:
pytest/test_webapi.py:59: in test_register_test
> assert res.status_code == 302
E assert <Response streamed [404 NOT FOUND]>.status_code == 302
Fail rate is roughly 50%.
Test in test_webapi.py looking thus:
def test_register_user(self):
res = self.client.get("/logout")
class MySMTPServer(smtpd.SMTPServer):
mails = []
def process_message(self, peer, mailfrom, rcpttos, data):
self.mails.append((rcpttos[0], data))
server = MySMTPServer(('localhost', 12345), None)
t = threading.Thread(target=asyncore.loop, args=(1,))
t.start()
time.sleep(.1)
try:
res = self.client.post("/register", data=self.registration)
assert res.status_code == 200
mail, hash = server.mails[0]
self.conn.fsync()
time.sleep(.1)
res = self.client.get('/activate/' + hash)
assert res.status_code == 302
finally:
server.close()
The relevant Flask methods from webapi.py:
#app.route("/register", methods=["POST"])
def register_user():
mail = flask.request.form['mail']
user = flask.request.form["user"]
pw = flask.request.form["pass"]
hash = users.register(user, pw, mail=mail)
return flask.jsonify({'_id': None}) # XXX
#app.route('/activate/<hash>', methods=['GET'])
def activate_user(hash):
key = users.activate(hash=hash)
if not key:
flask.abort(404)
return flask.redirect("/")
... are backed by action methods:
make_key = lambda : base64.encodestring(os.urandom(32)).strip()
def register(self, user, pw, **kw):
hash = self.make_key()
user = self.new(user, pw, activation=hash, **kw)
self._send_mail(**user)
return hash
def activate(self, hash):
user = self.users.find_one({'activation': hash})
if not user:
return None
key = self.make_key()
activation = {
'$unset': {'activation': 1},
'$set': {'status': 'active', 'key': key} }
self.users.update({'_id': user['_id']}, activation)
return user
... where self.users is a mongodb collection.
self.new() persists the entity using safe=True.
Interestingly, several other tests doing similar things never seem to encounter this problem.
I had thought that this would be enough to make sure that the persisted object would be visible to other threads in the pymongo connection pool. What part of the mongodb/pymongo documentation should I have read more carefully? Or is there some weird interaction with asyncore?
(ported from comments thread)
In the context of the tests, is the data value just the base64-encoded string of the activation key? base64 contains letters and digits, but also "+" and "/", both of which will be misinterpreted by URL parsers (particularly "/").