We Keep Coding

Unable to import a function/class from wheel file

I have created a wheel file and I m trying to invoke the method in synapse but am unable to do so.On trying to call the method _buildJSONdata() using below
import cloudapi
from cloudapi import cloudalert
api=cloudalert.CloudLink()
Data = {"Name": "ABC","email":"df#example.com","subject":"Alert"}
api._buildJSONdata(Data)
I am getting the error AttributeError: 'CloudLink' object has no attribute 'Data'.I have already defined the argument inside the definition and already using the data variable while invoking the said method
Contents of __init__.py
from .cloudalert import CloudLink
Contents of setup.py
import setuptools
with open("README.md", "r") as fh:
long_description = fh.read()
setuptools.setup(
name="CloudAPI",
version="0.0.1",
author="XYZ",
author_email="abc#example.com",
description="Package expose Cloud as library",
long_description=long_description,
long_description_content_type="text/markdown",
packages=setuptools.find_packages(),
install_requires=["adal"],
classifiers=["Programming Language :: Python :: 3",],
python_requires='>=3.7',
)
Folder structure
Below is cloudalert.py content
import json
import datetime
import hashlib
import json
import sys
import traceback
import adal
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import logging
from functools import wraps
import sys
def create_logger():
#create a logger object
#logger = logging.getLogger()
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logfile = logging.FileHandler('exc_logger.log')
#logfile = logging.StreamHandler(sys.stdout)
fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
formatter = logging.Formatter(fmt)
logfile.setFormatter(formatter)
logger.addHandler(logfile)
return logger
logger = create_logger()
def exception(logger):
def decorator(func):
#wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
issue = "exception in "+func.__name__+"\n"
issue = issue+"-------------------------\
------------------------------------------------\n"
logger.exception(issue)
raise
return wrapper
return decorator
class CloudLink(object):
_token = None
_instance = None
http = None
#staticmethod
def getInstance():
if not CloudLink._instance:
CloudLink._instance = CloudLink()
return CloudLink._instance
def __init__(self):
retry_strategy = Retry(
total=3,
backoff_factor=0,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["HEAD", "GET", "OPTIONS"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
self.http = requests.Session()
self.http.mount("https://", adapter)
self.http.mount("http://", adapter)
print("Inside init")
def parseJSON(self, t):
try:
eData = json.loads(t)
logger.info(f"Sending {eData} to Cloud")
self.sendToCloud(eData)
except ValueError as e:
print("Error: %s Please validate JSON in https://www.jsonschemavalidator.net/"% e)
return None # or: raise
def sendToCloud(self, eData):
#from notebookutils import mssparkutils
cloudtest = 'https://www.example.com/api/v1/event/fg'
cloudData = {"eData": eData, "metadata": self._buildMetadata()}
logger.info(f"Raising alert with data=({cloudData}")
response = self.http.post(
self.cloudtest, headers=self._buildHeaders(), json=cloudData
)
logger.info(f"cloud alert response={response}")
if response.status_code == 202 or response.status_code == 200:
logger.info("Mail sent to Cloud")
else:
raise Exception(f"Cloud reporting failed with Error {response}")
def _buildJSONdata(self,Data):
if len(self.Data) == 0:
raise Exception("JSON is empty")
else:
t = json.dumps(self.Data)
self.parseJSON(t)
def _buildMetadata(self):
return {
"messageType": "Send Email",
"messageVersion": "0.0.1",
"sender": "Send Email",
}
def _buildHeaders(self):
self._refreshADToken()
headers = {
"Authorization": "Bearer {}".format(self._token["accessToken"]),
"Content-type": "application/json",
"Accept": "text/plain",
}
return headers
def _refreshADToken(self):
def shouldRenew(token):
"""Returns True if the token should be renewed"""
expiresOn = datetime.datetime.strptime(
token["expiresOn"], "%Y-%m-%d %H:%M:%S.%f"
)
now = datetime.datetime.now()
return (expiresOn - now) < datetime.timedelta(minutes=5)
if not self._token or shouldRenew(self._token):
logger.info("Renewing credentials for Alerting")
result = None
try:
#from notebookutils import mssparkutils
cloudclient = "xxxxxxx"
clientid = "yyyyyyyyy"
clientcredentials = "ccccccccc"
authority_url = "https://login.microsoftonline.com/dddddddddd/"
context = adal.AuthenticationContext(authority_url)
result = context.acquire_token_with_client_credentials(cloudclient, clientid,clientcredentials)
except Exception as e:
error = "Failed to renew client credentials."
logger.info(error)
raise
if result and "accessToken" in result:
self._token = result
else:
logger.error(
"Failed to acquire bearer token. accessToken not found in result object on renewing credentials."
)
raise Exception("Could not acquire a bearer token")
Let me know in case if I m doing something wrong while invoking the function.

Can't get python exe to run

So i'm having issues getting my python 3.5 script to run as an exe. Here is the code I am using:
import base64
import imaplib
import json
import smtplib
import urllib.parse
import urllib.request
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import lxml.html
GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
GOOGLE_CLIENT_ID = '<FILL ME IN>'
GOOGLE_CLIENT_SECRET = '<FILL ME IN>'
GOOGLE_REFRESH_TOKEN = None
def command_to_url(command):
return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)
def url_escape(text):
return urllib.parse.quote(text, safe='~-._')
def url_unescape(text):
return urllib.parse.unquote(text)
def url_format_params(params):
param_fragments = []
for param in sorted(params.items(), key=lambda x: x[0]):
param_fragments.append('%s=%s' % (param[0], url_escape(param[1])))
return '&'.join(param_fragments)
def generate_permission_url(client_id, scope='https://mail.google.com/'):
params = {}
params['client_id'] = client_id
params['redirect_uri'] = REDIRECT_URI
params['scope'] = scope
params['response_type'] = 'code'
return '%s?%s' % (command_to_url('o/oauth2/auth'), url_format_params(params))
def call_authorize_tokens(client_id, client_secret, authorization_code):
params = {}
params['client_id'] = client_id
params['client_secret'] = client_secret
params['code'] = authorization_code
params['redirect_uri'] = REDIRECT_URI
params['grant_type'] = 'authorization_code'
request_url = command_to_url('o/oauth2/token')
response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode('UTF-8')).read().decode('UTF-8')
return json.loads(response)
def call_refresh_token(client_id, client_secret, refresh_token):
params = {}
params['client_id'] = client_id
params['client_secret'] = client_secret
params['refresh_token'] = refresh_token
params['grant_type'] = 'refresh_token'
request_url = command_to_url('o/oauth2/token')
response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode('UTF-8')).read().decode('UTF-8')
return json.loads(response)
def generate_oauth2_string(username, access_token, as_base64=False):
auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
if as_base64:
auth_string = base64.b64encode(auth_string.encode('ascii')).decode('ascii')
return auth_string
def test_imap(user, auth_string):
imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
imap_conn.debug = 4
imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
imap_conn.select('INBOX')
def test_smpt(user, base64_auth_string):
smtp_conn = smtplib.SMTP('smtp.gmail.com', 587)
smtp_conn.set_debuglevel(True)
smtp_conn.ehlo('test')
smtp_conn.starttls()
smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64_auth_string)
def get_authorization(google_client_id, google_client_secret):
scope = "https://mail.google.com/"
print('Navigate to the following URL to auth:', generate_permission_url(google_client_id, scope))
authorization_code = input('Enter verification code: ')
response = call_authorize_tokens(google_client_id, google_client_secret, authorization_code)
return response['refresh_token'], response['access_token'], response['expires_in']
def refresh_authorization(google_client_id, google_client_secret, refresh_token):
response = call_refresh_token(google_client_id, google_client_secret, refresh_token)
return response['access_token'], response['expires_in']
def send_mail(fromaddr, toaddr, subject, message):
access_token, expires_in = refresh_authorization(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, GOOGLE_REFRESH_TOKEN)
auth_string = generate_oauth2_string(fromaddr, access_token, as_base64=True)
msg = MIMEMultipart('related')
msg['Subject'] = subject
msg['From'] = fromaddr
msg['To'] = toaddr
msg.preamble = 'This is a multi-part message in MIME format.'
msg_alternative = MIMEMultipart('alternative')
msg.attach(msg_alternative)
part_text = MIMEText(lxml.html.fromstring(message).text_content().encode('utf-8'), 'plain', _charset='utf-8')
part_html = MIMEText(message.encode('utf-8'), 'html', _charset='utf-8')
msg_alternative.attach(part_text)
msg_alternative.attach(part_html)
server = smtplib.SMTP('smtp.gmail.com:587')
server.ehlo(GOOGLE_CLIENT_ID)
server.starttls()
server.docmd('AUTH', 'XOAUTH2 ' + auth_string)
server.sendmail(fromaddr, toaddr, msg.as_string())
server.quit()
def main():
if GOOGLE_REFRESH_TOKEN is None:
print('No refresh token found, obtaining one')
refresh_token, access_token, expires_in = get_authorization(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET)
print('Set the following as your GOOGLE_REFRESH_TOKEN:', refresh_token)
exit()
send_mail('--------#gmail.com', '--------#gmail.com',
'A mail from you from Python',
'<b>A mail from you from Python</b><br><br>' +
'So happy to hear from you!')
input("Success!")
print(__name__)
input("Wait!!")
if __name__ == '__main__':
main()
When i run the code in pyCharm is works great, when i use pyinstaller it creates the exe. When i run the EXE it will open a new cmd prmpt with no text in it, then it closes. Any ideas why it doesn't print the name or "Wait!!" on the screen?
I named the file main and it works great in PyCharm.
I'm not familiar with troubleshooting code that works in an IDE and does nothing as an exe. Is there a basic troubleshooting guide for that?

Most of the problems are errors caused by imported modules.
Here are two ways I think of
Use exception handling and use traceback.format_exc() to display the contents when an error occurs.
Errors are usually associated with the sys.stderr, so you can redirect stderr output a specified format to a specified file or the like.
Example:
import sys
from pathlib import Path
from os import startfile
import traceback
DEBUG = True
class CaptureStderr:
__slots__ = ('debug_flag',
'original',
'log',)
def __init__(self, debug_flag=False, log=None):
self.debug_flag = debug_flag
self.original = sys.stderr # Keep track of original
self.log = log
def new_log_file(self, log_path: Path = Path('temp.log').resolve()):
self.log = open(str(log_path), 'w')
return self.log
def start(self):
""" Start filtering and redirecting stderr """
sys.stderr = self
def stop(self):
""" Stop filtering and redirecting stderr """
sys.stderr = self.original
def write(self, message: str):
""" When sys.stderr.write is called, it will re directed here"""
if not self.debug_flag:
self.original.write(message)
self.original.flush()
return
if self.log:
self.log.write(message)
def main():
...
if __name__ == '__main__':
try: # method 1
from .notexistmodule import notexistclass
# Put modules that you are not sure whether it is 100% import successful on here.
main()
except:
print(traceback.format_exc())
# method 2: redirect sys.stderr
cs = CaptureStderr(DEBUG)
cs.new_log_file(Path('error.log'))
cs.start()
from .notexistmodule2 import notexistclass2
main()
cs.stop()
startfile(cs.log.name)

You probably have an error in the exe, most likely an import error.
Run your exe like this to see the error:
Create a .bat file to run the exe with the following commands
start C:\Path\To\.exe
pause
This will stop the command prompt from exiting until you press a key on the keyboard, letting you read the output.

Download and handle errors

I've been using a function that I took from the book Web Scraping with Python from O'Really by Ryan Mitchell:
import sys
import os.path
import socket
import random
import urllib2
import contextlib
import diskCache
import logging as logger
from bs4 import BeautifulSoup
DEFAULT_AGENT = 'Mozilla/5.0 Firefox/56.0'
DEFAULT_DELAY = 3
DEFAULT_RETRIES = 10
DEFAULT_TIMEOUT = 60
socket.setdefaulttimeout (DEFAULT_TIMEOUT)
def download (url, delay=DEFAULT_DELAY, user_agent=DEFAULT_AGENT, proxies=None, \
cache=None, num_retries=DEFAULT_RETRIES, timeout=DEFAULT_TIMEOUT, data=None):
result = None
if cache:
try:
result = cache[url]
except KeyError:
# url is not available in cache
pass
if result is not None and result['code'] is not None \
and num_retries > 0 and 500 <= result['code'] < 600:
# server error so ignore result from cache and re-download
result = None
if result is None:
proxy = random.choice(proxies) if proxies else None
headers = {'User-agent': user_agent}
result = call (url, headers, proxy=proxy, num_retries=num_retries, cache=cache)
if cache:
# save result to cache
cache[url] = result
return result['html']
def call (url, headers, proxy, num_retries, cache=None, data=None):
request = urllib2.Request(url, data, headers or {})
with contextlib.closing (urllib2.urlopen(request)) as connection:
try:
logger.info ('Downloading: %s', url)
html = connection.read ()
code = connection.getcode ()
except Exception as e:
logger.exception ('Download error:', str(e))
if cache:
del cache['url']
html = None
if hasattr (e, 'code'):
code = e.code
if num_retries > 0 and 500 <= code < 600:
return download (url, headers, num_retries-1, data) # retry server errors
else:
code = None
return {'html': html, 'code':code}
I wanted to know if there is a simpler way of handling the errors when downloading urls. I've seen that the requests library is a higher level and easier library and maybe it could simplify this. At the very least how would this code be for python3?
It would be something like
"""Functions used by the fetch module"""
# Standard library imports
import time
import socket
import logging as logger
from typing import Dict, Optional
# Third party imports
import requests
from requests.exceptions import HTTPError, Timeout
from bs4 import BeautifulSoup
# Constants
DEFAULT_AGENT = 'Mozilla/5.0 Firefox/56.0'
DEFAULT_DELAY = 3
DEFAULT_RETRIES = 10
DEFAULT_TIMEOUT = 60
socket.setdefaulttimeout(DEFAULT_TIMEOUT)
def fetch(url: str, retries: Optional[int] = DEFAULT_RETRIES) -> Dict:
"""Download an url"""
code = None
try:
logger.info('Downloading: %s', url)
resp = requests.get(url)
resp.raise_for_status()
code = resp.status_code
except (HTTPError, Timeout) as ex:
logger.exception("Couldn't download %s", ex)
return None
if code is not None and retries > 0 and \
500 <= code < 600: # Server error
logger.info('Retrying download')
time.sleep(DEFAULT_DELAY)
return fetch(url, retries-1)
return {'html': resp, 'code': code}

As you said this is a lot easier with requests
resp = requests.get(url, headers=headers, timeout=timeout)
print(resp.status_code)
print(resp.text)
# for an API use resp.json()
There is no exception raised by default. You can call resp.raise_for_status() if you do want to raise an exception.
See http://docs.python-requests.org/en/master/user/quickstart/ for details

How to login into an website which is authenticated by google + API using python?

How to get logged into some website using python code
i.e www.example.com/auth/gmail which is taking advantage of google+ API for user logins. Now I would like to login with my credentials (Gmail or What?) by using python code. Please help me how to approach towards this problem.
Thanks

Here you have an example by using the google-api-python-client
First of all you have to create your client secrets on the google developer console.
First time you execute the code you will have to authorize the script to use your gmail account, then you can save the credentials on a file as on the example, and use it for the future executions without requiring this authentication
from googleapiclient.discovery import build
from oauth2client import client
from googleapiclient.errors import HttpError
import base64
from email.mime.text import MIMEText
from apiclient import errors
from oauth2client import client
import httplib2
from googleapiclient.discovery import build
def getCredentials(secrets, scope,filename):
flow = client.flow_from_clientsecrets(
secrets,
scope=scope,
redirect_uri='urn:ietf:wg:oauth:2.0:oob')
auth_uri = flow.step1_get_authorize_url()
webbrowser.open(auth_uri)
auth_code = raw_input('Enter the auth code: ')
credentials = flow.step2_exchange(auth_code)
saveJson(filename,credentials.to_json())
def saveJson(filename, object):
with open(filename, 'w') as f:
json.dump(object, f)
def openJson(filename):
with open(filename, 'r') as f:
object = json.load(f)
return object
if __name__=='__main__':
client_secrets = 'client_secrets.json' #client secrets to use the API
credentials = 'auth_credentials.json'
if(firstRun) #create a file with the auth credentials
scope = 'https://www.googleapis.com/auth/gemail.send'
getCredentials(secrets,scope,credentials)
cre = client.Credentials.new_from_json(openJson(credentials))
http_auth = cre.authorize(httplib2.Http())
gmail = build('gmail', 'v1', http=http_auth)
#gmail.doSomething

this is old(sandbox) and is done really fast so, you have to refactor the code
import re
import sys
import imaplib
import getpass
import email
import datetime
import string
import get_mail_search
from sys import stdout
M = imaplib.IMAP4_SSL('imap.gmail.com')
class Get_mail(object):
"""docstring for Get_mail"""
def __init__(self, *args):
super(Get_mail, self).__init__()
c=1
self.login(c)
self.toast_msg()
raw_input()
def toast_msg(self, *args):
"""docstring for Get_mail"""
M = self.mailbox()
stdout.write("\n{}\n".format(get_mail_search.search_help_info))
serach_input = raw_input()
rv, data = M.search(None, serach_input)
if rv != 'OK':
print "No messages found!"
id_ls = data[0].split()
rev_id_ls = [i for i in reversed(id_ls)]
if rev_id_ls:
for o in rev_id_ls:
try:
msg_content = self.process_mailbox(M, o)
_date_ = msg_content[0]
_from_ = msg_content[1]
_to_ = msg_content[2]
_subject_ = msg_content[3]
_msg_ = msg_content[4]
stdout.write("$$$$$$$$$$$\nDate: {}\nFrom: {}\nTo: {}\nSubject: {}\nMSG: {}\n".format(_date_,_from_,_to_,_subject_,_msg_))
except Exception, e:
pass
else:
stdout.write("No {} Mail Found!".format(serach_input))
raw_input()
self.toast_msg()
def login(self, try_c, *args):
"""docstring for Get_mail"""
try:
stdout.write("\nMail:\n")
mail = raw_input()
if mail:
M.login(str(mail), getpass.getpass())
else:
sys.exit(1)
except imaplib.IMAP4.error:
if try_c<=3:
stdout.write("Versuch: {}/3\n".format(try_c))
stdout.write("Die eingegebene E-Mail-Adresse und das Passwort stimmen nicht uberein. Nochmal versuchen")
try_c+=1
self.login(try_c)
else:
sys.exit(1)
def mailbox(self, *args):
"""docstring for Get_mail"""
rv, mailboxes = M.list()
if rv == 'OK':
for menu in mailboxes:
print('{}'.format(menu))
rv, data = M.select("inbox")
if rv == 'OK':
return M
def eval_decode(self, header, *args):
"""docstring for Get_mail"""
return email.Header.decode_header(header)[0]
def process_mailbox(self, M, num, *args):
"""docstring for Get_mail"""
rv, header = M.fetch(num, '(RFC822)')
if rv != 'OK':
print "ERROR getting message", num
header_msg = email.message_from_string(header[0][1])
if header_msg.is_multipart():
body=[payload.get_payload(decode=True) for payload in header_msg.get_payload()]
else:
body=payload.get_payload(decode=True)
from_decode = self.eval_decode(header_msg['From'])
subject_decode = self.eval_decode(header_msg['Subject'])
date_decode = self.eval_decode(header_msg['Date'])
to_decode = self.eval_decode(header_msg['To'])
return (date_decode[0], from_decode[0], to_decode[0], subject_decode[0], str(body[0]))
def run():
try:
Get_mail()
except KeyboardInterrupt:
M.close()
M.logout()
sys.exit(1)
run()

Python HTTP Request

I used this script
from twisted.internet import reactor, threads
from urlparse import urlparse
import httplib
import itertools
concurrent = 200
finished=itertools.count(1)
reactor.suggestThreadPoolSize(concurrent)
def getStatus(ourl):
url = urlparse(ourl)
conn = httplib.HTTPConnection(url.netloc)
conn.request("HEAD", url.path)
res = conn.getresponse()
return res.status
def processResponse(response,url):
print response, url
processedOne()
def processError(error,url):
print "error", url#, error
processedOne()
def processedOne():
if finished.next()==added:
reactor.stop()
def addTask(url):
req = threads.deferToThread(getStatus, url)
req.addCallback(processResponse, url)
req.addErrback(processError, url)
added=0
for url in open('urllist.txt'):
added+=1
addTask(url.strip())
try:
reactor.run()
except KeyboardInterrupt:
reactor.stop()
when i try to run the script $ python test.py
it just print the url not do cUrl or send HTTP request ..
how could I send the HTTP or cURL process for each one
Thanks

This should work if if the format of your urls does not contain 'http://' However,
If they do contain 'http://' there is a solution for that in the comments
import httplib
def requester(url):
host = url.split('/')[0]
#if urls do contain 'http://' --> host = url.split('/')[2].replace('http://','')
req = url[url.find(host)+len(host):]
conn = httplib.HTTPConnection(host)
conn.request("HEAD","/"+req)
response = conn.getresponse()
print response.status, response.reason
#if you want data...
#data = response.read()
#print data
for url in open(urls.txt):
try:
requester(url)
except Error,e:
print Error, e
Furthermore, I reccomend checking out the httplib

Tested code, using inlineCallbacks and deferToThread. Also using defer.gatherResults to know when all the deferreds have been processed (instead of the counter method in the OP):
from twisted.internet import reactor, defer, utils
from twisted.internet.threads import deferToThread
from urlparse import urlparse
import httplib
threadDeferred = deferToThread.__get__
#threadDeferred
def get_url_head(url_arg):
url = urlparse(url_arg)
conn = httplib.HTTPConnection(url.netloc)
conn.request("HEAD", url.path)
res = conn.getresponse()
conn.close()
return res.status
#defer.inlineCallbacks
def check_url(sem,url_arg):
yield sem.acquire()
try:
result = yield get_url_head(url_arg)
defer.returnValue(result)
finally:
sem.release()
#defer.inlineCallbacks
def run(reactor,SEMAPHORE_SIZE=10):
sem = defer.DeferredSemaphore(SEMAPHORE_SIZE)
deferreds = []
failed_urls = []
responded_urls = []
with open('urllist.txt','r') as f:
for line in f:
url_arg = line.strip()
d = check_url(sem,url_arg)
d.addCallback(processResult,url_arg,responded_urls).addErrback(processErr,url_arg,failed_urls)
deferreds.append(d)
res = yield defer.gatherResults(deferreds)
# Do something else with failed_urls and responded_urls
reactor.callLater(0,reactor.stop)
def main():
from twisted.internet import reactor
reactor.callWhenRunning(run,reactor)
reactor.run()
def processResult(result,url_arg,responded_urls):
print "Reponse %s from %s" % (result,url_arg)
responded_urls.append((url_arg,result))
def processErr(err,url_arg,failed_urls):
print "Error checking %s: %s" % (url_arg,repr(err.value))
failed_urls.append((url_arg,err.value))
if __name__ == '__main__':
main()