We Keep Coding

How to make sure the logged message goes to a new line?

I'm trying to make my own logging system because I don't like Python's one, but I ran into a small problem. Whenever I log two or more things, only the last log is written. Example - I log 'code initialized' at the start of the program. Then, I log 'exiting'. The result in the log file is only 'exiting' How do I fix this? Here is my code.
File 1.
from functions import *
from variables import *
try:
open('key.txt', "r")
key = True
except FileNotFoundError:
key = False
if check_internet():
x = status_boot[0]
elif not check_internet():
x = status_boot[1]
log("archer initialized")
if key:
print("archer status: " + x)
elif not key:
print(status_boot[4])
File 2.
from datetime import datetime
from variables import *
import requests
def time():
now = datetime.now()
return now.strftime("%H:%M:%S")
def check_internet():
timeout = 5
try:
_ = requests.get(url, timeout=timeout)
return True
except requests.ConnectionError:
return False
def log(m):
write = open('archerlog.txt', 'w+')
return write.write(f'[{time()}] archer: {m}\n')
File 3.
url = "http://www.google.com/"
status_boot = ["online", "offline", "key invalid", "error", "Error: Key not accessible!"]
x = status_boot[3]

Search haveibeenpwned for all emails on a domain

I am able to use haveibeenpwned to search for 1 account compromise. However, I could not find an option to use the API key to search for compromise of all the email accounts on a domain. (For example. if the domain is xyz.com, I want to search for the compromise of abc#xyz.com, peter.charlie#xyz.com and so on). I am aware of the notification email that I can sign up for. But, that is a lengthy process and I prefer using the API.
So, I wrote a script to search against haveibeenpwned for all the email address of my domain, but it takes very long. I searched through a couple of Github projects, but I did not find any such implementation. Has anyone tried this before?
I have added the code below. I am using Multi threading approach, but still it takes very long, is there any other Optimization strategy I can use? Please help. Thank you.
import requests, json
import threading
from time import sleep
import datetime
import splunklib.client as client
import splunklib.results as results
date = datetime.datetime.now()
from itertools import islice
import linecache
import sys
def PrintException():
exc_type, exc_obj, tb = sys.exc_info()
f = tb.tb_frame
lineno = tb.tb_lineno
filename = f.f_code.co_filename
linecache.checkcache(filename)
line = linecache.getline(filename, lineno, f.f_globals)
print 'EXCEPTION IN ({}, LINE {} "{}"): {}'.format(filename, lineno, line.strip(), exc_obj)
class myThread (threading.Thread):
def __init__(self, threadID, name, list_emails):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.list_emails = list_emails
def run(self):
i=0
print "Starting " + self.name
for email in self.list_emails:
print i
i=i+1
result = check_pasteaccount(email)
print email
print result
print result
print "Exiting " + self.name
def check_pasteaccount(account):
account = str(account)
result = ""
URL = "https://haveibeenpwned.com/api/v3/pasteaccount/%s?truncateResponse=false" % (account)
# print(URL)
headers= {'hibp-api-key':api_key}
result = ""
try:
r = requests.get(url=URL,headers=headers)
# sleep(2)
status_code = r.status_code
if status_code == 200:
data = r.text
result = []
for entry in json.loads(data.decode('utf8')):
if int((date - datetime.datetime.strptime(entry['Date'], '%Y-%m-%dT%H:%M:%SZ')).days) > 120:
pass
else:
result.append(['Title: {0}'.format(entry['Title']), \
'Source: {0}'.format(['Source']), \
'Paste ID: {0}'.format(entry['Id'])])
if len(result) == 0:
result = "No paste reported for given account and time frame."
else:
paste_result = ""
for entry in result:
for item in entry:
paste_result += str(item) + "\r\n"
paste_result += "\r\n"
result = paste_result
elif status_code == 404:
result = "No paste for the account"
else:
if status_code == 429:
sleep(5)
# print "Limit exceeded, sleeping"
result = check_pasteaccount(account)
else:
result = "Exception"
print status_code
except Exception as e:
result = "Exception"
PrintException()
pass
return result
def split_every(n, iterable):
iterable = iter(iterable)
for chunk in iter(lambda: list(islice(iterable, n)), []):
yield chunk
def main():
print datetime.datetime.now()
# Fetching the list of email addresses from Splunk
list_emails = connect_splunk()
print datetime.datetime.now()
i=0
list_split = split_every(1000,list_emails)
threads=[]
for list in list_split:
i=i+1
thread_name = "Thread" + str(i)
thread = myThread(1, thread_name, list)
thread.start()
threads.append(thread)
# Wait for all the threads to complete
for t in threads:
t.join()
print "Completed Search"

Here's a shorter and maybe more efficient version of your script using the standard multiprocessing library instead of a hand-rolled thread system.
You'll need Python 3.6+ since we're using f-strings.
You'll need to install the tqdm module for fancy progress bars.
You can adjust the number of concurrent requests with the pool size parameter.
Output is written in machine-readable JSON Lines format into a timestamped file.
A single requests session is shared (per-worker), which means less time spent connecting to HIBP.
import datetime
import json
import multiprocessing
import random
import time
import requests
import tqdm
HIBP_PARAMS = {
"truncateResponse": "false",
}
HIBP_HEADERS = {
"hibp-api-key": "xxx",
}
sess = requests.Session()
def check_pasteaccount(account):
while True:
resp = sess.get(
url=f"https://haveibeenpwned.com/api/v3/pasteaccount/{account}",
params=HIBP_PARAMS,
headers=HIBP_HEADERS,
)
if resp.status_code == 429:
print("Quota exceeded, waiting for a while")
time.sleep(random.uniform(3, 7))
continue
if resp.status_code >= 400:
return {
"account": account,
"status": resp.status_code,
"result": resp.text,
}
return {
"account": account,
"status": resp.status_code,
"result": resp.json(),
}
def connect_splunk():
# TODO: return emails
return []
def main():
list_emails = [str(account) for account in connect_splunk()]
datestamp = datetime.datetime.now().isoformat().replace(":", "-")
output_filename = f"accounts-log-{datestamp}.jsonl"
print(f"Accounts to look up: {len(list_emails)}")
print(f"Output filename: {output_filename}")
with multiprocessing.Pool(processes=16) as p:
with open(output_filename, "a") as f:
results_iterable = p.imap_unordered(
check_pasteaccount, list_emails, chunksize=20
)
for result in tqdm.tqdm(
results_iterable,
total=len(list_emails),
unit="acc",
unit_scale=True,
):
print(json.dumps(result, sort_keys=True), file=f)
if __name__ == "__main__":
main()

returning values inside a function

I have python code looping thru json post and connecting to network device. All that works fine but i can not return back to the json client postman. Python 3. 4 Flask. I have tried many different solutions. All i'm trying to do is return results from my netmiko send commands
from flask import Flask, jsonify, request
import netmiko
from netmiko.ssh_autodetect import SSHDetect
from netmiko.ssh_exception import NetMikoTimeoutException
import time
import gevent
app = Flask(__name__)
#app.route('/myuri', methods=['GET','POST', 'DELETE'])
def post():
# Authentication
headers = request.headers
auth = headers.get("header key")
if auth == 'my key':
def firewall(command):
src_a = command[0]
src_p = command[1]
dst_a = command[2]
dst_p = command[3]
p_col = command[4]
p_show = command[5]
p_push = command[6]
ip = "1.1.1.1"
username = "bla"
password = "bla"
device = {"device_type": "autodetect", "host": ip,
"username": username, "password": password}
while True:
try:
guesser = SSHDetect(**device)
best_match = guesser.autodetect()
print(best_match)
if "None" in str(best_match):
continue
if "true" in str(p_show) and "juniper_junos" in
str(best_match):
device["device_type"] = best_match
connection = netmiko.ConnectHandler(**device)
time.sleep(1)
connection.enable()
resp = connection.send_command('show
configuration | display json | match ' + str(src_a))
resp1 = connection.send_command('show
configuration | display json | match ' + str(src_p))
resp2 = connection.send_command('show
configuration | display json | match ' + str(dst_a))
resp3 = connection.send_command('show
configuration | display json | match ' + str(dst_p))
connection.disconnect()
time.sleep(1)
returns = resp, resp1, resp2, resp3
print(returns) # this prints fine !!!!!
return return # Can't return back !!!!!!
except NetMikoTimeoutException:
return "Timeout Error" ### Note can't return this!
commands = []
data = request.get_json(force=True)
for x in data["firewall"]:
if 'SourceAddress' in x:
commands.append((x['SourceAddress'], x['SourcePort'],
x['DestinationAddress'], x['DestinationPort'],
x['Protocol'], x['show'], x['push']))
threads = [gevent.spawn(firewall, command) for command in
commands]
gevent.joinall(threads)
return "done" ###### how do i return the returns in function
Firewall
else:
return jsonify({"message": "ERROR: Unauthorized"}), 401
the python works finds device auto detect and logs in gets info i can print all of it just can't get those returns to return backenter code here

The return is a keyword, the variable with data in your code is returns
return returns # Will work !!!!!!

You got at typo with your return statement
return return # Can't return back !!!!!!

My python webhook isn't giving me results

I've been working on trying to edit a webhook that was originally meant to be used for a weather API to get to be used with a postcode/zipcode API. The original file is here: https://github.com/dialogflow/fulfillment-webhook-weather-python/blob/master/app.py
I can't understand where mine is different, I thought I had solved it when I replaced urlencode with quote but alas, it wasn't enough.
The problem is very unlikely to do with the source json request that collects the postcode in postcodeValue(). The api url comes out correct when you enter it into a browser and is presented quite simply.
https://api.getaddress.io/find/SW11%201th?api-key=I98umgPiu02GEMmHdmfg3w12959
Is it in the correct format? Maybe I need to convert it to become even more JSON then it already is. This question is essentially an end of day brain dump that I I'm hoping that someone can save me with.
from __future__ import print_function
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urlencode, quote
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import json
import os
from flask import Flask
from flask import request
from flask import make_response
# Flask app should start in global layout
app = Flask(__name__)
#this line is just naming conventions I reckon with a reference to expect to receive data as POST
#app.route('/webhook', methods=['POST'])
def webhook():
req = request.get_json(silent=True, force=True)
#who knows where this is getting printed
print("Request:")
print(json.dumps(req, indent=4))
res = processRequest(req)
res = json.dumps(res, indent=4)
# print(res)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
def processRequest(req):
if req.get("result").get("action") != "yahooWeatherForecast":
return {}
baseurl = "https://api.getaddress.io/find/"
apikey = "?api-key=I98umgPiu02GEMmHdmfg3w12959"
yql_query = postcodeValue(req)
if yql_query is None:
return {}
#this line is the actual api request
yql_url = baseurl + quote(yql_query) + apikey
result = urlopen(yql_url).read()
data = json.loads(result)
res = makeWebhookResult(data)
return res
#this function extracts an individual parameter and turns it into a string
def postcodeValue(req):
result = req.get("result")
parameters = result.get("parameters")
postcode = parameters.get("postcode")
if postcode is None:
return None
return postcode
#def housenoValue(req):
# result = req.get("result")
#parameters = result.get("parameters")
#houseno = parameters.get("houseno")
#if houseno is None:
# return None
#return houseno
def makeWebhookResult(data):
longitude = data.get("longitude")
if longitude is None:
return {}
#def makeWebhookResult(data):
# query = data.get('query')
# if query is None:
# return {}
# result = query.get('results')
# if result is None:
# return {}
# channel = result.get('channel')
# if channel is None:
# return {}
# item = channel.get('item')
# location = channel.get('location')
# units = channel.get('units')
# if (location is None) or (item is None) or (units is None):
# return {}
# condition = item.get('condition')
# if condition is None:
# return {}
# print(json.dumps(item, indent=4))
speech = "Sausage face " + longitude
print("Response:")
print(speech)
return {
"speech": speech,
"displayText": speech,
# "data": data,
# "contextOut": [],
"source": "apiai-weather-webhook-sample"
}
#More flask specific stuff
if __name__ == '__main__':
port = int(os.getenv('PORT', 5000))
print("Starting app on port %d" % port)
app.run(debug=False, port=port, host='0.0.0.0')

Here is a bit cleaner version of your code:
from urllib.request import urlopen
import os
from flask import Flask
app = Flask(__name__)
#app.route('/webhook', methods=['GET'])
def webhook():
res = processRequest()
return res
def processRequest():
try:
result = urlopen("https://api.getaddress.io/find/SW11%201th?api-key=I98umgPiu02GEMmHdmfg3w12959").read()
return result
except:
return "Error fetching data"
if __name__ == '__main__':
port = int(os.getenv('PORT', 5000))
print("Starting app on port %d" % port)
app.run(debug=False, port=port, host='0.0.0.0')
Open your browser and go to http://localhost:5000/webhook and you should see a response.

How to unit test on tornado?

I want to testing my tornado server
but, I don't know how...
I don't run my testing code...
please help me unit test on tornado
I don't speak English very well...
I am sorry that I have poor questions.
this is my tornado server code.
import tornado.ioloop
import os
from tornado import web, gen
from concurrent.futures import ThreadPoolExecutor
from json_output import on_correct_text
import json
thread_pool = ThreadPoolExecutor()
class cer(tornado.web.RequestHandler):
_thread_pool = thread_pool
#gen.coroutine
def post(self, json_data):
### temporary json data
_json = {'file_name' : 'tmp_text.txt', 'file_text' : 'temp text...'}
json_data = json.dumps(_json)
tmp_json = {'file_name': '', 'text': '', 'upload_start_timestamp':'', 'upload_end_timestamp': '','process_start_timestamp': '', 'process_end_timestamp': '', 'response_time_second': '', 'status': "upload_waiting"}
json_d = json.loads(json_data)
print(json_d)
jsonString = yield on_correct_text(json_d, tmp_json)
self.set_header("Content-Type", "application/json")
self.write(jsonString)
self.finish()
def make_app():
return tornado.web.Application([
(r"/([a-zA-Z0-9.:_/%{}]+)", cer)
]
)
if __name__ == "__main__":
app = make_app()
app.listen(8888)
tornado.ioloop.IOLoop.instance().start()
and this is json_output.py
import datetime, time, json
from concurrent.futures import ThreadPoolExecutor
from tornado import gen
thread_pool = ThreadPoolExecutor()
_thread_pool = thread_pool
#gen.coroutine
def on_correct_text(json_data, tmp_json):
file_up_starttime = datetime.datetime.now()
tmp_json['upload_start_timestamp'] = str(file_up_starttime)
tmp_json['status'] = "uploading"
try:
tmp_json['text'] = json_data['file_text']
tmp_json['file_name'] = json_data['file_name']
except:
tmp_json['status'] = "upload_faild"
print("upload falied")
jsonString = json.dumps(tmp_json, ensure_ascii=False)
return jsonString
start_time = time.time()
file_up_endtime = datetime.datetime.now()
tmp_json['upload_end_timestamp'] = str(file_up_endtime)
process_starttime = datetime.datetime.now()
tmp_json['process_start_timestamp'] = str(process_starttime)
tmp_json['status'] = "processing"
try:
process_endtime = datetime.datetime.now()
tmp_json['process_end_timestamp'] = str(process_endtime)
tmp_json['status'] = "complete"
except:
tmp_json['status'] = "process_faild"
print("process falied")
jsonString = json.dumps(tmp_json, ensure_ascii=False)
return jsonString
response_time = round((time.time() - start_time), 2)
tmp_json['response_time_second'] = str(response_time)
jsonString = json.dumps(tmp_json, ensure_ascii=False)
return jsonString
please help me...
I don't know how this tornado server unit test

There is some excellent documentation here on how to setup Unit Testing in Tornado:
http://www.tornadoweb.org/en/stable/testing.html
class cerTester(AsyncHTTPTestCase):
def get_app(self):
return make_app()
def get_url(self, path):
"""Returns an absolute url for the given path on the test server."""
return '%s://localhost:%s%s' % (self.get_protocol(),
self.get_http_port(), path)
def test_url(self):
#This is where your actual test will take place.
response = self.fetch('/sometest', method="POST")
assert response.code == 200
data = json.loads(response.body.decode('utf-8'))
assert data
#More Asserts