We Keep Coding

How to make sure the logged message goes to a new line?

I'm trying to make my own logging system because I don't like Python's one, but I ran into a small problem. Whenever I log two or more things, only the last log is written. Example - I log 'code initialized' at the start of the program. Then, I log 'exiting'. The result in the log file is only 'exiting' How do I fix this? Here is my code.
File 1.
from functions import *
from variables import *
try:
open('key.txt', "r")
key = True
except FileNotFoundError:
key = False
if check_internet():
x = status_boot[0]
elif not check_internet():
x = status_boot[1]
log("archer initialized")
if key:
print("archer status: " + x)
elif not key:
print(status_boot[4])
File 2.
from datetime import datetime
from variables import *
import requests
def time():
now = datetime.now()
return now.strftime("%H:%M:%S")
def check_internet():
timeout = 5
try:
_ = requests.get(url, timeout=timeout)
return True
except requests.ConnectionError:
return False
def log(m):
write = open('archerlog.txt', 'w+')
return write.write(f'[{time()}] archer: {m}\n')
File 3.
url = "http://www.google.com/"
status_boot = ["online", "offline", "key invalid", "error", "Error: Key not accessible!"]
x = status_boot[3]

Search Splunk API using python

What I am trying to do is perform a search on Splunk's API using python, I am able to get a session key but thats it. I'm new to both python and splunk so im a bit out-of-depth and any help would be really appreciated.
The error:
Traceback (most recent call last):
File "splunkAPI.py", line 31, in <module>
sid = minidom.parseString(r.text).getElementsByTagName('sid')[0].firstChild.nodeValue
IndexError: list index out of range
python:
import time # need for sleep
from xml.dom import minidom
import json, pprint
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
base_url = 'https://___________:8089'
username = '______'
password = '______'
search_query = "____________"
#-------------------------get session token------------------------
r = requests.get(base_url+"/servicesNS/admin/search/auth/login",
data={'username':username,'password':password}, verify=False)
session_key = minidom.parseString(r.text).getElementsByTagName('sessionKey')[0].firstChild.nodeValue
print ("Session Key:", session_key)
#-------------------- perform search -------------------------
r = requests.post(base_url + '/services/search/jobs/', data=search_query,
headers = { 'Authorization': ('Splunk %s' %session_key)},
verify = False)
sid = minidom.parseString(r.text).getElementsByTagName('sid')[0].firstChild.nodeValue
done = False
while not done:
r = requests.get(base_url + '/services/search/jobs/' + sid,
headers = { 'Authorization': ('Splunk %s' %session_key)},
verify = False)
response = minidom.parseString(r.text)
for node in response.getElementsByTagName("s:key"):
if node.hasAttribute("name") and node.getAttribute("name") == "dispatchState":
dispatchState = node.firstChild.nodeValue
print ("Search Status: ", dispatchState)
if dispatchState == "DONE":
done = True
else:
time.sleep(1)
r = requests.get(base_url + '/services/search/jobs/' + sid + '/results/',
headers = { 'Authorization': ('Splunk %s' %session_key)},
data={'output_mode': 'json'},
verify = False)
pprint.pprint(json.loads(r.text))

Hmm... that code looks awfully familiar :P Unfortunately, error checking wasn't that important when I wrote it.
The issue you see occurs if the search_query is not defined properly. It must start with search=. Also note that you need to include an initial search command if doing a standard Splunk search,
For example, search=search index=* will work, search=index=* will not work.
If you need to include quotes in your search string, I suggest you use something like the following format.
search_query = """search=search index=* "a search expression" | stats count"""

Tried this but did not give needed result not sure what is missing
import urllib
import httplib2 #import library
import json
import pprint
import time
import re
from xml.dom import minidom
searchquery = 'search index="movable_in" sourcetype="movable:in:assets" | stats avg(exposure_score)'
myhttp = httplib2.Http()
baseurl = 'https://xxxx.splunkxxx.com:8089'
usernamesp = 'xxxx'
passwordsp = 'xxxx'
def get_splunk_result(searchquery):
# Step 1: Get a session key
servercontent = myhttp.request(f'{baseurl}/services/auth/login', 'POST', headers={},
body=urllib.parse.urlencode({'username': usernamesp, 'password': passwordsp}))[1]
sessionkey = minidom.parseString(servercontent).getElementsByTagName('sessionKey')[0].childNodes[0].nodeValue
# print ("====>sessionkey: %s <====" % sessionkey)
sid = ''
# ------------------
if not searchquery.startswith('search'):
searchquery = f'search {searchquery}'
# Step 2: Get a sid with the search query
i = 0
while True:
time.sleep(1)
try:
searchjob = myhttp.request(f'{baseurl}/services/search/jobs', 'POST',
headers={F'Authorization': F'Splunk %s' % sessionkey},
body=urllib.parse.urlencode({'search': searchquery}))[1]
sid = minidom.parseString(searchjob).getElementsByTagName('sid')[0].childNodes[0].nodeValue
break
except:
i = i + 1
# print(i)
if (i > 30): break
# print("====>SID: %s <====" % sid)
# Step 3: Get search status
myhttp.add_credentials(usernamesp, passwordsp)
servicessearchstatusstr = '/services/search/jobs/%s/' % sid
isnotdone = True
while isnotdone:
searchstatus = myhttp.request(f'{baseurl}{servicessearchstatusstr}', 'GET')[1]
isdonestatus = re.compile('isDone">(0|1)')
strstatus = str(searchstatus)
isdonestatus = isdonestatus.search(strstatus).groups()[0]
if (isdonestatus == '1'):
isnotdone = False
# Step 4: Get the search result
services_search_results_str = '/services/search/jobs/%s/results?output_mode=json_rows&count=0' % sid
searchresults = myhttp.request(f'{baseurl}{services_search_results_str}', 'GET')[1]
searchresults = json.loads(searchresults)
# searchresults = splunk_result(searchresults)
return searchresults
output = get_splunk_result(searchquery)
print(output)

Changing Bitcoin Wallet Address Range for Plutus.py

I am working with the following script from https://github.com/Isaacdelly/Plutus/blob/master/plutus.py
The script works for wallet addresses in the 2^160 range. I am curious where in the script I can change this to look at the 2^128 range or 2^n range. Would it be possible to even have a window? Like 2^0 - 2^100?
Not trying to do anything malicious, just trying to get data to show that even selecting ranges is futile due to the large number of addresses.
# Plutus Bitcoin Brute Forcer
# Made by Isaac Delly
# https://github.com/Isaacdelly/Plutus
try:
import sys
import os
import time
import hashlib
import binascii
import multiprocessing
from multiprocessing import Process, Queue
from multiprocessing.pool import ThreadPool
import threading
import base58
import ecdsa
import requests
except ImportError:
import subprocess
subprocess.check_call(["python", '-m', 'pip', 'install', 'base58==1.0.0'])
subprocess.check_call(["python", '-m', 'pip', 'install', 'ecdsa==0.13'])
subprocess.check_call(["python", '-m', 'pip', 'install', 'requests==2.19.1'])
import base58
import ecdsa
import requests
def generate_private_key():
return binascii.hexlify(os.urandom(32)).decode('utf-8')
def private_key_to_WIF(private_key):
var80 = "80" + str(private_key)
var = hashlib.sha256(binascii.unhexlify(hashlib.sha256(binascii.unhexlify(var80)).hexdigest())).hexdigest()
return str(base58.b58encode(binascii.unhexlify(str(var80) + str(var[0:8]))), 'utf-8')
def private_key_to_public_key(private_key):
sign = ecdsa.SigningKey.from_string(binascii.unhexlify(private_key), curve = ecdsa.SECP256k1)
return ('04' + binascii.hexlify(sign.verifying_key.to_string()).decode('utf-8'))
def public_key_to_address(public_key):
alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
count = 0; val = 0
var = hashlib.new('ripemd160')
var.update(hashlib.sha256(binascii.unhexlify(public_key.encode())).digest())
doublehash = hashlib.sha256(hashlib.sha256(binascii.unhexlify(('00' + var.hexdigest()).encode())).digest()).hexdigest()
address = '00' + var.hexdigest() + doublehash[0:8]
for char in address:
if (char != '0'):
break
count += 1
count = count // 2
n = int(address, 16)
output = []
while (n > 0):
n, remainder = divmod (n, 58)
output.append(alphabet[remainder])
while (val < count):
output.append(alphabet[0])
val += 1
return ''.join(output[::-1])
def get_balance(address):
try:
response = requests.get("https://bitaps.com/api/address/" + str(address))
return int(response.json()['balance'])
except:
return -1
def data_export(queue):
while True:
private_key = generate_private_key()
public_key = private_key_to_public_key(private_key)
address = public_key_to_address(public_key)
data = (private_key, address)
queue.put(data, block = False)
def worker(queue):
while True:
if not queue.empty():
data = queue.get(block = True)
balance = get_balance(data[1])
process(data, balance)
def process(data, balance):
private_key = data[0]
address = data[1]
if (balance == 0):
print("{:<34}".format(str(address)) + ": " + str(balance))
if (balance > 0):
file = open("plutus.txt","a")
file.write("address: " + str(address) + "\n" +
"private key: " + str(private_key) + "\n" +
"WIF private key: " + str(private_key_to_WIF(private_key)) + "\n" +
"public key: " + str(private_key_to_public_key(private_key)).upper() + "\n" +
"balance: " + str(balance) + "\n\n")
file.close()
def thread(iterator):
processes = []
data = Queue()
data_factory = Process(target = data_export, args = (data,))
data_factory.daemon = True
processes.append(data_factory)
data_factory.start()
work = Process(target = worker, args = (data,))
work.daemon = True
processes.append(work)
work.start()
data_factory.join()
if __name__ == '__main__':
try:
pool = ThreadPool(processes = multiprocessing.cpu_count()*2)
pool.map(thread, range(0, 10))
except:
pool.close()
exit()
Thank you

You seem to be misunderstanding the purpose of the 2^160 bit range.
Each standard bitcoin address is tied to the HASH160 of the public key. A HASH160 is 160 bits long, which is why your search space is 2^160. If you are able to find two private keys for which the HASH160 of the public keys are equal, any of those two private keys can spend coins sent to that address.
Searching a smaller space does not make sense since you are no longer searching for bitcoin addresses. If you just want to search random hash functions, then you simply need to replace RIPEMD160 hash function with another one that has an output in whatever bitsize you wish to search.
Note that if you do that, the rest of the code talking about checking balances etc. will be of no use, since your output will no longer be a bitcoin address.

Best way to make thousands of get requests in python

Right now I am working on a python script which takes in a list of url's as an argument, then performs a GET request on each url and then searches through the output with xpath to fingerprint the website. It seems to work like a charm when the list is around 50 sites long, but anything after that causes the program to slow down to the point where it stop (usually around 150 sites). Scroll down to where you see main app logic and the relevant code it below. Right now I am just using 50 elements in the array and it works fine, but anything after makes the entire program stop. Any suggestions would be greatly appreciated!
#!/usr/bin/python
# Web Scraper
# 1.0
# Imports for file
from multiprocessing.dummy import Pool as ThreadPool
from threading import Thread
from Queue import Queue
from lxml import html
import requests
import time
import sys
# Get Raw HTML
def scrape(url):
try:
page = requests.get(url, timeout=2.0)
if page.status_code == requests.codes.ok:
html_page = html.fromstring(page.content)
s =requests.session()
s.close()
return html_page
else:
s =requests.session()
s.close()
return False
except:
s =requests.session()
s.close()
return False
# Format URL
def format_url(url):
if url.find("http://") == -1:
url = "http://"+url
if url[-1] == "/":
url = url[:-1]
return url
# Check if WordPress Site
def check_wordpress(tree):
scripts = tree.xpath("//script[contains(#src,'wp-content')]")
if len(scripts) > 0:
return True
return False
# Check WordPress Version
def wordpress_version(tree):
type = tree.xpath("//meta[#name='generator']/#content")
version = 0
if len(type) > 0:
details = type[0].split()
if len(details)>1 and details[0] == "WordPress":
if len(details) > 1:
version = details[1]
else:
version = type[0]
return version
# Find Contact Page
def find_contact_page(tree):
contact = tree.xpath("//a[contains(text(),'Contact')]/#href")
try_xpath = 1
while len(contact) == 0:
if try_xpath == 1:
contact = tree.xpath("//span[contains(text(),'Contact')]/../#href")
elif try_xpath == 2:
contact = tree.xpath("//p[contains(text(),'Contact')]/../#href")
elif try_xpath == 3:
break
try_xpath+=1
if len(contact) > 0:
contact = contact[0]
if contact.find('#') == -1:
if contact[0] == '/':
contact = url + "" + contact
print contact
# Juicer method
def juice(url):
url = format_url(url)
string = url
tree = scrape(url)
if tree == False:
return string + " \t\t\t No XML tree"
elif check_wordpress(tree) == True:
version = wordpress_version(tree)
return string + " \t\t\t WordPress: " + str(version)
else:
return string + " \t\t\t Not WordPress"
# Main App Logic Below ------------------------------------->
# Open list of websites from given argument
list = open(sys.argv[1],'r').read().split('\n')
# Juice url
def juice_url():
while True:
url = q.get()
result = juice(url)
print result
q.task_done()
# Create concurrent queues
concurrent = 50
q = Queue(concurrent)
for i in range(concurrent):
t = Thread(target=juice_url)
t.daemon = True
t.start()
# Add URL to Queue
time1 = time.time()
for url in list[0:50]:
q.put(url)
q.join()
# Calculate total time
total = time.time() - time1
print "Total Time: %f" % total
print "Average Time: %f" % (total/50)

get the first 10 google results using googleapi

I need to get the first 10 google results
for example:
... query = urllib.urlencode({'q' : 'example'})
...
... url = 'http://ajax.googleapis.com/ajax/services/search/web?v=1.0&%s' \
... % (query)
... search_results = urllib.urlopen(url)
... json = simplejson.loads(search_results.read())
... results = json['responseData']['results']
this will give me the results of the first page, but I`d like to get more google results, do anyone know how to do that?

I've done it in the past, here is complete example (i'm not python guru, but it works):
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys, getopt
import urllib
import simplejson
OPTIONS = ("m:", ["min="])
def print_usage():
s = "usage: " + sys.argv[0] + " "
for o in OPTIONS[0]:
if o != ":" : s += "[-" + o + "] "
print(s + "query_string\n")
def search(query, index, offset, min_count, quiet=False, rs=[]):
url = "http://ajax.googleapis.com/ajax/services/search/web?v=1.0&rsz=large&%s&start=%s" % (query, offset)
result = urllib.urlopen(url)
json = simplejson.loads(result.read())
status = json["responseStatus"]
if status == 200:
results = json["responseData"]["results"]
cursor = json["responseData"]["cursor"]
pages = cursor["pages"]
for r in results:
i = results.index(r) + (index -1) * len(results) + 1
u = r["unescapedUrl"]
rs.append(u)
if not quiet:
print("%3d. %s" % (i, u))
next_index = None
next_offset = None
for p in pages:
if p["label"] == index:
i = pages.index(p)
if i < len(pages) - 1:
next_index = pages[i+1]["label"]
next_offset = pages[i+1]["start"]
break
if next_index != None and next_offset != None:
if int(next_offset) < min_count:
search(query, next_index, next_offset, min_count, quiet, rs)
return rs
def main():
min_count = 64
try:
opts, args = getopt.getopt(sys.argv[1:], *OPTIONS)
for opt, arg in opts:
if opt in ("-m", "--min"):
min_count = int(arg)
assert len(args) > 0
except:
print_usage()
sys.exit(1)
qs = " ".join(args)
query = urllib.urlencode({"q" : qs})
search(query, 1, "0", min_count)
if __name__ == "__main__":
main()
Edit: i've fixed obvious command-line options mishandling; you can call this script as follows:
python gsearch.py --min=5 vanessa mae
--min switch means "at least 5 results" and is optional, you will get maximum allowed result count (64) if it is not specified.
Also, error handling is omitted for brevity.

See docs http://code.google.com/apis/websearch/docs/reference.html#_intro_fonje
You’re looking for the start parameter.
There’s no parameter to get more results in one response, but you can iterate via the start parameter.