I look to get Forward and Reverse look DNS look up for an address, i.e. a URL. I write below function with socket but time performance is about 4.5 second.
I would appreciate help and recommendation to improve time performance.
def DNS_lookups(address):
'''
Forward and reverse look DNS look up for address = URL
- forward: URL --> ip - check if is live
- reverse: ip --> domain name
'''
try:
ip_add = socket.gethostbyname(address)
try:
domain_name = socket.gethostbyaddr(ip_add)[0]
except socket.herror:
# print('oops')
return [ip_add, 'host not found']
except socket.gaierror:
# print('oops')
return ['False', 'NA']
#finally: # else:
# domain_name = socket.gethostbyaddr(socket.gethostbyname(address))[0]
return [ip_add, domain_name]
Related
So, someone started this project and it fell on my lap so I could fix it.
This is my first attempt at coding outside of school, so I'm not very experienced or good. Apologies if the solution was something obvious.
Basically this program is supposed to do a few things.
Accept one or more inputs of IP addresses
Scan the databases written into it
Return each database that flags the IP as malicious
Return any database that has an error related to the API key.
Return the location of the IP as reported by AbuseIPDB.(but I haven't even started that yet)
Right now, this is the error I'm getting.
Traceback (most recent call last):
File "script.py", line 119, in <module>
is_malicious, flagged_databases = check_ip_reputation(ip)
TypeError: cannot unpack non-iterable bool object
I have no idea how to correct that. I've rewritten a few lines to fix other errors but something new always comes up.
This is the code. Something to note, is that two databases are missing APIs. But those should return an error as mentioned above.
'''
import requests
def check_ip_reputation(ip_address):
# Set up a list to store the names of the databases that flag the IP as malicious
flagged_databases = []
# Set up the parameters for the AbuseIPDB request
params = {
'key': 'db327100238564236c6e25fe412ed23d80cfecab28691b0e672bd2a0798156250de5473bc648d255',
'ipAddress': ip_address
}
# Make the request to AbuseIPDB
try:
response = requests.get('https://api.abuseipdb.com/api/v2/check', params=params)
except requests.exceptions.RequestException as e:
print(f'Error making request to AbuseIPDB: {e}')
return False
# Extract the "abuseConfidenceScore" field from the response
abuse_score = response.json()['data']['abuseConfidenceScore']
# Set a threshold for the AbuseIPDB score
abuse_threshold = 50
# Check if the abuse score is above the threshold
if abuse_score >= abuse_threshold:
flagged_databases.append('AbuseIPDB')
# Set up the parameters for the VirusTotal request
params = {
'apikey': '7f21d9a126b73adf22ea100f883e38496f44412933a27cf1740858f3568be5e4',
'ip': ip_address
}
# Make the request to VirusTotal
try:
response = requests.get('https://www.virustotal.com/vtapi/v2/ip-address/report', params=params)
except requests.exceptions.RequestException as e:
print(f'Error making request to VirusTotal: {e}')
return False
# Extract the "response_code" field from the response
response_code = response.json()['response_code']
# Check if the response code indicates that the IP is listed
if response_code == 1:
flagged_databases.append('VirusTotal')
# Set up the parameters for the MXtoolbox request
params = {
'key': 'API_KEY',
'ip': ip_address
}
# Make the request to MXtoolbox
try:
response = requests.get('https://mxtoolbox.com/api/v1/lookup/blacklist/' + ip_address, params=params)
except requests.exceptions.RequestException as e:
print(f'Error making request to MXtoolbox: {e}')
return False
# Try to extract the "blacklist" field from the response
try:
blacklist = response.json()['blacklist']
except TypeError:
# If the response is a string, then the IP is not blacklisted
return False
# Check if the IP is listed in any of the blacklists
is_blacklisted = len(blacklist) > 0
# Return the result
return is_blacklisted
# Set up the parameters for the Talos request
params = {
'key': 'API_KEY',
'ip': ip_address
}
# Make the request to Talos
try:
response = requests.get('https://talosintelligence.com/documents/ip-blacklist', params=params)
except requests.exceptions.RequestException as e:
print(f'Error making request to Talos: {e}')
return False
# Check if the response code indicates that the IP is listed
if response.status_code == 200:
flagged_databases.append('Talos Intelligence')
##############################################################################
# Combine the results from all four databases
if(len(flagged_databases) > 0):
is_malicious = len(flagged_databases)
else:
is_malicious = 0
# Return the result
return is_malicious, flagged_databases;
##############################################################################
# Prompt the user for a list of IP addresses
ip_addresses_str = input("Enter a list of IP addresses separated by commas: ")
# Split the input string into a list of IP addresses
ip_addresses = ip_addresses_str.split(',')
# Strip any leading or trailing whitespace from the IP addresses
ip_addresses = [ip.strip() for ip in ip_addresses]
# Check the reputation of each IP address
for ip in ip_addresses:
is_malicious, flagged_databases = check_ip_reputation(ip)
if is_malicious:
print(f'{ip} has been flagged as malicious by the following databases: {", ".join(flagged_databases)}')
else:
print(f'{ip} has not been flagged as malicious by any of the OSINT databases.')
'''
Any help would be so, so appreciated.
Listed above, but I did try changing it so it could read strings and dictionary.
Here are some suggestions for making the code more robust.
I noticed that you were returning from the function in a few places, with different semantics for the result:
Errors were being returned as just a boolean
One of the checks, you were returning just a boolean rather than adding to the flagged databases
You had a return statement that matched the calling code, but you would not reach it because of the above return statements
The calling code expected a boolean and a list, but it would only ever get a boolean, which is why you got the error.
When you are querying multiple sources, and presuming you want to return all the information that is available rather than give up with one error, it may be good to put results from all of them into a data structure and then return what you have, including the errors. Let the calling code decide whether having errors is a problem since some of the results may be useful.
import requests
def check_ip_reputation(ip_address):
# Set up a list to store the names of the databases that flag the IP as malicious
databaseResults = {'Errors': [], 'ReportingMalicious': [], 'ReportingClean': []}
# Set up the parameters for the AbuseIPDB request
params = {
'key': 'db327100238564236c6e25fe412ed23d80cfecab28691b0e672bd2a0798156250de5473bc648d255',
'ipAddress': ip_address
}
# Make the request to AbuseIPDB
try:
response = requests.get('https://api.abuseipdb.com/api/v2/check', params=params)
except requests.exceptions.RequestException as e:
databaseResults['Errors'].append('AbuseIPDB')
# Extract the "abuseConfidenceScore" field from the response
abuse_score = response.json()['data']['abuseConfidenceScore']
# Set a threshold for the AbuseIPDB score
abuse_threshold = 50
# Check if the abuse score is above the threshold
if abuse_score >= abuse_threshold:
databaseResults['ReportingMalicious'].append('AbuseIPDB')
else:
databaseResults['ReportingClean'].append('AbuseIPDB')
# Set up the parameters for the VirusTotal request
params = {
'apikey': '7f21d9a126b73adf22ea100f883e38496f44412933a27cf1740858f3568be5e4',
'ip': ip_address
}
# Make the request to VirusTotal
try:
response = requests.get('https://www.virustotal.com/vtapi/v2/ip-address/report', params=params)
except requests.exceptions.RequestException as e:
databaseResults['Errors'].append('VirusTotal')
# Extract the "response_code" field from the response
response_code = response.json()['response_code']
# Check if the response code indicates that the IP is listed
if response_code == 1:
databaseResults['ReportingMalicious'].append('VirusTotal')
else:
databaseResults['ReportingClean'].append('VirusTotal')
# Set up the parameters for the MXtoolbox request
params = {
'key': 'API_KEY',
'ip': ip_address
}
# Make the request to MXtoolbox
try:
response = requests.get('https://mxtoolbox.com/api/v1/lookup/blacklist/' + ip_address, params=params)
except requests.exceptions.RequestException as e:
databaseResults['Errors'].append('MXtoolbox')
# Try to extract the "blacklist" field from the response
try:
blacklist = response.json()['blacklist']
is_blacklisted = len(blacklist) > 0
except TypeError:
is_blacklisted = False
# Return the result
if is_blacklisted:
databaseResults['ReportingMalicious'].append('MXtoolbox')
else:
databaseResults['ReportingClean'].append('MXtoolbox')
# Set up the parameters for the Talos request
params = {
'key': 'API_KEY',
'ip': ip_address
}
# Make the request to Talos
try:
response = requests.get('https://talosintelligence.com/documents/ip-blacklist', params=params)
except requests.exceptions.RequestException as e:
databaseResults['Errors'].append('TalosIntelligence')
# Check if the response code indicates that the IP is listed
if response.status_code == 200:
databaseResults['ReportingMalicious'].append('TalosIntelligence')
else:
databaseResults['ReportingClean'].append('TalosIntelligence')
##############################################################################
# Combine the results from all four databases
is_malicious = len(databaseResults['ReportingMalicious']) > 0
# Return the result
return is_malicious, databaseResults;
##############################################################################
# Prompt the user for a list of IP addresses
ip_addresses_str = input("Enter a list of IP addresses separated by commas: ")
# Split the input string into a list of IP addresses
ip_addresses = ip_addresses_str.split(',')
# Strip any leading or trailing whitespace from the IP addresses
ip_addresses = [ip.strip() for ip in ip_addresses]
# Check the reputation of each IP address
for ip in ip_addresses:
is_malicious, flagged_databases = check_ip_reputation(ip)
if is_malicious:
print(f'{ip} has been flagged as malicious by the following databases: ' + ", ".join([db for db in flagged_databases['ReportingMalicious']]) + '}')
else:
print(f'{ip} has not been flagged as malicious by any of the OSINT databases.')
I have modified this code python-paged-ldap-snippet.py from https://gist.github.com/mattfahrner/c228ead9c516fc322d3a
My problem is that when I change my SEARCHFILTER from '(&(objectCategory=person)(objectClass=user))' to '(&(objectCategory=person)(objectClass=user)(memberOf=CN=Users0,OU=Groups,DC=ad,DC=company,DC=com))'
it runs just fine.
If it is on SEARCHFILTER='(&(objectCategory=person)(objectClass=user))', I notice that the code is not entering the writeToFile function.
The objective of the code is to dump all the user information and parse the info into a file.
I tried running LDAPSEARCH against '(&(objectCategory=person)(objectClass=user))' and I manage to get the output .
Not sure what is wrong. Suggestions are greatly appreciated.
Thank you.
#!/usr/bin/python
import sys
import ldap
import os
LDAPSERVER='ldap://xxx.xxx.xxx.xxx:389'
BASEDN='dc=ad,dc=company,dc=com'
LDAPUSER = "CN=LDAPuser,OU=XXX,OU=Users,DC=ad,DC=company,DC=com"
LDAPPASSWORD = 'LDAPpassword'
PAGESIZE = 20000
ATTRLIST = ['sAMAccountName','uid']
SEARCHFILTER='(&(objectCategory=person)(objectClass=user))'
#SEARCHFILTER='(&(objectCategory=person)(objectClass=user)(memberOf=CN=Users0,OU=Groups,DC=ad,DC=company,DC=com))'
data = []
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_ALLOW)
ldap.set_option(ldap.OPT_REFERRALS, 0)
l = ldap.initialize(LDAPSERVER)
l.protocol_version = 3 # Paged results only apply to LDAP v3
try:
l.simple_bind_s(LDAPUSER, LDAPPASSWORD)
print ' Login Done, Searching data'
except ldap.LDAPError as e:
exit('LDAP bind failed: %s' % e)
lc = ldap.controls.SimplePagedResultsControl(True,size=PAGESIZE,cookie='')
def writeToFile(data):
print ' Writing data to file'
#code to print all output into CVS file
while True:
try:
msgid = l.search_ext(BASEDN, ldap.SCOPE_SUBTREE, SEARCHFILTER, ATTRLIST, serverctrls=[lc])
except ldap.LDAPError as e:
sys.exit('LDAP search failed: %s' % e)
try:
rtype, rdata, rmsgid, serverctrls = l.result3(msgid)
except ldap.LDAPError as e:
sys.exit('Could not pull LDAP results: %s' % e)
for dn, attrs in rdata:
data.append(attrs)
pctrls = [
c for c in serverctrls if c.controlType == ldap.controls.SimplePagedResultsControl.controlType ]
if not pctrls:
print >> sys.stderr, 'Warning: Server ignores RFC 2696 control.'
break
cookie = pctrls[0].cookie
if not cookie:
writeToFile(data)
print 'Task Complete'
break
lc.controlValue = (PAGESIZE, cookie)
PAGESIZE = 20000
Lower your page size to a value <= 1000, since that's the max AD will give you at a time anyway. It's possible that it's waiting for 20000 records before requesting the next page and never getting it.
Problem:
I am trying to make a backend web framework as a challenge and I'm trying to implement web
sockets but whenever I make the handshake my browser says that it received no response and the status is 'finished'.
Things I have tried:
I have tried making the request to my own server using the requests module and same Sec Websocket Key as on another website and the key was the same as that other websites response.
I have also tried running it on a node.js server with the same code for the client and the whole thing worked as expected so it's a serverside problem.
Changing browsers and updating browser versions.
Changing from localhost to my private IP and my local machine name.
code
# Ignore missing stuff, some stuff is missing to reduce the length and shouldn't be needed to debug
#app.route('/')
def ws(request: dict):
if request.get('Connection') == 'Upgrade' and request.get('Upgrade') == 'websocket':
# Get returns None if key doesn't exist [] raises keyerror
print(f"Sec-Websocket-Key: {request['Sec-WebSocket-Key']}")
# Prints correct key (:
request['Sec-WebSocket-Key'] += '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
SecWebSocketAccept = hashlib.sha1(request['Sec-WebSocket-Key'].encode())
SecWebSocketAccept = SecWebSocketAccept.digest()
SecWebSocketAccept = base64.b64encode(SecWebSocketAccept).decode()
return HttpResponse(status_code=101,
message='Switching Protocols',
headers= {
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Accept': SecWebSocketAccept,
'Sec-WebSocket-Protocol': 'chat'
}, AutoGenerateExtraHeaders=False)
else:
return RenderTemplate('example.html', minimize=False)
# warning: remove minimize only for testing
# HttpResponse class
class HttpResponse:
def __init__(
self,
data: str = '',
status_code: int = 200,
message: str = 'OK',
headers: dict = {},
AutoGenerateExtraHeaders=True
)-> None:
""" todo: add docs on class and methods"""
self.protocol = 'HTTP/1.1'
self.status = f'{status_code} {message}'
self.data = data
self.headers = headers
if AutoGenerateExtraHeaders:
contentmd5 = hashlib.md5(data.encode()).digest()
self.headers['Content-MD5'] = base64.b64encode(contentmd5)
self.headers['Content-Length'] = len(data.encode())
# todo: add more headers such as date,
self.headers = str(headers)[2:].replace('{', '').replace('}', '').replace(', \'', '\r\n\'').replace('\'', '')
# String version of headers
def encode(self) -> bytes:
return str(self).encode()
# Handle Connection Code
def handleConnection(self, connection: socket.socket, address) -> None:
"""handles incoming requests
Args:
connection (socket.socket): client socket connection
address: address of incoming connection
"""
request = connection.recv(1024).decode()
# print(request)
if not request: return
request: dict = self.dictFromRequest(request)
route = self.app.getRoute(request['route']) # returns route object which is made up of a function and a URL defined by app.route(route: str) decor
method: str = request['method']
protocol: str = request['protocol']
# 404 Page not found
# Call event
event = self.app.callEvent(RequestEvent(request))
if event.timeOut: return
# Handle
if protocol == 'HTTP':
# todo: cors and csrf
if method == 'GET':
response = route.call(request)
print(f'encoded: {response.encode()}, \n\ndecoded: {response}')
# ouupt:
# encoded: b'HTTP/1.1 101 Switching Protocols\r\nUpgrade:
# websocket\r\nConnection:
# Upgrade\r\nSec-WebSocket-Accept: TwmkkaET4SyBJad/5OzZNHxaZ/o=\r\nSec
# -WebSocket-Protocol:
# chat\r\n\r\n',
#
# decoded: HTTP/1.1 101 Switching Protocols
# Upgrade: websocket
# Connection: Upgrade
# Sec-WebSocket-Accept: TwmkkaET4SyBJad/5OzZNHxaZ/o=
# Sec-WebSocket-Protocol: chat
#
#
connection.send(response.encode())
return connection.close()
def dictFromRequest(self, headers: str) -> dict:
# todo: just lol
dict = {}
dict['method'] = headers.split('\r\n\r\n')[0].split('\r\n')[0].split(' ')[0]
dict['protocol'] = headers.split('\r\n')[0].split(' ')[-1].split('/')[0]
dict['route'] = headers.split('\r\n\r\n')[0].split('\r\n')[0].split(' ')[1].split('?')[0]
dict['query'] = ''.join(headers.split('\n\n')[0].split('\r\n')[0].split(' ')[1].split('?')[1:])
dict['data'] = headers.split('\r\n\r\n')[1]
for header in headers.split('\r\n\r\n')[0].split('\r\n')[1:]:
dict[header.split(': ')[0]] = header.split(': ')[1]
return dict
P.S. pls don't judge I'm only 13 lol
I am using Python to connect to Couchbase Database using below kind of string and the last part of the IP dynamically keep changing so I want to keep trying until successful connect string available:
With below construct the problem is if connection good at IP 10.xxx.xx.112 it is not breaking and its still trying IP 10.xxx.xx.113 and failing as no DB connection available there . I want to break when the good IP and connection available . Please follow the lines after try , except :
I am pretty sure there is better way to write this construct pro grammatically in python but I am missing something .
try:
COUCHBASE_CONNSTR = "couchbase://10.xxx.xx.110:30493" # From outside the cluster (K8s target IP may not be static always )
try:
COUCHBASE_CONNSTR = "couchbase://10.xxx.xx.111:30493"
try:
COUCHBASE_CONNSTR = "couchbase://10.xxx.xx.112:30493" # From outside the cluster (K8s target IP may not be static always )
try:
COUCHBASE_CONNSTR = "couchbase://10.80.xx.113:30493"
except:
print("3")
except:
print("4")
except:
print("5")
except:
print("6")
COUCHBASE_USER = "Administrator"
COUCHBASE_BUCKET_PASSWORD = "password"
cluster = Cluster("COUCHBASE_CONNSTR")
authenticator = PasswordAuthenticator(
"COUCHBASE_USER", "COUCHBASE_BUCKET_PASSWORD"
)
cluster.authenticate(authenticator)
cb = cluster.open_bucket("samplebucketname")
EDIT: Now contains connect procedure.
Just loop over the address space like this:
COUCHBASE_USER = "Administrator"
COUCHBASE_BUCKET_PASSWORD = "password"
authenticator = PasswordAuthenticator(
COUCHBASE_USER, COUCHBASE_BUCKET_PASSWORD
)
bucket = None
for num in range(110, 255):
ip = f"10.xxx.xx.{num}" # <-- replace x with your numbers
try:
cluster = Cluster(f"couchbase://{ip}:30493")
cluster.authenticate(authenticator)
bucket = cluster.open_bucket("samplebucketname")
# this assumes: no exception ==> connected,
# better check bucket itself
print(f"Successfully connected at {ip}")
break
except Exception as e: # <-- better specify actual expected exceptions!
print(f"Could not connect to {ip}: {e}")
# if still not connected
if bucket is None:
raise ValueError('Could not connect')
you could define all the urls in a dict and iterate over them,
urls = {"couchbase://10.xxx.xx.110:30493": "msg", "couchbase://10.xxx.xx.111:30493": "msg1"}
try:
for url, msg in urls.items():
COUCHBASE_CONNSTR = url
except:
print(msg)
I want to build a complex network of twitter followers.
I'm using the function api.GetFriends :
def get_friend_list_by_user (user, api) :
friends_lists = api.GetFriends(repr(user.id))
return friends_lists
The problem is that for the same twitter users, sometimes it works and sometimes doesn't.
When I'm debugging it,
the code is dead at that part on the api.py:
if enforce_auth:
if not self.__auth:
raise TwitterError("The twitter.Api instance must be authenticated.")
if url and self.sleep_on_rate_limit:
limit = self.CheckRateLimit(url)
if limit.remaining == 0:
try:
stime = max(int(limit.reset - time.time()) + 10, 0)
logger.debug('Rate limited requesting [%s], sleeping for [%s]', url, stime)
time.sleep(stime)
except ValueError:
pass
if not data:
data = {}
The stime value is 443.