Ive created a simple dns.query function, I am attempting to add the results into a list or potentially a dictionary. However I cant work out how to achieve it, I have tried list.append(subdomain, item), ive tried using the join function and I have tried to use the update function respectably.
Any pointers would be appreciated.
ORIGINAL
def get_brutes(subdomain):
targets = []
try:
myResolver = dns.resolver.Resolver()
myResolver.nameservers = ['8.8.8.8']
myAnswers = myResolver.query(subdomain)
for item in myAnswers.rrset:
targets.append(subdomain,item)
except Exception as e:
pass
return targets
FIX
def get_brutes(subdomain):
targets = []
try:
myResolver = dns.resolver.Resolver()
myResolver.nameservers = ['8.8.8.8']
myAnswers = myResolver.query(subdomain)
for item in myAnswers.rrset:
targets.append(subdomain + ' ' + str(item))
except Exception as e:
pass
return targets
Related
I use try/except to catch problems when reading a file line-by-line. The try block contains a series of manipulations, the last of which is usually the cause of the exception. Surprisingly, I noticed that all previous manipulations are executed within the try block even when an exception is raised. This is a problem when trying to turn the dictionary I created to a data frame, because the length of the lists is unequal.
This code creates the problem:
d = {'dates':[],'states':[], 'longitude':[], 'latitude':[], 'tweet_ids':[], 'user_ids':[], 'source':[]}
for file in f:
print("Processing file "+file)
t1 = file.split('/')[-1].split("_")
date = t1[0]
state_code = t1[1]
state = list(states_ref.loc[states_ref.code==state_code]['abbr'])[0]
collection = JsonCollection(file)
counter = 0
for tweet in collection.get_iterator():
counter += 1
try:
d['dates'].append(date)
d['states'].append(state)
t2 = tweet_parser.get_entity_field('geo', tweet)
if t2 == None:
d['longitude'].append(t2)
d['latitude'].append(t2)
else:
d['longitude'].append(t2['coordinates'][1])
d['latitude'].append(t2['coordinates'][0])
#note: the 3 lines bellow are the ones that can raise an exception
temp = tweet_parser.get_entity_field('source', tweet)
t5 = re.findall(r'>(.*?)<', temp)[0]
d['source'].append(t5)
except:
c += 1
print("Tweet {} in file {} had a problem and got skipped".format(counter, file))
print("This is a total of {} tweets I am missing from the {} archive I process.".format(c, sys.argv[1]))
next
tab = pd.DataFrame.from_dict(d)
I have fixed the problem by moving the manipulation that is prone to giving the error at the top, but I would like to better understand why try/except is behaving like this. Any ideas?
This code works:
d = {'dates':[],'states':[], 'longitude':[], 'latitude':[], 'tweet_ids':[], 'user_ids':[], 'source':[]}
for file in f:
print("Processing file "+file)
t1 = file.split('/')[-1].split("_")
date = t1[0]
state_code = t1[1]
state = list(states_ref.loc[states_ref.code==state_code]['abbr'])[0]
collection = JsonCollection(file)
counter = 0
for tweet in collection.get_iterator():
counter += 1
try:
#note: the 3 lines bellow are the ones that can raise an exception
temp = tweet_parser.get_entity_field('source', tweet)
t5 = re.findall(r'>(.*?)<', temp)[0]
d['source'].append(t5)
d['dates'].append(date)
d['states'].append(state)
t2 = tweet_parser.get_entity_field('geo', tweet)
if t2 == None:
d['longitude'].append(t2)
d['latitude'].append(t2)
else:
d['longitude'].append(t2['coordinates'][1])
d['latitude'].append(t2['coordinates'][0])
except:
c += 1
print("Tweet {} in file {} had a problem and got skipped".format(counter, file))
print("This is a total of {} tweets I am missing from the {} archive I process.".format(c, sys.argv[1]))
next
tab = pd.DataFrame.from_dict(d)
You could always use a temporal object to hold the output of your functions before appending to the target object. That way if something fails, it will raise an exception before putting data into the target object.
try:
#Put all data into a temporal Dictionary
#Can raise an exception here
temp = tweet_parser.get_entity_field('source', tweet)
t2 = tweet_parser.get_entity_field('geo', tweet)
tempDictionary = {
"source" : re.findall(r'>(.*?)<', temp)[0],
"latitude" : None if (t2 is None) else t2['coordinates'][1],
"longitude" : None if (t2 is None) else t2['coordinates'][0]
}
#Append data from temporal Dictionary
d['source'].append(tempDictionary['source'])
d['latitude'].append(tempDictionary['latitude'])
d['longitude'].append(tempDictionary['longitude'])
d['dates'].append(date)
d['states'].append(state)
except:
c += 1
print("Tweet {} in file {} had a problem and got skipped".format(counter, file))
print("This is a total of {} tweets I am missing from the {} archive I process.".format(c, sys.argv[1]))
I'm trying to retrieve a list of images in a directory, and store this list into a attribute of a class. But when i try to get the list from the object it appears empty.This is the class:
class Hunting(models.Model):
code = models.CharField(max_length=10)
description = models.CharField(max_length=500)
images = {}
def __str__(self):
res = ''
for image in self.images:
res = res+image
return res
def __unicode__(self):
return self.id
And then this is what i do to populate and read the list:
page = int(page)
huntings = Paginator(Hunting.objects.all(), 9)
images = {}
for hunting in huntings.page(page):
dir_name = settings.IMAGES_ROOT+'\\theme1\\images\\cotos\\'+str(hunting.id)+'\\' # insert the path to your directory
path = os.path.join(settings.STATIC_URL, dir_name)
print >>sys.stderr, '--PATH:' + path
if os.path.exists(path):
print >>sys.stderr, '--LIST IMAGE HUNTING '+str(hunting.id)+'--'
img_list = os.listdir(path)
hunting.images=img_list
print >>sys.stderr, hunting.images
for hunting in huntings.page(page):
print >>sys.stderr, '*******hunting '+str(hunting.id)+':'
print >>sys.stderr, hunting.images
So, when i am iterating the first time and printing the hunting.images list i got:
--LIST IMAGE HUNTING 2--
['1_tn.jpg', '2_tn.jpg', '3_tn.jpg', '4_tn.jpg']
--LIST IMAGE HUNTING 4--
['coto10.png', 'coto11.png', 'coto12.png']
But the second iteration, trying to get the list i stored before i get an empty list
*******hunting 2:
{}
*******hunting 4:
{}
I think what happened is that you call huntings.page(page) for both of the loops. The first loop you did assign the images to the result of huntings.page(page), but then you didn't keep the result, so for the second loop you call huntings.page(page) again and you got the same initial result as before.
I'm not sure what you want to achieve here, but you need to get the return value first in a variable then update it:
# keep the return value in variable huntings
huntings = [for hunting in huntings.page(page)]
for hunting in huntings:
# do your hunting.images = img_list
for hunting in huntings:
print >>sys.stderr, hunting.images
Basically trying to modify the this tutorial for Currency's instead of
stocks: http://pythonprogramming.net/advanced-matplotlib-graphing-charting-tutorial/
with my current code I get:
Error: main loop can only concatenate tuple (not "str") to tuple
The code:
import urllib2
import time
CurrencysToPull = 'audusd','eurusd','usdcad'
def pullData(Currency):
try:
fileline = Currency+'.txt'
urlToVisit = 'http://finance.yahoo.com/echarts?s=Currency=X#{"allowChartStacking":true}/'+Currency+'/chartdata;type=quote:range=1y/csv'
sourcecode = urllib2.urlopen(urlToVisit).read()
splitSource = sourcecode.split('\n')
for eachLine in splitSource:
splitLine = eachLine.split(',')
if len(splitLine)==6:
if 'valuse' not in eachLine:
saveFile = open(fileLine,'a')
lineToWrite = eachLine+'\n'
saveFile.write(lineToWrite)
print 'Pulled', Currency
print 'sleeping'
time.sleep(1)
except Exception,(e):
print('main loop'), str(e)
for eachStock in CurrencysToPull:
pullData(CurrencysToPull)
You are passing in the CurrencysToPull tuple to your function:
for eachStock in CurrencysToPull:
pullData(CurrencysToPull)
which you then try to concatenate a string to:
fileline = Currency+'.txt'
You probably meant to pass in eachStock instead:
for eachStock in CurrencysToPull:
pullData(eachStock)
The error is in this line :
fileline = Currency+'.txt'
Currency is a tuple, .txt is a string
In your for loop you are passing CurrencysToPull instead of eachStock.
it should be:
for eachStock in CurrencysToPull:
You could have got better information about the errors using traceback in your exception handling.
except Exception,(e):
print('main loop'), str(e)
print(traceback.format_exc())
I am trying to use "directory path" and "prefirx_pattern" from config file.
I get correct results in vdir2 and vprefix2 variable but list local_file_list is still empty.
result
vdir2 is"/home/ab_meta/abfiles/"
vprefix2 is "rp_pck."
[]
code
def get_files(self):
try:
print "vdir2 is" + os.environ['dir_path']
print "vprefix2 is "+ os.environ['prefix_pattern']
local_file_list = filter(os.path.isfile, glob.glob(os.environ['dir_path'] + os.environ['prefix_pattern'] + "*"))
print local_file_list
local_file_list.sort(key=lambda s: os.path.getmtime(os.path.join(os.environ['dir_path'], s)))
except Exception, e:
print e
self.m_logger.error("Exception: Process threw an exception " + str(e))
log.sendlog("error",50)
sys.exit(1)
return local_file_list
I have tried another way as given below but again list is coming as empty.
2nd Option :
def get_config(self):
try:
v_dir_path = os.environ['dir_path']
v_mail_prefix = os.environ['mail_prefix']
self.m_dir_path = v_dir_path
self.m_prefix_pattern = v_prefix_pattern
self.m_mail_prefix = v_mail_prefix
except KeyError, key:
self.m_logger.error("ERROR: Unable to retrieve the key " + str(key))
except Exception, e:
print e
self.m_logger.error("Error: job_prefix Unable to get variables " + str(e))
sys.exit(1)
def get_files(self):
try:
local_file_list = filter(os.path.isfile, glob.glob(self.m_dir_path + self.m_prefix_pattern + "*"))
local_file_list.sort(key=lambda s: os.path.getmtime(os.path.join(os.environ['dir_path'], s)))
except Exception, e:
print e
Thanks
Sandy
Outside of this program, wherever you set the environment variables, you are setting them incorrectly. Your environment variables have quote characters in them.
Set your environment varaibles to have the path data, but no quotes.
Assign the enviornment variable and then pass the path you are interested in into the function.
Accessing global state from within your function can make it hard to follow and debug.
Use os.walk to get the list of files, it returns a tuple of the root dir, a list of dirs, and a list of files. For me its cleaner than using os.isfile to filter.
Use a list comprehension to filter the list of files returned by os.walk.
I'm presuming the prints statements are for debugging so left them out.
vdir2 = os.environ['dir_path']
vprefix2 = os.environ['prefix_pattern']
def get_files(vpath):
for root, dirs, files in os.walk(vpath):
local_file_list = [f for f in files if f.startswith(vprefix2)]
local_file_list.sort(key=lambda x: os.path.getmtime(x))
return local_file_list
import requests
import json
def decrementList(words):
for w in [words] + [words[:-x] for x in range(1,len(words))]:
url = 'http://ws.spotify.com/search/1/track.json?q='
request = requests.get(url + "%20".join(w))
json_dict = json.loads(request.content)
track_title = ' '.join(w)
for track in json_dict["tracks"]:
if track["name"].lower() == track_title.lower() and track['href']:
return "http://open.spotify.com/track/" + track["href"][14:], words[len(w):], track["href"][14:]
return "Sorry, no more track matches found!", None
if __name__ == "__main__":
message = "baby asdf".split()
size = len(message)
while message:
href, new_list, for_playlist = decrementList(message)
message = new_list
#print href
playlist = []
playlist.append(for_playlist)
print playlist
In the code above, print playlistis returning two separate lists. I realize that this is occurring because the list append happens within a while loop. How can I make these both append to the same empty list, not two separate lists?
make declaration of list and print out of while loop:
playlist = []
while message:
#....
print playlist
That's because you reassign the name playlist to an empty list before calling append():
playlist = []
If you put it before the while loop, you should get the expected result.
playlist = []
while message: