Python function returns inconsistent results - python

I wrote a script that lists EC2 instances in Amazon Web Services. It writes the results to confluence. But it's behaving oddly.
I'm on windows 10. The first time I open a powershell terminal and run it it reports the correct number of servers in the AWS account.
The next time I run it (without changing anything at all) it doubles the resutls. And each time you run it again with the same arguments it reports the same incorrect (doubled) amount.
This is the function that lists the instances and this is where I think the trouble is, but I'm having trouble finding it:
def list_instances(aws_account, aws_account_number, interactive, regions, fieldnames, show_details):
today, aws_env_list, output_file, output_file_name, fieldnames = initialize(interactive, aws_account)     options = arguments()     instance_list = ''     session = ''     ec2 = ''     account_found = ''     PrivateDNS = None     block_device_list = None     instance_count = 0     account_type_message = ''     profile_missing_message = ''     region = ''
# Set the ec2 dictionary
ec2info = {}
# Write the file headers
if interactive == 1:
with open(output_file, mode='w+') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=fieldnames, delimiter=',', lineterminator='\n')
            writer.writeheader()
if 'gov' in aws_account and not 'admin' in aws_account:
try:
session = boto3.Session(profile_name=aws_account,region_name=region)
account_found = 'yes'
except botocore.exceptions.ProfileNotFound as e:
message = f"An exception has occurred: {e}"
account_found = 'no'
banner(message)
else:
try:
session = boto3.Session(profile_name=aws_account,region_name=region)
account_found = 'yes'
except botocore.exceptions.ProfileNotFound as e:
message = f"An exception has occurred: {e}"
account_found = 'no'
banner(message)
print(Fore.CYAN)
report_gov_or_comm(aws_account, account_found)
print(Fore.RESET)
for region in regions:
if 'gov' in aws_account and not 'admin' in aws_account:
try:
session = boto3.Session(profile_name=aws_account,region_name=region)
except botocore.exceptions.ProfileNotFound as e:
profile_missing_message = f"An exception has occurred: {e}"                 account_found = 'no' pass else: try:                 session = boto3.Session(profile_name=aws_account,region_name=region)                 account_found = 'yes' except botocore.exceptions.ProfileNotFound as e:                 profile_missing_message = f"An exception has occurred: {e}" pass try:             ec2 = session.client("ec2") except Exception as e: pass
# Loop through the instances
try:
instance_list = ec2.describe_instances()
except Exception as e:
pass
try:
for reservation in instance_list["Reservations"]:
for instance in reservation.get("Instances", []):
instance_count = instance_count + 1
launch_time = instance["LaunchTime"]
launch_time_friendly = launch_time.strftime("%B %d %Y")
tree = objectpath.Tree(block_devices = set(tree.execute('$..BlockDeviceMappings['Ebs']['VolumeId']'))
if block_devices:
block_devices = list(block_devices)
block_devices = str(block_devices).replace('[','').replace(']','').replace("'",'')
else:
block_devices = None
private_ips =  set(tree.execute('$..PrivateIpAddress'))
if private_ips:
private_ips_list = list(private_ips)                         private_ips_list = str(private_ips_list).replace('[','').replace(']','').replace(''','')
else:
private_ips_list = None
type(private_ips_list)
public_ips =  set(tree.execute('$..PublicIp'))
if len(public_ips) == 0:
public_ips = None
if public_ips:
public_ips_list = list(public_ips)
public_ips_list = str(public_ips_list).replace('[','').replace(']','').replace("'",'')
else:
public_ips_list = None
if 'KeyName' in instance:
key_name = instance['KeyName']
else:
key_name = None
name = None
if 'Tags' in instance:
try:
tags = instance['Tags']
name = None
for tag in tags:
if tag["Key"] == "Name":
name = tag["Value"]
if tag["Key"] == "Engagement" or tag["Key"] == "Engagement Code":
engagement = tag["Value"]
except ValueError:
print("Instance: %s has no tags" % instance_id
pass
if 'VpcId' in instance:
vpc_id = instance['VpcId'] else:                         vpc_id = None if 'PrivateDnsName' in instance:                         private_dns = instance['PrivateDnsName'] else:                         private_dns = None if 'Platform' in instance:                         platform = instance['Platform'] else:                         platform = None print(f"Platform: {platform}")                     ec2info[instance['InstanceId']] = { 'AWS Account': aws_account, 'Account Number': aws_account_number, 'Name': name, 'Instance ID': instance['InstanceId'], 'Volumes': block_devices, 'Private IP': private_ips_list, 'Public IP': public_ips_list, 'Private DNS': private_dns, 'Availability Zone': instance['Placement']['AvailabilityZone'], 'VPC ID': vpc_id, 'Type': instance['InstanceType'], 'Platform': platform, 'Key Pair Name': key_name, 'State': instance['State']['Name'], 'Launch Date': launch_time_friendly                     } with open(output_file,'a') as csv_file:                         writer = csv.DictWriter(csv_file, fieldnames=fieldnames, delimiter=',', lineterminator='\n')                         writer.writerow({'AWS Account': aws_account, "Account Number": aws_account_number, 'Name': name, 'Instance ID': instance["InstanceId"], 'Volumes': block_devices,  'Private IP': private_ips_list, 'Public IP': public_ips_list, 'Private DNS': private_dns, 'Availability Zone': instance['Placement']['AvailabilityZone'], 'VPC ID': vpc_id, 'Type': instance["InstanceType"], 'Platform': platform, 'Key Pair Name': key_name, 'State': instance["State"]["Name"], 'Launch Date': launch_time_friendly})
if show_details == 'y' or show_details == 'yes': for instance_id, instance in ec2info.items(): if account_found == 'yes': print(Fore.RESET + "-------------------------------------") for key in [ 'AWS Account', 'Account Number', 'Name', 'Instance ID', 'Volumes', 'Private IP', 'Public IP', 'Private DNS', 'Availability Zone', 'VPC ID', 'Type', 'Platform', 'Key Pair Name', 'State', 'Launch Date'                                 ]: print(Fore.GREEN + f"{key}: {instance.get(key)}") print(Fore.RESET + "-------------------------------------") else: pass                     ec2info = {} with open(output_file,'a') as csv_file:                         csv_file.close() except Exception as e: pass if profile_missing_message == '*':         banner(profile_missing_message) print(Fore.GREEN)     report_instance_stats(instance_count, aws_account, account_found) print(Fore.RESET + '\n') return output_file
This is a paste of the whole code for context: aws_ec2_list_instances.py
The first time you run it from the command line it reports the correct total of servers in EC2 (can be verified in the AWS console):
----------------------------------------------------------
There are: 51 EC2 instances in AWS Account: company-lab.
----------------------------------------------------------
The next time it's run with ABSOLUTELY NOTHING changed it reports this total:
----------------------------------------------------------
There are: 102 EC2 instances in AWS Account: company-lab.
----------------------------------------------------------
You literally just up arrow the command and it doubles the results. When it writes to confluence you can see duplicate servers listed. It does that each time you up arrow to run it again, with the same incorrect total (102 servers).
If you close the powershell command line and open again it's back to reporting the correct result (51 servers) which corresponds to what you see in the AWS console.
What the heck is happening, here? Why is it doing that and how do I correct the problem?

This is pretty mysterious! I don't think I'm going to be able to debug it without access to your environment. My suggestion is that you use pdb to try to figure out how instance_count is being incremented past 51. I'd recommend adding import pdb; pdb.set_trace() at line 210, that is, right after for reservation in instance_list["Reservations"]:. You can then inspect the value of instance_count each time through the loop, see whether instance_list["Reservations"] actually has duplicate data somehow, etc.

Related

Pyetrade / Etrade API for option-chains function only returns options for apple?

I'm trying to get some option chains using the pyetrade package. I'm working in Sandbox mode for a newly made Etrade account.
When I execute the following code, it executes fine, but the returned information is incorrect: I keep getting options for Apple between 2012 and 2015, instead of current Exxon-Mobil options (what I'm inputting). This is also true for if I ask for Google or Facebook or Netflix, I just keep getting outdated Apple options.
I'm not sure where I messed up, or if this is just something that's part of sandbox mode, so that's why I asked for help. Thank you!
(Note: Some of the code is sourced from: https://github.com/1rocketdude/pyetrade_option_chains/blob/master/etrade_option_chains.py)
The following is the function to get the option chain from the API:
def getOption(thisSymbol):
#Renew session / or start session
try:
authManager.renew_access_token()
except:
authenticate() #this works fine
#make a market object to pull what you need from
market = pyetrade.ETradeMarket(
consumer_key,
consumer_secret,
tokens['oauth_token'],
tokens['oauth_token_secret'],
dev=True
)
try:
#the dates returned are also
q = market.get_option_expire_date(thisSymbol,resp_format='xml')
#just formats the dates to be more comprehensible:
expiration_dates = option_expire_dates_from_xml(q)
except Exception:
raise
rtn = []
for this_expiry_date in expiration_dates:
q = market.get_option_chains(thisSymbol, this_expiry_date)
chains = q['OptionChainResponse']['OptionPair']
rtn.append(chains)
print()
return rtn
ret = getOption("XOM")
print(ret[0])
The API provider is explicit on this:
Note:
E*TRADE's sandbox doesn't actually produce correct option chains so this will return an error.
The sandbox is still useful for debugging e.g. the OAuth stuff.
No one could hardly make the sandbox-ed code work otherwise.

Try/Exception giving multiple results in Python

I have this program that should take the user input (an application name installed) and then it will open the application with subprocess.Popen() . All posibles application names are in a dictionary as keys, and the value is the path to the application .exe file. I'm using a try/Exception to see if there will be a KeyError (if the input name doens't exist in the dictionary).
So the programm should take the user input, see if the application name is in the dictionary, and then open the application. If the name isn't on the dictionary, it will give an error message and then ask for a name again.
But when I enter a non-existent name n times, it will run the finally block n times too. How to solve this?
import subprocess
apps ={
"vlc":"app_path\\vlc.exe",
"sublime":"app_path\\sublime_text.exe",
"chrome":"app_path\\chrome.exe",
"opera":"app_path\\launcher.exe",
"torrent":"app_path.\\qbittorrent.exe"
}
def test():
answer = str(input("What program do you want to run: ")).lower()
try:
print(apps[answer])
except KeyError:
print(f"{answer.capitalize()} application unknown. Try again with a valid application name")
print("===============================================================")
test()
except:
print("==============================")
print("Unknown error.")
else:
subprocess.Popen(apps[answer])
finally:
print("===========================================")
print("FINISHED")
test()
You're using recursion here, so obviously the finally block will run as many times as you enter call the function.
The code within finally is always executed regardless if the try block raises an error or not.
W3Schools has good examples for this: https://www.w3schools.com/python/python_try_except.asp
Instead of using try/except, you can simply use a while loop and apps.get(answer, 'ERROR') or if answer in apps to check if the entered input is in the apps dictionary. The following solution may be optimal.
def test():
while True:
answer = input("What program do you want to run: ").lower()
if answer in apps:
try:
res = subprocess.Popen(apps[answer]).communicate()
print("===========================================")
print("FINISHED")
return res
except:
print("==============================")
print("Unknown error.")
else:
print(f"{answer.capitalize()} application unknown. Try again with a valid application name")
print("===============================================================")
test()

Create exception for like already assigned in the InstaPy library

I have the algorithm below that performs likes and comments on instagram photos via URL, my goal is to run this script on an AWS machine within 5 minutes which will generate 288 comments per day. However, whenever he realizes that he already executes LIKE in the photo, he returns "LIKED 0 images | ALREADY LIKED: 1", and aborts the process. I wanted to create a deal for when LIKE already exists, just execute the comment, is it possible? If yes you can help me with provisioning examples or the solution in question, thank you!
from instapy import InstaPy
from instapy import smart_run
def connect(insta_username,insta_password):
try:
session = InstaPy(username=insta_username,
password=insta_password,
headless_browser=False)
return session
except Exception as e:
print("[FAILED] Caused by: {}".format(e))
def action(url,insta_username,insta_password):
session = connect(insta_username,insta_password)
try:
with smart_run(session):
# Always follow and like before, the algorithm understands how it has already been executed when used in place of the like function
# Define interaction settings
session.set_do_comment(enabled=True, percentage=100)
session.set_comments(["#user-id"])
session.set_user_interact(amount=1, randomize=False, percentage=100, media='Photo')
# Start the feature
session.interact_by_URL(urls=url, randomize=False, interact=True)
session.end()
except Exception as e:
print("[FAILED] Caused by: {}".format(e))
if __name__ == "__main__":
pics = ["https://www.instagram.com/p/id-here"]
for url in pics:
action(url,'username','password')

Is there a way to detect whether a command prompt is available in python/django?

I have some code that will some times be run from a command line (django management command) and sometimes not (django changelist action). In this code, if a certain exception is raised, I can get some user input and keep going if the a command prompt (stdin) is available. Otherwise, I need to just let the exception propagate or do something different.
e.g.
def copy_account_settings(old_acct_domain, new_acct_domain):
try:
new_account = Account.objects.get(domain = new_acct_domain)
except Account.DoesNotExist:
print ("Couldn't find an account matching %s." % new_acct_domain)
if <command prompt is available>:
print "Would you like to create the account? (y/n)"
if raw_input().lower().strip()='y':
# get some more input and create the account and move on
else:
raise
How would you do this?
Perhaps you can check for a TTY?
import os
if os.isatty(0):
That should return true if the session is interactive and false if not.

PySVN - Determine if a repository exists

I'm writing a small script that manages several SVN repositories. Users pass through the ID of the repository they want to change (the root of the repos are of the form https://www.mydomain.com/).
I need to check if the given repo actually exists. I've tried using Client.list to see if I can find any files, like so:
client = pysvn.Client()
client.list("https://.../<username>/")
But if the repo does not exist then the script hangs on the list line. From digging through the tracebacks it looks like pysvn is actually hanging on the login credentials callback (client.callback_get_login - which I have implemented but omitted, it does not fail if the repo exists).
Can you suggest how I can determine if a repo exists or not using pysvn?
Cheers,
Pete
I couldn't reproduce your hanging in credentials callback problem, so it might need an expanded description of the problem. I'm running pysvn 1.7.2 on Ubuntu 10.04, Python 2.6.6.
When I try to list a non-existent remote repository with client.list() it raises an exception. You could also use client.info2() to check for existence of a remote repository:
head_rev = pysvn.Revision(pysvn.opt_revision_kind.head)
bad_repo = 'https://.../xyz_i_dont_exist'
good_repo = 'https://.../real_project'
for url in (bad_repo, good_repo):
try:
info = client.info2(url, revision=head_rev, recurse=False)
print url, 'exists.'
except pysvn._pysvn_2_6.ClientError, ex:
if 'non-existent' in ex.args[0]:
print url, 'does not exist'
else:
print url, 'error:', ex.args[0]
Peter,
My team and I have experienced the same challenge. Samplebias, try providing a callback_get_login function but set your callback_server_ssl_trust_prompt to return (True, trust_dict['failures'], True). IFF subversion has not cached your server certificate trust settings, then you may find the info2() (or Peter's list() command) hangs (it's not actually hanging, it just takes intermittently much longer time to return). Oddly, when you CTRL-C the interpreter in these scenarios, you'll get indication that it hung on the login callback, not the server_cert verification. Play around with your ~/.subversion/auth settings (in particular the svn.simple and svn.ssl.server directories) and you'll see different amounts of 'hang time'. Look at pysvn.Client.callback_cancel if you need to handle situations which truly never return.
Considering: http://pysvn.tigris.org/docs/pysvn_prog_ref.html#pysvn_client_callback_ssl_server_trust_prompt you need to decide what your desired behavior is. Do you want ONLY to allow those connections for which you already have a cached trust answer? Or, do you want to ALWAYS accept regardless of server certificate verification (WARNING: this could (obviously) have negative security implications). Consider the following suggestion:
import pysvn
URL1 = "https://exists.your.org/svn/repos/dev/trunk/current"
URL2 = "https://doesntexit.your.org/svn/repos/dev/trunk/current"
URL3 = "https://exists.your.org/svn/repos/dev/trunk/youDontHavePermissionsBranch"
ALWAYS = "ALWAYS"
NEVER = "NEVER"
DESIRED_BEHAVIOR = ALWAYS
def ssl_server_certificate_trust_prompt(trust_dict):
if DESIRED_BEHAVIOR == NEVER:
return (False, 0, False)
elif DESIRED_BEHAVIOR == ALWAYS:
return (True, trust_dict['failures'], True)
raise Exception, "Unsupported behavior"
def testURL(url):
try:
c.info2(url)
return True
except pysvn.ClientError, ce:
if ('non-existant' in ce.args[0]) or ('Host not found' in ce.args[0]):
return False
else:
raise ce
c = pysvn.Client()
c.callback_ssl_server_trust_prompt = lambda t: (False, t['failures'], True)
c.callback_get_login = lambda x, y, z: (True, "uname", "pw", False)
if not testURL(URL1): print "Test1 failed."
if testURL(URL2): print "Test2 failed."
try:
testURL(URL3)
print "Test3 failed."
except: pass
In actuality, you probably don't want to get as fancy as I have with the return values. I do think it was important to consider a potential 403 returned by the server and the "Host not found" scenario separately.

Categories