I have an interesting behavior happening with my program.
i have the following methods:
def getMarket(self, Currency):
return self.public_api('GetMarket/' + Currency + '_BTC')
def getBalance(self, Currency):
self.api_params.clear()
self.api_params['Currency'] = Currency
return self.private_api('GetBalance')
my_api = buyBot(API_KEY, API_SECRET)
pumpCoin = my_api.getMarket('OSC')
pumpRawRate = pumpCoin['Data']['High']
pumpRawQty = .02
pumpBuyRate = my_api.calculateBuy(pumpRawRate)
pumpQty = float(pumpRawQty)/float(pumpBuyRate)
pumpSellRate = pumpCoin['Data']['Low']
pumpSellCoin = my_api.getBalance('OSC')
pumpSellAmount = pumpSellCoin["Data"]["Total"]
print str(pumpRawRate) + '\n' + str(pumpBuyRate) + '\n' + str(pumpSellRate) + '\n' + str(pumpQty) + '\n' + str(pumpSellAmount)`
From section: pumpCoin = my_api.getMarket('OSC') to pumpSellRate = pumpCoin['Data']['Low'], i have no problems getting the information and working with it.
Problem seems to be starting with line: pumpSellCoin = my_api.getBalance('OSC')
I get the following Error message:
Traceback (most recent call last):
File "C:\XXXXXX.py", line 92, in <module>
pumpSellAmount = pumpSellCoin["Data"]["Total"]
TypeError: string indices must be integers, not str
if i run: print (my_api.getBalance('OSC'), i am able to see all the private API information that is retrieved by that call, however i am not sure why it is giving me a problem when i try to call 1 specific item in the stack.
Let me know if you need any more information on this.
Any help will be greatly appreciated.
I have looked at the other posts and so far i can't seem to figure out the exact cause.
This is the private_api code
def private_api(self, meth):
time.sleep(1)
params = self.api_params
url = self.apisite + meth
nonce = str(int(time.time()))
post_data = json.dumps(params)
hash = hashlib.md5()
hash.update(post_data)
base64hash = base64.b64encode(hash.digest())
sig = self.apikey + "POST" + urllib.quote_plus(url).lower() + nonce + base64hash
hmacsig = base64.b64encode(hmac.new(base64.b64decode(self.apisecret), sig, hashlib.sha256).digest())
hdr = "amx " + self.apikey + ":" + hmacsig + ":" + nonce
headers = { 'Authorization': hdr, 'Content-Type':'application/json; charset=utf-8' }
request = urllib2.Request(url, data=post_data, headers=headers)
return urllib2.urlopen(request).read()
Please add this to your code:
print('pumpSellCoin', type(pumpSellCoin["Data"]), type(pumpSellCoin["Data"]["Total"]))
pumpSellAmount = pumpSellCoin["Data"]["Total"]
This will show you that one of your variables is a list or a string and not a dictionary and you need to access is using a number and not a name like "Data" or "Total"
Try this example:
test = 'abcde'
print(type(test))
print(test[0])
print(test[2:4])
print(test['whatever']) # this results in TypeError: string indices must be integers
if i run the program as follows:
my_api = buyBot(API_KEY, API_SECRET)
pumpCoin = my_api.getMarket('OSC')
pumpRawRate = pumpCoin['Data']['High']
pumpRawQty = .02
pumpBuyRate = my_api.calculateBuy(pumpRawRate)
pumpQty = float(pumpRawQty)/float(pumpBuyRate)
pumpSellRate = pumpCoin['Data']['Low']
pumpSellBal = my_api.getBalance('OSC')
print pumpSellBal
#print('pumpSellBal', type(pumpSellBal["Data"]), type(pumpSellBal["Data"]["Total"]))
#pumpSellAmount = pumpSellBal['Data']['Total']
print str(pumpRawRate) + '\n' + str(pumpBuyRate) + '\n' + str(pumpSellRate) + '\n' + str(pumpQty) #+ '\n' + str(pumpSellAmount)
i get the following results:
{"Success":true,"Error":null,"Data":[{"CurrencyId":235,"Symbol":"OSC","Total":8561.03652012,"Available":0.00000000,"Unconfirmed":0.00000000,"HeldForTrades":8561.03652012,"PendingWithdraw":0.00000000,"Address":null,"Status":"OK","StatusMessage":null,"BaseAddress":null}]}
1.61e-06
2.415e-06
1.25e-06
8281.57349896
So i am definitely able to communicate back and forward, however the issue only seems to be when i try to work with a single piece of information from pumpSellBal = my_api.getBalance('OSC')
Related
I am writing a Python script that will use the .getsessiontoken() method and write the new temporary credentials to the aws credentials file at ~/.aws/credentials. The profile, in my case will be named [temp-token].
I can write to the file successfully, but then I thought, what if that profile already exists. I would have to find and replace subsequent lines after the profile name. I'm still new to Python and trying to understand what the problem is:
import os
import boto3 as b
mfa_serial_number = "arn:aws:iam::xxxxxxxxxxxx:mfa/your-iam-user-here"
mfa_otp = input("Enter your otp:" )
def get_sts(mfa_serial_number, mfa_otp):
session = b.Session(profile_name='default', region_name='us-east-1')
sts = session.client('sts')
if mfa_serial_number is not None:
response = sts.get_session_token(
SerialNumber=mfa_serial_number, TokenCode=mfa_otp)
else:
response = sts.get_session_token()
temp_credentials = response['Credentials']
print(temp_credentials)
token_reponse = ["[temp-token] \n", "aws_access_key_id = " + temp_credentials["AccessKeyId"], "\n", "aws_secret_access_key = " + temp_credentials["SecretAccessKey"],"\n", "aws_session_token = " + temp_credentials["SessionToken"],"\n"]
# Look for the temp-token profile
pfile = "[temp-token]"
with open("/Users/YOUR USER/.aws/credentials", "r") as credentials:
aws_credential_file = credentials.readlines()
#print(aws_credential_file)
for line in aws_credential_file:
if line.find(pfile) != -1:
print('Profile Exists!')
#Prints the line number where the word was found
profile_line = aws_credential_file.index(line)
new_aws_access_key_line_num = profile_line + 1
new_secret_access_key_line_num = profile_line + 2
new_session_token_key_line_num = profile_line + 3
print("its still working", profile_line)
print(new_aws_access_key_line_num)
aws_credential_file[new_aws_access_key_line_num] = "aws_access_key_id = " + temp_credentials["AccessKeyId"], "\n"
# aws_credential_file[new_secret_access_key_line_num] = "aws_secret_access_key = " + temp_credentials["SecretAccessKey"],"\n"
# aws_credential_file[new_session_token_key_line_num] = "aws_session_token = " + temp_credentials["SessionToken"],"\n"
#print()
else:
aws_credential_file.writelines(token_reponse)
credentials.close()
get_sts(mfa_serial_number, mfa_otp)
The error I get is:
its still working 3
4
Traceback (most recent call last):
File "/Users/YOUR USER/code/sts.py", line 49, in
get_sts(mfa_serial_number, mfa_otp)
File "/Users/YOUR USER/code/sts.py", line 29, in get_sts
if line.find(pfile) != -1:
AttributeError: 'tuple' object has no attribute 'find'
Is there a better way to do this?
It's because of the ,\n in aws_credential_file
I am trying to call several api datasets within a for loop in order to change the call and then append those datasets together into a larger dataframe.
I have written this code which works to call the first dataset but then returns this error for the next call.
`url = base + "max=" + maxrec + "&" "type=" + item + "&" + "freq=" + freq + "&" + "px=" +px + "&" + "ps=" + str(ps) + "&" + "r="+ r + "&" + "p=" + p + "&" + "rg=" +rg + "&" + "cc=" + cc + "&" + "fmt=" + fmt
TypeError: must be str, not Response`
Here is my current code
import requests
import pandas as pd
base = "http://comtrade.un.org/api/get?"
maxrec = "50000"
item = "C"
freq = "A"
px="H0"
ps="all"
r="all"
p="0"
rg="2"
cc="AG2"
fmt="json"
comtrade = pd.DataFrame(columns=[])
for year in range(1991,2018):
ps="{}".format(year)
url = base + "max=" + maxrec + "&" "type=" + item + "&" + "freq=" + freq + "&" + "px=" +px + "&" + "ps=" + str(ps) + "&" + "r="+ r + "&" + "p=" + p + "&" + "rg=" +rg + "&" + "cc=" + cc + "&" + "fmt=" + fmt
r = requests.get(url)
x = r.json()
new = pd.DataFrame(x["dataset"])
comtrade = comtrade.append(new)
Let requests assemble the URL for you.
common_params = {
"max": maxrec,
"type": item,
"freq": freq,
# etc
}
for year in range(1991,2018):
response = requests.get(base, params=dict(common_params, ps=str(year))
response_data = response.json()
new = pd.DataFrame(response_data["dataset"])
comtrade = comtrade.append(new)
Disclaimer: the other answer is correct and you should use it.
However, your actual problem stems from the fact that you are overriding r here:
r = requests.get(url)
x = r.json()
During the next iteration r will still be that value and not the one you initialized it with in the first place. You could simply rename it to result to avoid that problem. Better let the requests library do the work though.
I am making a program that searches the computer for .fts and .fits files in which it opens the file and retrieves info that corresponds to a specific keyword in the header and renames the file to that keyword.
I am having a problem where i keep receiving a KeyError becasue a header keyword I am searching for is not found in the file. Is there a way around this? I want to be able to search various keywords and do something even if that key word does not exist in the file.
Here is code:
from astropy.io import fits
import os
for i in os.listdir(os.getcwd()):
if i.endswith(".fits") or i.endswith(".fts"):
hdulist = fits.open(i)
DATEOBS_header = hdulist[0].header['DATE-OBS']
EXPTIME_header = int(round(hdulist[0].header['EXPTIME']))
CCDTEMP_header = int(round(hdulist[0].header['CCD-TEMP']))
XBINNING_header = hdulist[0].header['XBINNING']
FILTER_header = hdulist[0].header['FILTER']
IMAGETYP_header = hdulist[0].header['IMAGETYP']
OBJECT_header = hdulist[0].header['OBJECT']
DATEandTIME = DATEOBS_header[0:]
YEAR = DATEandTIME[0:4]
MONTH = DATEandTIME[5:7]
DAY = DATEandTIME[8:10]
#TIME = DATEOBS_header[11:]
HOUR = DATEandTIME[11:13]
MINUTE = DATEandTIME[14:16]
SECONDS = DATEandTIME[17:]
DATE = str(YEAR) + str(MONTH) + str(DAY) + 'at' + str(HOUR) + str(MINUTE) + str(SECONDS)
if IMAGETYP_header == 'Light Frame':
newname = str(OBJECT_header) + '_' + str(DATE) + '_' + str(CCDTEMP_header) + 'temp_' + str(XBINNING_header) + 'bin_' + str(EXPTIME_header) + 'exptime_' + str(FILTER_header) + '.fits'
if IMAGETYP_header == 'Dark Frame':
newname = 'Dark_' + str(DATE) + 'at' + str(TIME) + '_' + str(CCDTEMP_header) + 'temp_' + str(XBINNING_header) + 'bin_' + str(EXPTIME_header) + 'exptime' + '.fits'
if IMAGETYP_header == 'Flat Field':
newname = 'Flat_' + str(DATE) + 'at' + str(TIME) + '_' + str(CCDTEMP_header) + 'temp_' + str(XBINNING_header) + 'bin_' + str(EXPTIME_header) + 'exptime_' + str(FILTER_header) + '.fits'
prevname = i
os.rename(prevname, newname)
hdulist.close()
continue
else:
continue
This is the Error I get:
Traceback (most recent call last):
File "glo1.py", line 9, in <module>
DATEOBS_header = hdulist[0].header['DATE-OBS']
File "/home/luisgeesb/.local/lib/python2.7/site-packages/astropy/io/fits/header.py", line 151, in __getitem__
card = self._cards[self._cardindex(key)]
File "/home/luisgeesb/.local/lib/python2.7/site-packages/astropy/io/fits/header.py", line 1723, in _cardindex
raise KeyError("Keyword %r not found." % keyword)
KeyError: "Keyword 'DATE-OBS' not found."
To prevent these kinds of exceptions from stopping your program, you can either catch them, like this:
try:
DATEOBS_header = hdulist[0].header['DATE-OBS']
except KeyError:
DATEOBS_header = None
Or, use the .get() method of dictionaries, which checks if a key exists and if it doesn't returns a default value, instead of raising an exception. The default value returned is None.
If you do this, you will also need to set some sensible defaults, or catch those cases where you are casting the values (since you cannot cast None).
Finally, whenever you are reading from files - you should always assume the data is malformed/junk and do a bit of defensive programming. In your code, you are assuming that the values returned for CCDTEMP is a number, but what if the file is corrupted or has a blank? Your application doesn't handle this case.
Here is some code that attempts to catch as many errors as possible:
DATEOBS_header = hdulist[0].header.get('DATE-OBS')
XBINNING_header = hdulist[0].header.get('XBINNING')
FILTER_header = hdulist[0].header.get('FILTER')
IMAGETYP_header = hdulist[0].header.get('IMAGETYP')
OBJECT_header = hdulist[0].header.get('OBJECT')
# For these two, you need to either set a default
# Here I am setting the default to 0, ------------v
EXPTIME_header = int(round(hdulist[0].header.get('EXPTIME', 0)))
# Or you need to check for a value :
ccdtemp_value = hdulist[0].header.get('CCD-TEMP')
try:
ccdtemp_value = int(round(ccdtemp_value))
except ValueError:
# This means, the value was either None (header does not exist)
# or it was something that can't be converted to a number
# since it cannot be converted to a number, we do not know
# if the value is None or something like an empty string,
# so we explicitly set the value to None
ccdtemp_value = None
CCDTEMP_header = ccdtemp_value
Assuming hdulist[0].header gives you a dict instance, you can do something like
DATEOBS_header = hdulist[0].header.get('DATE-OBS')
Which would return a None if the key 'DATE-OBS' does not exist.
See https://docs.python.org/2/library/stdtypes.html#dict.get for more details.
So this is pretty much a web scraping program. I feel as if it is nearly finished, but I have no clue how to fix this!
Traceback (most recent call last):
File "AgriMet4.py", line 424, in <module>
orig_column = convert(station, webpage, data_type)
File "AgriMet4.py", line 244, in convert
ag_date, ag_time, ag_data_str = line.split()
ValueError: too many values to unpack
Here are the parts that are flagged:
# Break line into components
ag_date, ag_time, ag_data_str = line.split()
ag_datetime = ag_date + " " + ag_time
ag_datetime = datetime.datetime.strptime(ag_datetime, "%m/%d/%Y %H:%M")
ag_data = float(ag_data_str)
and
columns = []
for data_type in data_types:
webpage = download(station, data_type, effective_begin_date, effective_end_date)
orig_column = convert(station, webpage, data_type)
std_column = fill_gaps(orig_column, interval, data_type,
effective_begin_date, effective_end_date)
adjusted_column = adjust_datetimes(station, time_zone_offset, std_column)
columns.append(adjusted_column)
EDIT: I've made the code shorter. Sorry!
As the error indicates, there are too many values on the right hand side of your equation (line.split()).
You can find the documentation for string splitting. You seem to have more than 3 values to assign. If you don't know how many values you will get, you can use the * syntax to hold any remaining values.
line = "Hello there world"
word1, *remaining_words = line.split()
I've figured out my problem!
My download definition had the incorrect data_types, which caused it to gather all six-seven data types at once instead of one at a time.
def download(station, data_types, begin_date, end_date):
query_string = ('station=' + station
+ '&year=' + str(begin_date.year)
+ '&month=' + str(begin_date.month)
+ '&day=' + str(begin_date.day)
+ '&year=' + str(end_date.year)
+ '&month=' + str(end_date.month)
+ '&day=' + str(end_date.day)
+ '&pcode=' + data_types)
url = AGRIMET_WEBSITE + '?' + query_string
i'm getting json data from the facebook-graph-api about:
my relationship with my friends
my friends relationships with each other.
right now my program looks like this (in python pseudo code, please note some variables have been changed for privacy):
import json
import requests
# protected
_accessCode = "someAccessToken"
_accessStr = "?access_token=" + _accessCode
_myID = "myIDNumber"
r = requests.get("https://graph.facebook.com/" + _myID + "/friends/" + _accessStr)
raw = json.loads(r.text)
terminate = len(raw["data"])
# list used to store the friend/friend relationships
a = list()
for j in range(0, terminate + 1):
# calculate terminating displacement:
term_displacement = terminate - (j + 1)
print("Currently processing: " + str(j) + " of " + str(terminate))
for dj in range(1, term_displacement + 1):
# construct urls based on the raw data:
url = "https://graph.facebook.com/" + raw["data"][j]["id"] + "/friends/" + raw["data"][j + dj]["id"] + "/" + _accessStr
# visit site *THIS IS THE BOTTLENECK*:
reqTemp = requests.get(url)
rawTemp = json.loads(reqTemp.text)
if len(rawTemp["data"]) != 0:
# data dumps to list which dumps to file
a.append(str(raw["data"][j]["id"]) + "," + str(rawTemp["data"][0]["id"]))
outputFile = "C:/Users/franklin/Documents/gen/friendsRaw.csv"
output = open(outputFile, "w")
# write all me/friend relationship to file
for k in range(0, terminate):
output.write(_myID + "," + raw["data"][k]["id"] + "\n")
# write all friend/friend relationships to file
for i in range(0, len(a)):
output.write(a[i])
output.close()
So what its doing is: first it calls my page and gets my friend list (this is allowed through the facebook api using an access_token) calling a friend's friend list is NOT allowed but I can work around that by requesting a relationship between a friend on my list and another friend on my list. so in part two (indicated by the double for loops) i'm making another request to see if some friend, a, is also a friend of b, (both of which are on my list); if so there will be a json object of length one with friend a's name.
but with about 357 friends there's literally thousands of page requests that need to be made. in other words the program is spending a lot of time just waiting around for the json-requests.
my question is then can this be rewritten to be more efficient? currently, due to security restrictions, calling a friend's friend list attribute is disallowed. and it doesn't look like the api will allow this. are there any python tricks that can make this run faster? maybe parallelism?
Update modified code is pasted below in the answers section.
Update this is the solution I came up with. Thanks #DMCS for the FQL suggestion but I just decided to use what I had. I will post the FQL solution up when I get a chance to study the implementation. As you can see this method just makes use of more condensed API calls.
Incidentally for future reference the API call limit is 600 calls per 600 seconds, per token & per IP, so for every unique IP address, with a unique access token, the number of calls is limited to 1 call per second. I'm not sure what that means for asynchronous calling #Gerrat, but there is that.
import json
import requests
# protected
_accessCode = "someaccesscode"
_accessStr = "?access_token=" + _accessCode
_myID = "someidnumber"
r = requests.get("https://graph.facebook.com/"
+ _myID + "/friends/" + _accessStr)
raw = json.loads(r.text)
terminate = len(raw["data"])
a = list()
for k in range(0, terminate - 1):
friendID = raw["data"][k]["id"]
friendName = raw["data"][k]["name"]
url = ("https://graph.facebook.com/me/mutualfriends/"
+ friendID + _accessStr)
req = requests.get(url)
temp = json.loads(req.text)
print("Processing: " + str(k + 1) + " of " + str(terminate))
for j in range(0, len(temp["data"])):
a.append(friendID + "," + temp["data"][j]["id"] + ","
+ friendName + "," + temp["data"][j]["name"])
# dump contents to file:
outputFile = "C:/Users/franklin/Documents/gen/friendsRaw.csv"
output = open(outputFile, "w")
print("Dumping to file...")
# write all me/friend relationships to file
for k in range(0, terminate):
output.write(_myID + "," + raw["data"][k]["id"]
+ ",me," + str(raw["data"][k]["name"].encode("utf-8", "ignore")) + "\n")
# write all friend/friend relationships to file
for i in range(0, len(a)):
output.write(str(a[i].encode("utf-8", "ignore")) + "\n")
output.close()
This isn't likely optimal, but I tweaked your code a bit to use Requests async method (untested):
import json
import requests
from requests import async
# protected
_accessCode = "someAccessToken"
_accessStr = "?access_token=" + _accessCode
_myID = "myIDNumber"
r = requests.get("https://graph.facebook.com/" + _myID + "/friends/" + _accessStr)
raw = json.loads(r.text)
terminate = len(raw["data"])
# list used to store the friend/friend relationships
a = list()
def add_to_list(reqTemp):
rawTemp = json.loads(reqTemp.text)
if len(rawTemp["data"]) != 0:
# data dumps to list which dumps to file
a.append(str(raw["data"][j]["id"]) + "," + str(rawTemp["data"][0]["id"]))
async_list = []
for j in range(0, terminate + 1):
# calculate terminating displacement:
term_displacement = terminate - (j + 1)
print("Currently processing: " + str(j) + " of " + str(terminate))
for dj in range(1, term_displacement + 1):
# construct urls based on the raw data:
url = "https://graph.facebook.com/" + raw["data"][j]["id"] + "/friends/" + raw["data"][j + dj]["id"] + "/" + _accessStr
req = async.get(url, hooks = {'response': add_to_list})
async_list.append(req)
# gather up all the results
async.map(async_list)
outputFile = "C:/Users/franklin/Documents/gen/friendsRaw.csv"
output = open(outputFile, "w")