We Keep Coding

Python AWS Lambda Url invokes twice

I seem to have a problem with my lambda function. My function invokes only once when I test it on AWS. However, when I invoke the lambda function through the function URL it executes twice! Has anyone experienced this before?
import json
import boto3
dynamodb = boto3.resource('dynamodb')
ddbClient = boto3.client('dynamodb')
def LatestDBrow():
...
return row
#Invoke lambda function
def lambda_handler(event, context):
scratchPadlist = []
scratchPadlist = LatestDBrow()
#print(scratchPadlist)
#Put TS row into variables
inBatchTrial = scratchPadlist[0]
inNoOfUsersProcessed = scratchPadlist[1]
inl1Alpha = scratchPadlist[2]
inl1Beta = scratchPadlist[3]
#Update wanted values
inBatchTrial += 1
inl1Alpha = 1
inl1Beta = 1
TableColNames = ['Batch Trial','No. of users processed','L1 Alpha',
'L1 Beta','L1 N','L1 Rewards',
'L1 Click Pct','L2 Alpha','L2 Beta',
'L2 N','L2 Rewards','L2 Click Pct',
'Next Page Shown']
TSlist = [inBatchTrial, inNoOfUsersProcessed, inl1Alpha,
inl1Beta, inl1N, inl1Rewards,
inl1ClickPct, inl2Alpha, inl2Beta,
inl2N, inl2Rewards,inl2ClickPct,
innextPageShown]
dbDict = {TableColNames[i]: str(TSlist[i]) for i in range(len(TableColNames))}
table1 = dynamodb.Table("TestDB")
table1.put_item(Item = dbDict)

How to retrieve Redis database values in order of key using Python

I have a python application where a client retrieves csv data row by row from a server using a grpc stream. Data from each row is added to a dictionary which in turn is saved to a redis database. When I attempt to retrieve the data from the redis database in a seperate flask application, the data is not coming out in order and is duplicating much of the time. How can I retrieve the data in order of the key without duplicates?
Client
def run():
#Average number of comments metric
average_num_comments = 0
response_count = 0
comment_count = 0
try:
conn = redis.StrictRedis(host='redis', port=6379)
conn.flushdb()
except Exception as ex:
print('Error:', ex)
while True:
with grpc.insecure_channel('redditserver:50051') as channel:
stub = route_guide_pb2_grpc.RouteGuideStub(channel)
responses = stub.SendRedditPost(route_guide_pb2.PostRequestReddit(response='Recieved'))
#Single post with most letters in title
lg_post_title = ''
for response in responses:
response_count += 1
comment_count = int(response.num_comments) + comment_count
average_num_comments = avg(response_count, comment_count)
if (len(response.title) > len(lg_post_title)):
lg_post_title = response.title
redisdict = {"Largest Post Title": lg_post_title, "Comment Count": comment_count, "Average No. Comments": average_num_comments}
try:
conn = redis.StrictRedis(host='redis', port=6379)
conn.hmset(response_count, redisdict)
except Exception as ex:
print('Error:', ex)
time.sleep(2)
Flask Application
[![#app.route('/')
def get_page():
data = ''
try:
conn = redis.StrictRedis(host='redis', port=6379, decode_responses=True)
for key in conn.scan_iter():
value = conn.hgetall(key)
data = value
time.sleep(2)
print("KEY: " + key, file=sys.stderr)
print(data, file=sys.stderr)
except Exception as ex:
data = 'Error:' + str(ex)
return render_template('index.html', x=data)][1]][1]

Python access the Login directly without using With Function as Key :

Here is the complete code where i am trying to use ForexConnect().get_history(.... instead of fx.get_history( and i do not want this line of code "with ForexConnect() as fx:" how to achieve this ,the last section of code given produces excpetion issues .
Why i do not want to use "with ForexConnect() as fx:" The reason is once the session "with ForexConnect() as fx:" the function is logged out .My idea is to be in the session after once logged in .So i do not want to try this with "with ForexConnect() as fx:"
import argparse
import pandas as pd
from forexconnect import ForexConnect, fxcorepy
import common_samples
parser = False
def parse_args():
parser = argparse.ArgumentParser(description='Process command parameters.')
common_samples.add_main_arguments(parser)
common_samples.add_instrument_timeframe_arguments(parser)
common_samples.add_date_arguments(parser)
common_samples.add_max_bars_arguments(parser)
args = parser.parse_args()
return args
def main():
if parser == False :
#args = parse_args()
str_user_id = 'Dxxxx'
str_password = 'xxxxx'
str_url = "http://www.fxcorporate.com/Hosts.jsp"
str_connection = 'Demo'
str_session_id = 'None'
str_pin = 'None'
str_instrument = 'USOil'
str_timeframe = 'W1'
quotes_count = 5
date_from = None
date_to = None
else :
args = parse_args()
str_user_id = args.l
str_password = args.p
str_url = args.u
str_connection = args.c
str_session_id = args.session
str_pin = args.pin
str_instrument = args.i
str_timeframe = args.timeframe
quotes_count = args.quotescount
date_from = args.datefrom
date_to = args.dateto
with ForexConnect() as fx:
try:
fx.login(str_user_id, str_password, str_url,
str_connection, str_session_id, str_pin,
common_samples.session_status_changed)
print("")
print("Requesting a price history...")
print(str_instrument,str_timeframe,date_from,date_to,quotes_count)
history = fx.get_history(str_instrument, str_timeframe, date_from, date_to, quotes_count)
current_unit, _ = ForexConnect.parse_timeframe(str_timeframe)
date_format = '%d.%m.%Y %H:%M:%S'
print("print history ",history)
df = pd.DataFrame(history, columns=['Date', 'BidOpen', 'BidHigh','BidLow', 'BidClose', 'AskOpen', 'AskHigh', 'AskLow', 'AskClose', 'Volume'])
print(df)
if current_unit == fxcorepy.O2GTimeFrameUnit.TICK:
print("Date, Bid, Ask")
print(history.dtype.names)
for row in history:
print("{0:s}, {1:,.5f}, {2:,.5f}".format(
pd.to_datetime(str(row['Date'])).strftime(date_format), row['Bid'], row['Ask']))
else:
print("Date, BidOpen, BidHigh, BidLow, BidClose, Volume")
for row in history:
print("{0:s}, {1:,.5f}, {2:,.5f}, {3:,.5f}, {4:,.5f}, {5:d}".format(
pd.to_datetime(str(row['Date'])).strftime(date_format), row['BidOpen'], row['BidHigh'],
row['BidLow'], row['BidClose'], row['Volume']))
except Exception as e:
common_samples.print_exception(e)
try:
fx.logout()
except Exception as e:
common_samples.print_exception(e)
if __name__ == "__main__":
main()
print("")
input("Done! Press enter key to exit\n")
Here i want to login once and be in the logged in session forever.
With the below function this is working fine .But here the problem is once the With section is over the session is disconnected.
with ForexConnect() as fx:
try:
fx.login(str_user_id, str_password, str_url,
str_connection, str_session_id, str_pin,
common_samples.session_status_changed)
history = fx.get_history(str_instrument, str_timeframe, date_from, date_to, quotes_count)
current_unit, _ = ForexConnect.parse_timeframe(str_timeframe)
To stay in the session tried the below code without using the "With" and as :
Here the login is successful but could not get the data in history = ForexConnect().get_history
Error Code :
ForexConnect().login(str_user_id, str_password, str_url,
str_connection, str_session_id, str_pin,
common_samples.session_status_changed)
**history = ForexConnect().get_history(str_instrument, str_timeframe, date_from, date_to, quotes_count)**
How to make it the ** ** code work without any exception error without using With --- as : keyworkds.
what is the alternative for this with and as : why when i try to access as history = ForexConnect().get_history ( ... it is giving error how to overcome this issue.

AWS Lambda - How do I convert my code to work in AWS?

I'm struggling to get a Lambda function working. I have a python script to access twitter API, pull information, and export that information into an excel sheet. I'm trying to transfer python script over to AWS/Lambda, and I'm having a lot of trouble.
What I've done so far: Created AWS account, setup S3 to have a bucket, and poked around trying to get things to work.
I think the main area I'm struggling is how to go from a python script that I'm executing via local CLI and transforming that code into lambda-capable code. I'm not sure I understand how the lambda_handler function works, what the event or context arguments actually mean (despite watching a half dozen different tutorial videos), or how to integrate my existing functions into Lambda in the context of the lambda_handler, and I'm just very confused and hoping someone might be able to help me get some clarity!
Code that I'm using to pull twitter data (just a sample):
import time
import datetime
import keys
import pandas as pd
from twython import Twython, TwythonError
import pymysql
def lambda_handler(event, context):
def oauth_authenticate():
twitter_oauth = Twython(keys.APP_KEY, keys.APP_SECRET, oauth_version=2)
ACCESS_TOKEN = twitter_oauth.obtain_access_token()
twitter = Twython(keys.APP_KEY, access_token = ACCESS_TOKEN)
return twitter
def get_username():
"""
Prompts for the screen name of targetted account
"""
username = input("Enter the Twitter screenname you'd like information on. Do not include '#':")
return username
def get_user_followers(username):
"""
Returns data on all accounts following the targetted user.
WARNING: The number of followers can be huge, and the data isn't very valuable
"""
#username = get_username()
#import pdb; pdb.set_trace()
twitter = oauth_authenticate()
datestamp = str(datetime.datetime.now().strftime("%Y-%m-%d"))
target = twitter.lookup_user(screen_name = username)
for y in target:
target_id = y['id_str']
next_cursor = -1
index = 0
followersdata = {}
while next_cursor:
try:
get_followers = twitter.get_followers_list(screen_name = username,
count = 200,
cursor = next_cursor)
for x in get_followers['users']:
followersdata[index] = {}
followersdata[index]['screen_name'] = x['screen_name']
followersdata[index]['id_str'] = x['id_str']
followersdata[index]['name'] = x['name']
followersdata[index]['description'] = x['description']
followersdata[index]['date_checked'] = datestamp
followersdata[index]['targeted_account_id'] = target_id
index = index + 1
next_cursor = get_followers["next_cursor"]
except TwythonError as e:
print(e)
remainder = (float(twitter.get_lastfunction_header(header = 'x-rate-limit-reset')) \
- time.time())+1
print("Rate limit exceeded. Waiting for:", remainder/60, "minutes")
print("Current Time is:", time.strftime("%I:%M:%S"))
del twitter
time.sleep(remainder)
twitter = oauth_authenticate()
continue
followersDF = pd.DataFrame.from_dict(followersdata, orient = "index")
followersDF.to_excel("%s-%s-follower list.xlsx" % (username, datestamp),
index = False, encoding = 'utf-8')

Flask Python: Unable to get parameters using request.args

I want to request some parameters from a external web app. I create an API with flask and query data from MySQL. I able to query the data if I gave a fix input but not when using request.args. I try both request.args['name'] and request.args.get('name') but return the output of Exception path.
Below is my current code. I comment out the fix input I used.
from flask import Flask,jsonify,abort,make_response,request,render_template
import MySQLdb
import MySQLdb.cursors
#app.route('/KLSE/search', methods=['GET'])
def KLSEsearch():
db = MySQLdb.connect(host='vinus.mysql.pythonanywhere-services.com',user='vinus',passwd='Vindex2016',db='vinus$default',cursorclass=MySQLdb.cursors.DictCursor)
curs = db.cursor()
#name ='P'
#macd = 'H'
#volumeMin = '_'
#volumeMax = '_'
#stoch ='H1'
#bollinger ='H'
#rsi ='H1'
#atr ='LV'
#trade = 'HOLD'
#limit = 3
#offSet = 1
name = request.args.get('name')
volumeMin = request.args['volumeMin']
volumeMax = request.args['volumeMax']
macd = request.args['macd']
stoch = request.args['stoch']
bollinger = request.args['bollinger']
rsi = request.args['rsi']
atr = request.args['atr']
trade = request.args['trade']
limit = request.args['limit']
offSet = request.args['offSet']
query0 = "SELECT * FROM KLSE WHERE Stock LIKE '%s' AND"
#query1 = "(Vindex BETWEEN (IF(%s='_',-5000,%s)) AND (IF(%s='_',5000,%s))) AND "
query2 = "(Volume_changes_pc BETWEEN (IF (%s='_',-5000,%s)) AND (IF(%s='_',5000,%s))) AND "
query3 = "MACD LIKE %s AND "
query4 = "STOCH LIKE %s AND "
query5 = "BOLLINGER LIKE %s AND "
query6 = "RSI LIKE %s AND "
query7 = "ATR LIKE %s AND "
query8 = "TRADE LIKE %s LIMIT %s OFFSET %s"
query = query0+query2+query3+query4+query5+query6+query7+query8
input = name+"%",volumeMin,volumeMin,volumeMax,volumeMax,macd,stoch,bollinger,rsi,atr,trade,limit,offSet
try:
curs.execute(query,(input))
g = curs.fetchall()
except Exception:
return 'Error: unable to fetch items'
#return "hihi"
return jsonify({'Stock': g})
The output with fix value as below. I think it shows the query to MySQL is correct.
http://vinus.pythonanywhere.com/KLSE/search1
For the user input value, which use the args,
http://vinus.pythonanywhere.com/KLSE/search?atr=%&bollinger=%&macd=%&name=M&rsi=%&stoch=%&volumeMax=&volumeMin=&trade=HOLD&limit=5&offSet=1
What is the right way, get the parameters? volumeMin,volumeMax,limit and offSet are in float and integers.

You have to serialize your data first.
def serialize():
return {
"id" : g.id,
"volumeMin" : g.name,
"volumeMax" : g.address,
"macd" : g.city,
"stoch" : g.state,
"zipCode" : g.zipCode,
"bollinger" : g.bollinger,
}
#app.route("/KLSE/search/.json")
def stock_json():
query = your.db.query()
return jsonify(Stock=[i.serialize for i in query])