We Keep Coding

Python flask WTForms populating a selectfield using mysql

Im trying to populate my SelectFormField 'College' and 'Courses' using data from my database. I want the options in the courses to change based on the selected college (e.g. College = College of Engineering and Technology, Courses = BS mechanical eng., BS Civil eng. and etc...
this is my attempt
class StudentForm(FlaskForm):
image = FileField('Choose an Image', validators=[FileAllowed(['jpg', 'jpeg', 'png'], 'Images only!'),])
IDN = StringField('ID',[validators.DataRequired(), validators.Length(min=9, max=9)])
Fname = StringField('First Name', [validators.DataRequired()])
Mname = StringField('Middle Name', [validators.DataRequired()])
Lname = StringField('Last Name', [validators.DataRequired()])
Gender = SelectField('Gender', choices=gen, validators=[Optional()])
Year_Level = SelectField('Year Level', choices=year_level, validators=[DataRequired()])
College = SelectField('College',choices =[], validate_choice=False)
Courses = SelectField('Courses',choices =[], validate_choice=False)
Submit = SubmitField('ADD')
def __init__(self):
super(StudentForm, self).__init__()
self.College.choices = self.get_colleges()
self.Courses.choices = self.get_courses()
def get_colleges(self):
conn = mysql.connection
curs = conn.cursor()
sql = "SELECT id FROM college"
curs.execute(sql)
result = curs.fetchall()
conn.commit()
curs.close()
colleges = [("", "")] + [(str(r[0]), r[0]) for r in result]
print(colleges)
return colleges
def get_courses(self):
selected_college = self.College.data
print(selected_college)
if selected_college:
conn = mysql.connection
curs = conn.cursor()
sql = "SELECT courses.id FROM courses JOIN college ON courses.college_id = college.id WHERE college.id = %s"
curs.execute(sql, (selected_college,))
result = curs.fetchall()
conn.commit()
curs.close()
courses = [(str(r[0]), r[0]) for r in result]
return courses
else:
return []
The problem is that the selected college in the get_courses is none and I don't know any other way to pass the currently selected college to the get_courses. Any help is appreciated.

Python3- what's the proper way to connect to database across different methods?

Need help please.
I want to create 2 additional functions that will insert into the database each table instead of in one function as below in def insert_to_db(self).
Is the correct way to create 2 additional functions or to leave it as it is or any other way?
If it's to create another function how do I pass the db and cursor to the new functions?
main.py
class ConfigMain:
def __init__(self, filepath):
env_params = get_env_params()
self.db = connect_to_db(user=env_params.user_name,
password=env_params.password,
host=env_params.host,
port=env_params.port,
database=env_params.database)
self.filepath = filepath
ConfigMain.py
from main import ConfigMain
import openpyxl
import mariadb
import logging
class DBConfig(ConfigMain):
def insert_to_db(self):
conn = self.db[0]
cursor = self.db[1]
wb = openpyxl.load_workbook(filename=self.filepath, data_only=True)
s = 0
file_sheets = wb.sheetnames
for s in range(len(file_sheets)):
# order_invoice Sheet
if file_sheets[s] == 'Sheet1':
# table = 'order_invoice'
num_rows_inserted = 0
num_rows_inserted_item = 0
try:
wb.active = s
sheet = wb.active
row_count = sheet.max_row
column_count = sheet.max_column
for r in range(2, row_count + 1):
order_invoice_id = order_invoice_id.replace(" ", "")
billing_cycle_id = str(billing_year) + str(billing_month)
# Assign values from each row
values = (order_invoice_id, billing_cycle_id)
table = 'order_invoice'
query = """INSERT INTO """ + table + """(order_invoice_id, billing_cycle_id) VALUES (%s, %s) """
# Execute sql Query
cursor.execute(query, values)
num_rows_inserted = num_rows_inserted + cursor.rowcount
table = 'order_item'
# transformations
order_invoice_id = order_invoice_id.replace(" ", "")
order_invoice_item_id = str(order_invoice_id) + str(order_invoice_item_id)
# # Assign values from each row
values = ( order_invoice_item_id, order_invoice_id)
query = """INSERT INTO """ + table + """(order_invoice_item_id,
order_invoice_id) VALUES (%s,%s ) """
# Execute sql Query
cursor.execute(query, values)
except mariadb.IntegrityError as e:
print(f"{e} - for table '{table}' ")
raise
except mariadb.DataError as e:
print(f"{e} - for table '{table}' ")
raise
Thank you!

How to loop through queries and add the results to a single dataframe?

I have a Flask application that currently makes a connection to a remote server and then a particular DB. I have a function that takes the users date range input and then queries data based upon the given dates.
from flask import make_response, redirect, render_template, session, url_for, Blueprint,request
import pandas as pd
import pymssql
from core.database import ConnectionFactory
from core.query import *
dat_a = Blueprint('dat_a', __name__)
ButtonPressed = 0
#dat_a.route('/download', methods=['POST','GET'])
def download():
if session.get('logged_in') != True:
return redirect(url_for('views.index'))
else:
if request.method == 'POST':
return render_template('data.html')
else:
Start = request.args.get('Start Date')
End = request.args.get('End Date')
connection = pymssql.connect(ConnectionFactory.SQL_HOST, ConnectionFactory.SQL_USER_xxx, ConnectionFactory.SQL_PW_xxx,ConnectionFactory.SQL_DB_xxx)
cur=connection.cursor()
query_1 = """
select some column from table
from xx
where date BETWEEN (%s) AND (%s)
"""
query_2 = """
select some column from table
"""
results = []
q_list = [query_1,query_2]
for query in q_list:
cur.execute(query,(Start,End))
results.append(cur)
print(results)
columns = [d[0] for d in cur.description]
data = pd.DataFrame(cur.fetchall(),columns=columns)
print(data)
resp = make_response(data.to_csv())
resp.headers["Content-Disposition"] = "attachment; filename=Data.csv"
resp.headers["Content-Type"] = "text/csv"
return resp
The problem I'm having is getting my for loop to store the data from query_1 into a list and then move it to my dataframe. Currently data from query_2 will be stored into the list and then pushed to the dataframe, but that's inaccurate as it's just fetching random data at that point.
results = []
q_list = [query_1,query_2]
for query in q_list:
cur.execute(query,(Start,End))
results.append(cur)
print(results)
columns = [d[0] for d in cur.description]
data = pd.DataFrame(cur.fetchall(),columns=columns)
I have tried a nested for loop that calls upon each query separately, has it's own connection, and dataframe but that didn't change the results either. Is there a more efficient way to go about this with pandas?

You might be looking for something like this...
cur = connection.cursor()
results = []
for query, args in [
("select some_column from xx where date BETWEEN (%s) AND (%s)", (Start, End)),
("select some_column from yy", ()),
]:
cur.execute(query, args)
results.extend(cur.fetchall())
# Will be overwritten on each iteration, but it's fine as long as the columns are the same for each query
columns = [d[0] for d in cur.description]
data = pd.DataFrame(results, columns=columns)

Insert List valuesinto SQlite3 database with Python error

ctime = []
name = []
minprice = []
maxprice = []
stock = []
historical_sold = []
sold = []
option_name = []
option_stock = []
option_price = []
#Connect to SQL database
conn = sqlite3.connect('etracker.db')
#Create cursor to work with database
c = conn.cursor()
c.execute('''Create TABLE if not exists server("prices")''')
with open('eurls.csv', 'r') as f:
csv_reader = csv.reader(f)
for row in csv_reader:
asins.append(row[0])
asin = asin
date = datetime.datetime.today().strftime("%d/%m/%Y %H:%M:%S")
url = f"https://e.{tld}/api/item/get"
querystring = {"itemid":f"{itemid}","shopid":f"{shopid}"}
payload = ""
headers = {
"cookie": "SPC_SI=9egeYgAAAABlNUJsazZUbPQ60gAAAAAAeVpFRmJWb00%3D; REC_T_ID=c56cc396-9f13-11ec-b054-2cea7fad64d2;",
}
response = requests.request("GET", url, data=payload, headers=headers, params=querystring)
# json object
result_json = response.json()
# starting point
result = result_json['data']
# These are the results from the API based on starting point ['data']
name = result['name']
minprice = result['price_min']/100000
maxprice = result['price_max']/100000
stock = result['normal_stock']
historical_sold = result['historical_sold']
sold = result['sold']
# starting point
option_items = result_json['data']['models']
for option_item in option_items:
# option name
try:
option_name.append(option_item['name'])
except:
option_name.append('')
# option stock
try:
option_stock.append(option_item['normal_stock'])
except:
option_stock.append('')
# option price
try:
option_price.append(option_item['price']/100000)
except:
option_price.append('')
print(option_name, option_stock, option_price)
print(date, name, minprice, maxprice, stock, historical_sold, sold, asin)
c.execute('''INSERT INTO prices VALUES(?,?,?,?,?,?,?,?,?,?,?)''', (date, name, minprice, maxprice, stock, historical_sold, sold, asin))
print(f'Added data for {name}, {minprice}')
#Insert links into table
def data_entry():
for item in option_name:
c.execute("INSERT INTO server(prices) VALUES(?)", (option_name))
#conn.commit()
data_entry() # ==> call the function
conn.commit()
I am getting an error and unable to add those to the SQL database as some products may have 3 options some may have 20 options
How do I handle that? I read that I need to loop through the list and insert it but when I tried it I get this error
sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement uses 11, and there are 8 supplied.
Thank you

How to use threads for huggingface transformers

I'm trying to run a hugging face model, mode exactly "cardiffnlp/twitter-roberta-base-sentiment" on threads. But at the same time, I want just one single instance of it because it's really costly in terms of time.
In other words, I have multiple CSV files (several thousand) and each of them has around 20k-30k lines and I want that each line from all of them to be executed by the huggingface model, as you probably can imagine already this is the reason why I don't want to instantiate a model for each thread (where each thread would be used just to read one line and write it in the database).
The problem with my approach is that when I'm running the code is going to give me an error from huggingface model.
RuntimeError: Already borrowed
Could any of you help me to understand how cand I fix it?
Hugging face model:
class EmotionDetection(object):
def __init__(self, model_name="cardiffnlp/twitter-roberta-base-sentiment"):
self.model_name = model_name
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)
self.classifier = TextClassificationPipeline(model=model, tokenizer=tokenizer, return_all_scores=True,
task="sentiment-analysis", device=0)
def get_emotion_by_label(self, label: str):
if label == "LABEL_0":
return "negative"
elif label == "LABEL_1":
return "neutral"
elif label == "LABEL_2":
return "positive"
else:
print("SOMETHING IS WRONG")
return ""
def get_emotion(self, phrase):
results = self.classifier(phrase)
res = dict()
for result in results:
for emotion in result:
res.update({self.get_emotion_by_label(emotion['label']): emotion['score']})
return res
My code for generating database:
class GenerateDbThread(object):
def __init__(self, text: str, created_at: datetime.datetime, get_emotion_function, cursor, table_name):
self.table_name = table_name
self.text = text
self.created_at = created_at
emotions = get_emotion_function(self.text)
self.pos = emotions['positive']
self.neg = emotions['negative']
self.neu = emotions['neutral']
self.cursor = cursor
def execute(self):
query = f"INSERT INTO {self.table_name}(date, positive, negative, neutral, tweet) " \
f"VALUES (datetime('{str(self.created_at)}'),{self.pos},{self.neg},{self.neu}, '{self.text}')"
self.cursor.execute(query)
self.cursor.commit()
def get_all_data_files_path(data_dir: str):
return [f for f in os.listdir(data_dir) if os.path.isfile(os.path.join(data_dir, f))]
def run(file: str, table_name: str):
df = pd.read_csv(os.path.join('data', file), delimiter=',')
for index, row in df.iterrows():
text = row['tweet']
language = row['language']
split_data = row['created_at'].split(" ")
GTB_Time = f"{split_data[2]} {split_data[3]} {split_data[4]}"
created_at = datetime.datetime.strptime(row['created_at'], f"%Y-%m-%d %H:%M:%S {GTB_Time}")
if language == "en":
GenerateDbThread(text, created_at, emotion_detector.get_emotion, cursor, table_name)
def init_db(db_name, table_name):
conn = sqlite3.connect(db_name)
cursor = conn.cursor()
cursor.execute(f"""
CREATE TABLE IF NOT EXISTS {table_name} (
uid INTEGER PRIMARY KEY AUTOINCREMENT,
date DATETIME NOT NULL,
positive REAL NOT NULL,
negative REAL NOT NULL,
neutral REAL NOT NULL,
text TEXT NOT NULL
)""")
cursor.execute(f"CREATE INDEX IF NOT EXISTS ix_tweets_index ON {table_name}(uid)")
cursor.close()
ex = ThreadPoolExecutor(max_workers=10)
files = get_all_data_files_path('data')
init_db("DB_NAME.db", "TABLE_NAME")
emotion_detector = EmotionDetection()
conn = sqlite3.connect("DB_NAME.db")
cursor = conn.cursor()
pbar = tqdm(total=len(files))
futures = [ex.submit(run, file, "TABLE_NAME") for file in files]
for future in futures:
res = future.result()
pbar.update(1)
pbar.close()