I have a class which contains functions that connect to snowflake (using snowflake connector) and perform data cleaning. My thought was to create multiple functions to do separate data cleaning.
I've defined one variable in a function called "work_data" called "self.calls" and I've defined that function as a pandas dataframe and I'd like to access that dataframe in another function.
Here's my current process:
I first initialized the variable in the init function and made it a blank list for now. After I've run the "work_data" function and try and run the "finalized_data" function..but I get the blank list instead of the panadas data frame.
Here's my code so far:
class SnowReader:
sql_pslink ='select * from xyz in xyz";'
sql_account= 'select * from xyz in xyz;'
sql_call = 'select * from xyz in xyz;'
sql_sales = 'select * from xyz in xyz;'
sql_address = 'select * from xyz in xyz;'
def __init__(self) -> None:
self.database = "xyz"
self.username = INTEGRATION_USER
self.password = SNOW_PASSWORD
self.account = "xyz"
self.warehouse = "xyz"
self.role = "xyz"
self.schema = "xyz"
self.conn = self._connect_snow()
self.calls = []
def _connect_snow(self):
try:
self.conn = snowflake.connector.connect(
user=self.username,
password=self.password,
account=self.account,
warehouse=self.warehouse,
database=self.database,
role=self.role,
schema=self.schema,
)
logger.info("You are connected to Snowflake")
except Exception as ex:
if ex.errno == 250001:
logger.error(
f"Invalid username/password, please re-enter username and password.."
)
def work_data(self):
if not self.conn:
self._connect_snow()
link = pd.read_sql(SnowReader.sql_pslink, self.conn)
account = pd.read_sql(SnowReader.sql_account, self.conn)
self.calls = pd.read_sql(SnowReader.sql_call, self.conn)
sales = pd.read_sql(SnowReader.sql_sales, self.conn)
Calls_merged = self.calls.groupby('xyz', as_index=False)['xyz'].count()
Account_step1 = Calls_merged.merge(account,left_on='xyz', right_on='xyz', how="left" )
Sales_merged = sales.groupby(['xyz'], as_index=False)['xyz'].count()
Account_final = Sales_merged.merge(Account_step1,left_on='xyz', right_on='xyz', how="left" )
Master_Address = pd.read_sql(SnowReader.sql_address, self.conn)
return Account_final, Master_Address, link, self.calls
def finalize_data(self):
return self.calls
a, b, c, d= SnowReader().work_data()
display(a,b,c, d)
testt = SnowReader().finalize_data()
testt
Related
So I have a class in my python file that I want to seperate from the main python file because I will need to modify it a couple time and use it in the main one. However since the class needs access to the kivy file to read the ids I cannot seperate this class from the main one without loading the kv file twice which messes everything up. How can I basically make this class importable?
class RightSide(BoxLayout):
order_size = []
overlay = StringProperty('Current Order will be displayed here')
tabname = parameter_length
iterator = 0
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.orientation = 'vertical'
for i in range(len(menu)):
self.add_widget(Skeleton())
def tally(self):
global OID
order_size = self.order_size
print(OID)
OID = OID + 1
print(OID)
rightside = kv.ids.one.two.three.newlay.rightside
for skel in rightside.children:
if isinstance(skel, Skeleton):
if self.iterator > len(menu)-1:
self.iterator = 0
order_size.clear()
self.tabname[self.iterator] = skel.drink_name, skel.ids.label_id.text
self.iterator = self.iterator + 1
order_size.insert(0, skel.ids.label_id.text)
order_size.insert(0, OID)
self.tabname.reverse()
self.tabname.insert(0, OID)
makeitastring = ''.join(map(str, self.tabname))
self.overlay = makeitastring
con = sql.connect(host='xxxxxx', user='root', password='xxxxxxx.', database='new_schema')
cur = con.cursor()
cur.execute(query, order_size)
con.commit()
con.close()
def clean(self):
rightside = kv.ids.one.two.three.newlay.rightside
for skel in rightside.children:
if isinstance(skel, Skeleton):
skel.ids.label_id.text = '0'
skel.count = 0
kv = Builder.load_file("b.kv")
So the solution was to declare the kv variable in a seperate py file using "import new_file.py" and accessing it via "new_file.kv"
I want to create a class with methods, so that I do not have to create multiple functions.
Below is my code, I want to get a class with two methods. Method 1: SQL Query, method 2: sql insert.
Any tipp is greatly appreciated.
Stefan
def dbconnect():
dbconn = pymysql.connect(host='192.168.1.2', port=3307, user='username', passwd='password', db='dbname')
try:
cur = dbconn.cursor()
sqlQuery = "select * from data"
sqlQuerygetlast = "SELECT * FROM data ORDER BY id DESC LIMIT 1"
sqlQuerygetlast10 = "SELECT * FROM data ORDER BY id DESC LIMIT 10"
cur.execute(sqlQuerygetlast10)
rows = cur.fetchall()
for row in rows:
print(row)
except Exception as e:
print("Exeception occured:{}".format(e))
finally:
#dbconn.commit()
dbconn.close()
My objective is to call the methods from my code, i.e. query a select statement.
Thanks a lot
Stefan
I guess you mean that you don't want to create multiple connections?
Then you should implement it as a Context manager:
class DB:
def __init__(self):
self.dbconn = None
def get_last(self, n)
try:
cur = self.dbconn.cursor()
sqlQuerygetlast = "SELECT * FROM data ORDER BY id DESC LIMIT {}".format(n)
cur.execute(sqlQuerygetlast)
rows = cur.fetchall()
for row in rows:
print(row)
except Exception as e:
print("Exeception occured:{}".format(e))
finally:
# self.dbconn.commit()
def some_other_method(self):
self.dbconn.do_something()
def __enter__(self):
self.dbconn = pymysql.connect(
host='192.168.1.2',
port=3307,
user='username',
passwd='password',
db='dbname'
)
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
dbconn.close()
return True
and use it as follows:
with DB() as db:
db.get_last(1)
db.get_last(10)
db.some_other_method()
This will create only one instance of a database connection and close after it is finished.
Writing a class in Python is fairly simple.
Here's an example of how to write the classic Person class and how to use properties and methods.
Specifically, the presentation method is also making use of the name property.
From this example you can move on and build your Database class implementation quite easily:
class Person:
def __init__(self, name, age):
self.name = name
self.age = age
def presentation(self):
print("Hello my name is " + self.name)
p1 = Person("John", 36)
p1.presentation()
Remember that in all the methods of a class you will have to specify the reserved keyword self as parameter (plus, clearly, any other parameter you might need).
I am trying to SELECT the data in the column base on the given arg.
but an error appear Incorrect number of bindings supplied
class TheClass():
def __init__(self):
self.con = connect('mydb.db')
self.cur = self.con.cursor()
def getonedata(self, text):
self.variable = text
self.cur.execute("""SELECT username, age, password FROM usertbl WHERE username =?""",(self.variable))
self.result = self.cur.fetchone()
return self.result
def getoneresult(self, text):
for row in self.getonedata(text):
self.row1 = row[0]
self.row2 = row[1]
self.row3 = row[2]
print("{} {} {}".format(self.row1, self.row2, self.row3))
app = TheClass()
app.getonedata('IgiveUP')
You have to supply parameters as a tuple to the execute method, and in Python, a single-item tuple has to be specified with an additional comma:
self.cur.execute("""SELECT username, age, password FROM usertbl WHERE username =?""",(self.variable,))
I have a script that can run a number of different reports, based on what input it receives via the command line. All reports read from a database and return the results as a Pandas Dataframe object.
Here is the super-class, (omitting a large number of property getter and setter functions):
import mysql.connector
import pandas as p
import config
class Report(object):
_connection = None
_cursor = None
def __init__(self):
self._user = config.user
self._password = config.password
self._host = config.host
self._database = config.database
self._port = config.port
self._body_text = "Hello,\n\nPlease find attached these reports:\n\n"
self._connection = mysql.connector.connect(user=self._user, password=self._password, db=self._database,
host=self._host, port=self._port)
self._cursor = self._connection.cursor()
#property
def user(self):
return self._user
#user.setter
def user(self, value):
self._user = value
. . .
#property
def body_text(self):
return self._body_text
#body_text.setter
def body_text(self, value):
self._body_text = value
def append_body_text(self, value):
self._body_text += value
def get_data(self, server_cursor, query, columns):
server_cursor.execute(self, query)
results = server_cursor.fetchall()
data = p.DataFrame(data=results, columns=[columns])
return data
def get_today(self):
return self.today
def close(self):
self._connection_web.close()
self._connection_raw.close()
#staticmethod
def insert_variables_into_sql_statement(query, external_data):
final_query = query % external_data
return final_query
#staticmethod
def create_string_from_column(serial):
created_list = serial.tolist()
string = ', '.join(map(str, created_list))
return string
#staticmethod
def write_to_csv(data_frame, file_name):
data_frame.to_csv(config.baseDirectory + file_name, sep=',', index=False)
def generate_report(self):
data = self.get_data(self._cursor_web, self._query, self._columns)
self.write_to_csv(data, self._filename)
self.close()
Here is how my subclasses are structured:
class ExampleReport(Report):
def __init__(self):
Report.__init__(self)
self._query = """
SELECT
u.first_name AS 'First Name',
u.last_name AS 'Last Name'
FROM users AS u
"""
self._columns = "'FirstName', 'LastName'"
self._filename = "example_report.csv"
self.append_body_text("* All users")
In my main method I call the method like this:
report = Reports.ExampleReport()
report.generate_report()
When I do this, I get the following error:
AttributeError: 'ExampleReport' object has no attribute 'encode'
My database connections worked without a problem when it was terribly constructed procedural code (a working version is currently in production). It has broken now that I've tried to make it object oriented. Does anyone have any idea what I've done wrong?
EDIT: SOLVED MY OWN PROBLEM! In the get_data function in the super-class, the second line contained an erroneous self argument passed into server_cursor.execute(query) line. Once it was taken out, the error goes away.
SOLVED MY OWN PROBLEM! In the get_data function in the super-class, the second line contained an erroneous self argument passed into server_cursor.execute(query) line. Once it was taken out, the error goes away.
I'm new to Python. I've just created a sqlite3 db wrapper class to handle the database interaction in my application. My question is how do I return success/failure messages from my db wrapper class methods, addRow, updRow (which add and update rows) to the invoking program?
Here's the class code I cobbled together so far:-
class dbManager(object):
def __init__(self, db):
self.conn = lite.connect(db)
self.conn.execute('pragma foreign_keys = on')
self.conn.execute('pragma synchronous=off')
self.conn.commit()
self.cur = self.conn.cursor()
def query(self, arg):
self.cur.execute(arg)
self.conn.commit()
return self.cur
def addRow(self, tablename, data):
"""
Insert data into a table. The data does not have to be escaped.
"""
global actInserts
# Create a new cursor
# tc = self.conn.cursor() # opened at init
tablelist = ""
valueholder = ""
valuelist = []
for key, value in data.items():
if len(tablelist) > 0:
tablelist += ', '
valueholder += ', '
# Add to table column list
tablelist += key
# Add a holder
valueholder += '?'
# build the insert values
valuelist.append(value)
# Perform and commit the insert
try:
dbResponse = self.cur.execute("INSERT INTO " + tablename + " (" + tablelist + ") VALUES (" + valueholder + ");", valuelist)
actInserts += 1
except lite.Error, e:
dbResponse = 'Sqlite Error NP: ' + e.args[0]
print 'Sqlite Error NP: ' + e.args[0]
return dbResponse
def closeConnection (self):
self.conn.commit()
self.conn.close()
def __del__(self):
self.conn.close()
I think there are 2 ways you can do this. You can:
Change the return type of you function to something like a tuple of (error_code, SQL_results), or
Throw an exception if the query fails (or don't catch the one you already are handling) and leave the exception handling logic to the user (calling program).
I think option 2 is the better way.
Also, I think your query building would be more clear (or at least more concise) using something like the following (assumes that data is a dictionary)
tablelist = ','.join(list(data.viewkeys()))
valueholder = ','.join(['?' for i in data.viewkeys()])
valuelist = list(data.viewvalues())