Im trying to export MySQL select query results to CSV file.
Im using lambda for this, and my database is in RDS.
I can able to connect and run the queries.
But when I try to save the select query results in CSV format, its not working.
Need python developers help to write this script.
Note: Unicodewritter will not work on lambda.
Lambda function:
import sys
import logging
import rds_config
import pymysql
#rds settings
rds_host = "connection_link"
name = rds_config.db_username
password = rds_config.db_password
db_name = rds_config.db_name
logger = logging.getLogger()
logger.setLevel(logging.INFO)
try:
conn = pymysql.connect(rds_host, user=name, passwd=password, db=db_name, connect_timeout=5)
except:
logger.error("ERROR: Unexpected error: Could not connect to MySql instance.")
sys.exit()
logger.info("SUCCESS: Connection to RDS mysql instance succeeded")
def handler(event, context):
"""
This function fetches content from mysql RDS instance
"""
item_count = 0
with conn.cursor() as cur:
cur.execute("create table Employee3 ( EmpID int NOT NULL, Name varchar(255) NOT NULL, PRIMARY KEY (EmpID))")
cur.execute('insert into Employee3 (EmpID, Name) values(1, "Joe")')
cur.execute('insert into Employee3 (EmpID, Name) values(2, "Bob")')
cur.execute('insert into Employee3 (EmpID, Name) values(3, "Mary")')
conn.commit()
cur.execute("select * from Employee3")
for row in cur:
item_count += 1
logger.info(row)
#print(row)
The below standard method should work to write all rows :
with conn.cursor() as cur:
cur.execute("create table Employee3 ( EmpID int NOT NULL, Name varchar(255) NOT NULL, PRIMARY KEY (EmpID))")
cur.execute('insert into Employee3 (EmpID, Name) values(1, "Joe")')
cur.execute('insert into Employee3 (EmpID, Name) values(2, "Bob")')
cur.execute('insert into Employee3 (EmpID, Name) values(3, "Mary")')
conn.commit()
cur.execute("select * from Employee3")
res = cur.fetchall()
with open('output.csv','w') as fileout:
writer = csv.writer(fileout)
writer.writerows(res)
If you need to process lines row by row, you can use the writerow instead :
cur.execute("select * from Employee3")
with open('output.csv','w') as fileout:
writer = csv.writer(fileout)
for row in cur:
writer.writerow(row)
Related
I working on a project and I want to use one database in two python file
but, when I run every project they created database for self
if you know please tell me how I can use that
import sqlite3
def connect():
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"CREATE TABLE IF NOT EXISTS salary (id INTEGER PRIMARY KEY , name text, age INTEGER , price INTEGER )"
)
conn.commit()
conn.close()
def insert(name, age, price):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"INSERT INTO salary VALUES (NULL ,?,?,?)", (name ,age ,price)
)
conn.commit()
conn.close()
def view():
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"SELECT * FROM salary"
)
rows = cur.fetchall()
conn.close()
return rows
def search(name="", age="", price=""):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"SELECT * FROM salary WHERE name = ? OR age = ? OR price = ?", (name, age, price)
)
rows = cur.fetchall()
conn.close()
return rows
def delete(id):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute("DELETE FROM salary WHERE id=?", (id,))
conn.commit()
conn.close()
def update(id, name, age, price):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"UPDATE salary SET name = ?, age = ?, price = ? WHERE id = ?", (name, age, price, id)
)
conn.commit()
conn.close()
def update_pay_money(name, price):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"UPDATE salary SET price = ? WHERE name = ?", (price, name)
)
conn.commit()
conn.close()
connect()
enter image description here
Giving exact path like /path/to/waiters.db while connecting to your database should solve your problem?
This line should be changed while connecting to database.
conn = sqlite3.connect("/path/to/waiters.db")
as Other mentioned for using a sqllite3 db in multiple files you can use their absolute or relative path, for example if you have 'DBs' & 'section-1' & 'section-2' directories and your python file are in section directories you can access the database file in each section by using somthing like this '"../DBs/waiters.db"' and so on for others... but whatf of you try make multiple tables in a database file in tgat way you don need to have multiple databases and its the standard way,
hope it's help
I am new in mysql, I have a table with two cols tag_id and time_stamp, I use a python connector. I need to insert new tag_id just only if did not insert the same tag_id in last 5 min (or some other duration). How can I do that using python mysql.connector?
create table:
import mysql.connector
from mysql.connector import Error
try:
connection = mysql.connector.connect(host='localhost',
database='test',
user='root',
password='root')
mySql_Create_Table_Query = """CREATE TABLE tags (
Id int(11) NOT NULL AUTO_INCREMENT,
tag_id varchar(250) NOT NULL,
time_stamp Datetime NOT NULL,
PRIMARY KEY (Id)) """
cursor = connection.cursor()
result = cursor.execute(mySql_Create_Table_Query)
print("Laptop Table created successfully ")
except mysql.connector.Error as error:
print("Failed to create table in MySQL: {}".format(error))
finally:
if (connection.is_connected()):
cursor.close()
connection.close()
print("MySQL connection is closed")
Insertion function:
def insertVariblesIntoTable(tag, time_stamp):
try:
connection = mysql.connector.connect(host='localhost',
database='test',
user='root',
password='root')
cursor = connection.cursor()
mySql_insert_query = """INSERT INTO tags(tag_id, time_stamp )
VALUES (%s, %s) """
recordTuple = (tag, time_stamp)
cursor.execute(mySql_insert_query, recordTuple)
connection.commit()
print("Record inserted successfully into tags table")
except mysql.connector.Error as error:
print("Failed to insert into MySQL table {}".format(error))
finally:
if (connection.is_connected()):
cursor.close()
connection.close()
print("MySQL connection is closed")
I'd just do a query of the table like this:
import arrow
check = '''select max(timestamp) from tags where tag_id = {}'''
try:
with conn.cursor() as curs:
curs.execute(check.format(tag_id))
max_time = curs.fetchone()
if max_time <= arrow.utcnow().shift(minutes=-5).format('YYYY-MM-DD HH:mm:ss'):
#run the inserts
else:
pass
I'm writing unit tests to test my environment.
I have created tests such as:
def test_database_file_present_and_readable(self):
self.assertTrue(os.access(path_db_file, os.R_OK))
def test_connect_to_db(self):
conn = sqlite3.connect(path_db_file)
conn.close()
def test_create_table(self):
conn = sqlite3.connect(path_db_file)
cur = conn.cursor()
cur.execute("CREATE TABLE test_table (id integer PRIMARY KEY, name text)")
conn.commit()
conn.close()
def test_insert_into_table(self):
conn = sqlite3.connect(path_db_file)
cur = conn.cursor()
cur.execute("insert into test_table (name) values (?)", ["Test value"])
conn.commit()
conn.close()
def test_update_table(self):
conn = sqlite3.connect(path_db_file)
cur = conn.cursor()
cur.execute("update test_table set id = 2 where id = ?", [1])
conn.commit()
conn.close()
def test_delete_from_table(self):
conn = sqlite3.connect(path_db_file)
cur = conn.cursor()
cur.execute("delete from test_table where id = ?", [2])
conn.commit()
conn.close()
def test_if_test_table_is_empty(self):
conn = sqlite3.connect(path_db_file)
cur = conn.cursor()
result = cur.execute("select exists(select 1 from test_table)").fetchall()
conn.commit()
conn.close()
self.assertTrue(result == 1)
def test_delete_table(self):
conn = sqlite3.connect(path_db_file)
cur = conn.cursor()
cur.execute("drop table test_table")
conn.commit()
conn.close()
And during program execution order of tests is unknown - how to set the order or how to clean up database after creating tests with table creation?
You can get pointers about test method execution order here: Python unittest.TestCase execution order
One suggestion - if you are going for such testing, it's better to mock external dependencies like sqlite & test only the code you've written.
So I have a database containing the products I will be holding, within this database I have a basket table where users can add items they would like to buy. For some reason, I am unable to take a selection within the view window and copy that data into my basket table.
Here is the function I have created for moving the data.
def Move():
if not tree.selection():
error = tkMessageBox.showerror("Error", "Cannot move nothing to basket")
else:
result = tkMessageBox.askquestion('CCOS', 'Do you want to add this to the basket?', icon="warning")
if result == 'yes':
curItem = tree.selection()
print(curItem)
contents = (tree.item(curItem))
selecteditem = contents['values']
Database()
cursor.execute("INSERT INTO `basket` (product_name, product_qty, product_price) VALUES(?, ?, ?)",
(str(PRODUCT_NAME.get()), int(PRODUCT_QTY.get()), int(PRODUCT_PRICE.get())))
conn.commit()
PRODUCT_ID.set("")
PRODUCT_NAME.set("")
PRODUCT_PRICE.set("")
PRODUCT_QTY.set("")
cursor.close()
conn.close()
Comment:
I have a range of different functions in my code, Database() is its own function that creates the database and cursor = conn.connect(). I am not getting an error, but when i run the function, no data is copied into the table.
When I call print(curItem), all that is outputted is : ('I002',)
Here is the database function:
def Database():
global conn, cursor
conn = sqlite3.connect("main_storage.db")
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS `admin` (admin_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, username TEXT, password TEXT)")
cursor.execute("CREATE TABLE IF NOT EXISTS `product` (product_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, product_name TEXT, product_qty TEXT, product_price TEXT)")
cursor.execute("CREATE TABLE IF NOT EXISTS `basket` (product_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, product_name TEXT, product_qty TEXT, product_price TEXT)")
cursor.execute("SELECT * FROM `admin` WHERE `username` = 'admin' AND `password` = 'admin'")
if cursor.fetchone() is None:
cursor.execute("INSERT INTO `admin` (username, password) VALUES('admin', 'admin')")
conn.commit()
I am having trouble inserting a record into a MySQL database from python. This is what I am doing.
def testMain2():
conn = MySQLdb.connect(charset='utf8', host="localhost", user="root", passwd="root", db="epf")
cursor = conn.cursor()
tableName = "test_table"
columnsDef = "(export_date BIGINT, storefront_id INT, genre_id INT, album_id INT, album_rank INT)"
exStr = """CREATE TABLE %s %s""" % (tableName, columnsDef)
cursor.execute(exStr)
#Escape the record
values = ["1305104402172", "12", "34", "56", "78"]
values = [conn.literal(aField) for aField in values]
stringList = "(%s)" % (", ".join(values))
columns = "(export_date, storefront_id, genre_id, album_id, album_rank)"
insertStmt = """INSERT INTO %s %s VALUES %s""" % (tableName, columns, stringList)
cursor.execute(insertStmt)
cursor.close()
conn.close()
The table is created however nothing is in the table. I can run the INSERT statement successfully in Terminal with the same credentials.
Any suggestions on what I may be doing wrong?
You haven't committed the transaction.
conn.commit()
(The MySQLdb library sets autocommit to False when connecting to MySQL. This means that you need to manually call commit or your changes will never make it into the database.)