We Keep Coding

Merge SQL tables into One using Python

I'm trying to merge several SQL tables from a DB where I don't have direct access, but just the ODBC connection open. All my tables contain the same structure and format. I currently have a code that does pretty much the old school thing but my DB tables are huge with millions of records, hence it's affecting my memory and storage. Any better to run this program and manage memory more efficiently.
Currently what I do:
import pandas as pd, shutil, glob
table1= pd.read_sql_query("select * from table1")
table1.to_csv(":/file1.csv")
table2= pd.read_sql_query("select * from table2")
table2.to_csv(":/file2.csv")
table3= pd.read_sql_query("select * from table3")
table3.to_csv(":/file3.csv")
#Merging the files
path = r'\\file.*' #Path were your files are located
allFiles = glob.glob(path + "/*.csv")
allFiles.sort()
with open('C:\\Desktop\\Outuput_file.csv', 'wb') as outfile:
for i, fname in enumerate(allFiles):
with open(fname, 'rb') as infile:
if i != 0:
infile.readline()
shutil.copyfileobj(infile, outfile)
print(fname + " has been imported.")

You can use append mode in pandas to_csv function.
For example:
table1= pd.read_sql_query("select * from table1")
table1.to_csv("C:\\Desktop\\Outuput_file.csv", mode='a')
or
table2= pd.read_sql_query("select * from table2")
table2.to_csv("C:\\Desktop\\Outuput_file.csv", mode='a', header=False)
referance

Python- execute SQL queries from list of tables and store results in seperate file for each table name

I am trying to read a file which contains a list of table_names and I want to execute a simple query:
SELECT *
FROM $TABLE_NAME
from each SQL Server database.
The results of which I need to store in a separate .csv file.
Can you please help how to achieve this?

You have to read data from server and write into csv:
get data from sql:
import pyodbc
import csv
mydb = pyodbc.connect("Driver={SQL Server Native Client 11.0};"
"Server=Server;"
"Database=Database;"
"uid=username;pwd=password")
cursor = mydb.cursor()
sql = """SELECT * FROM $TABLE_NAME"""
cursor.execute(sql)
row = cursor.fetchall()
write data into csv:
with open('test.csv', 'w', newline= '') as f:
a = csv.writer(f, delimiter=',')
a.writerow(["Header 1", "Header 2"]) ## etc
a.writerows(row)

Give this code a try.
import pyodbc
import csv
# SQL Server Connection settings
conn = pyodbc.connect("Driver={SQL Server Native Client 11.0};"
"Server=server;"
"Database=dbName;"
"uid=User;pwd=password"
"Trusted_Connection=yes;")
cursor = conn.cursor()
inputFile= open("absolute_inputfile_path","w+")
outputDataLocation="absolute_outputfile_path"
# Reading inout file line by line, assuming each line is a table name
line = inputFile.readline()
while line:
tableName = line
line = f.readline()
query = "SELECT * FROM " + str(tableName)
# Read query data
cursor.execute(query)
rows = cursor.fetchall()
# Write to File as CSV
fileWriter = open(outputDataLocation + "/" + str(tableName), 'w')
myFile = csv.writer(fileWriter)
myFile.writerows(rows)
fileWriter.close()
inputFile.close()

Python MySQLdb - Export query to csv without line terminators

So basically im using MySQLdb query dialy images of my tables and i want to save them in .csv but one of the fields has line terminators (\n) and i cant figure out how to get rid of them so my csv doesnt break.
Here is the python im using:
results = cur.execute(sql)
db = MySQLdb.connect(host="",
user="",
passwd="",
db="" )
cur = db.cursor()
sql = """" big query here """
results = cur.execute(sql)
with open("out.csv", "wb") as csv_file:
csv_writer = csv.writer(csv_file)
csv_writer.writerow([i[0] for i in cur.description])
csv_writer.writerow(cur)
Is there a easy way to replace \n chars for just spaces?

Try this:
import csv
import sys
csv_writer = csv.writer(sys.stdout, lineterminator='\n')
Or:
with open("out.csv","wb",newline='') as csv_file:

If the newline is in appearing in the text of your column maybe something like this wouldwork.
csv_writer.writerow([i[0].replace('\n',' ') for i in cur.description])

SQL query output to .csv

I am running SQL query from python API and want to collect data in Structured(column-wise data under their header).CSV format.
This is the code so far I have.
sql = "SELECT id,author From researches WHERE id < 20 "
cursor.execute(sql)
data = cursor.fetchall()
print (data)
with open('metadata.csv', 'w', newline='') as f_handle:
writer = csv.writer(f_handle)
header = ['id', 'author']
writer.writerow(header)
for row in data:
writer.writerow(row)
Now the data is being printed on the console but not getting in .CSV file this is what I am getting as output:
What is that I am missing?

Here is a simple example of what you are trying to do:
import sqlite3 as db
import csv
# Run your query, the result is stored as `data`
with db.connect('vehicles.db') as conn:
cur = conn.cursor()
sql = "SELECT make, style, color, plate FROM vehicle_vehicle"
cur.execute(sql)
data = cur.fetchall()
# Create the csv file
with open('vehicle.csv', 'w', newline='') as f_handle:
writer = csv.writer(f_handle)
# Add the header/column names
header = ['make', 'style', 'color', 'plate']
writer.writerow(header)
# Iterate over `data` and write to the csv file
for row in data:
writer.writerow(row)

import pandas as pd
import numpy as np
from sqlalchemy import create_engine
from urllib.parse import quote_plus
params = quote_plus(r'Driver={SQL Server};Server=server_name; Database=DB_name;Trusted_Connection=yes;')
engine = create_engine("mssql+pyodbc:///?odbc_connect=%s" % params)
sql_string = '''SELECT id,author From researches WHERE id < 20 '''
final_data_fetch = pd.read_sql_query(sql_string, engine)
final_data_fetch.to_csv('file_name.csv')
Hope this helps!

with mysql - export csv with mysqlclient library - utf8
import csv
import MySQLdb as mariadb;
import sys
tablelue="extracted_table"
try:
conn = mariadb.connect(
host="127.0.0.1",
port=3306,
user="me",
password="mypasswd",
database="mydb")
cur = conn.cursor()
instruction="show columns from " + tablelue
cur.execute(instruction)
myresult = cur.fetchall()
work=list()
i=0
for x in myresult:
work.append(x[0])
i=i+1
wsql = "SELECT * FROM " + tablelue
cur.execute(wsql)
wdata = cur.fetchall()
# Create the csv file
fichecrit=tablelue+".csv"
with open(fichecrit, 'w', newline='', encoding="utf8") as f_handle:
writer = csv.writer(f_handle,delimiter=";")
# Add the header/column names
header = work
writer.writerow(header)
# Iterate over `data` and write to the csv file
for row in wdata:
writer.writerow(row)
conn.close()
except Exception as e:
print(f"Error: {e}")
sys.exit(0)

You can dump all results to the csv file without looping:
data = cursor.fetchall()
...
writer.writerows(data)

Python csv from database query adding a custom column to csv file

here is what I try to achieve my current code is working fine I get the query to run on my sql server but I will need to gather information from several servers. How would I add a column with the dbserver listed in that column?
import pyodbc
import csv
f = open("dblist.ini")
dbserver,UID,PWD = [ variable[variable.find("=")+1 :] for variable in f.readline().split("~")]
connectstring = "DRIVER={SQL server};SERVER=" + dbserver + ";DATABASE=master;UID="+UID+";PWD="+PWD
cnxn = pyodbc.connect(connectstring)
cursor = cnxn.cursor()
fd = open('mssql1.txt', 'r')
sqlFile = fd.read()
fd.close()
cursor.execute(sqlFile)
with open("out.csv", "wb") as csv_file:
csv_writer = csv.writer(csv_file, delimiter = '!')
csv_writer.writerow([i[0] for i in cursor.description]) # write headers
csv_writer.writerows(cursor)

You could add the extra information in your sql query. For example:
select "dbServerName", * from table;
Your cursor will return with an extra column in front of your real data that has the db Server name. The downside to this method is you're transferring a little more extra data.