import csv
import psycopg2
conn = psycopg2.connect(database=" ", user=" ", password=" ", host=" ", port= )
cur = conn.cursor()
with open('21.csv', 'r') as f:
next(f)
cur.copy_from(f, 'temp_questions', sep=',')
conn.commit()
i have try to insert data into my db i got error:
cur.copy_from(f, 'temp_questions', sep=',')
psycopg2.errors.QueryCanceled: COPY from stdin failed: error in .read() call: exceptions.ValueError Mixing iteration and read methods would lose data
CONTEXT: COPY temp_questions, line 1
in my csv file -i have 18 column and
table(database)- id with 18 column
i don't know how to insert data
import csv
db=conn.connect('test.db')
print("connected succesfully")
csv_file="test.csv"
with open(csv_file,'r') as csv_file:
csvreader=csv.reader(csv_file)
fields=next(csvreader)
sql_insert_query='INSERT INTO Test (name,age) VALUES(?,?)'
db.executemany(sql_insert_query, csvreader)
print("inserted")
data=db.execute("SELECT * FROM Test")
for i in data:
print(i)
Read the data from csv file and use executemany to insert an array of elements to the database.
Related
I am a new Python programmer and trying to import a sample CSV file into my Postgres database using python script.
I have CSV file with name abstable1 it has 3 headers:
absid, name, number
I have many such files in a folder
I want to create a table into PostgreSQL with the same name as the CSV file for all.
Here is the code which I tried to just create a table for one file to test:
import psycopg2
import csv
import os
#filePath = 'c:\\Python27\\Scripts\\abstable1.csv'
conn = psycopg2.connect("host= hostnamexx dbname=dbnamexx user= usernamexx password= pwdxx")
print("Connecting to Database")
cur = conn.cursor()
#Uncomment to execute the code below to create a table
cur.execute("""CREATE TABLE abs.abstable1(
absid varchar(10) PRIMARY KEY,
name integer,
number integer
)
""")
#to copy the csv data into created table
with open('abstable1.csv', 'r') as f:
next(f)
cur.copy_from(f, 'abs.abstable1', sep=',')
conn.commit()
conn.close()
This is the error that I am getting:
File "c:\Python27\Scripts\testabs.py", line 26, in <module>
cur.copy_from(f, 'abs.abstable1', sep=',')
psycopg2.errors.QueryCanceled: COPY from stdin failed: error in .read() call: exceptions.ValueError Mixing iteration and read methods would lose data
CONTEXT: COPY abstable1, line 1
Any recommendation or alternate solution to resolve this issue is highly appreciated.
Here's what worked for me by: import glob
This code automatically reads all CSV files in a folder and Creates a table with Same name as of the file.
Although I'm still trying to figure out how to extract specific datatypes according to the data in CSV.
But as far as table creation is concerned, this works like a charm for all CSV files in a folder.
import csv
import psycopg2
import os
import glob
conn = psycopg2.connect("host= hostnamexx dbname=dbnamexx user= usernamexx password=
pwdxx")
print("Connecting to Database")
csvPath = "./TestDataLGA/"
# Loop through each CSV
for filename in glob.glob(csvPath+"*.csv"):
# Create a table name
tablename = filename.replace("./TestDataLGA\\", "").replace(".csv", "")
print tablename
# Open file
fileInput = open(filename, "r")
# Extract first line of file
firstLine = fileInput.readline().strip()
# Split columns into an array [...]
columns = firstLine.split(",")
# Build SQL code to drop table if exists and create table
sqlQueryCreate = 'DROP TABLE IF EXISTS '+ tablename + ";\n"
sqlQueryCreate += 'CREATE TABLE'+ tablename + "("
#some loop or function according to your requiremennt
# Define columns for table
for column in columns:
sqlQueryCreate += column + " VARCHAR(64),\n"
sqlQueryCreate = sqlQueryCreate[:-2]
sqlQueryCreate += ");"
cur = conn.cursor()
cur.execute(sqlQueryCreate)
conn.commit()
cur.close()
i tried your code and works fine
import psycopg2
conn = psycopg2.connect("host= 127.0.0.1 dbname=testdb user=postgres password=postgres")
print("Connecting to Database")
cur = conn.cursor()
'''cur.execute("""CREATE TABLE abstable1(
absid varchar(10) PRIMARY KEY,
name integer,
number integer
)
""")'''
with open('lolo.csv', 'r') as f:
next(f)
cur.copy_from(f, 'abstable1', sep=',', columns=('absid', 'name', 'number'))
conn.commit()
conn.close()
although i had to make some changes for it to work:
i had to name the table abstable1 because using abs.abstable1 postgres assumes that i'm using the schema abs, maybe you created that schema on your database if not check on that, also i'm using python 3.7
i noticed that you are using python 2.7(which i think is no longer supported), this may cause issues, since you say you are learning i would recommend that you use python 3 since it is more used now and you most likely encounter code written on it and you would have to be adapting your code to fit your python 2.7
I post my solution here based on #Rose answer.
I used sqlalchemy, a JSON file as config and glob.
import json
import glob
from sqlalchemy import create_engine, text
def create_tables_from_files(files_folder, engine, config):
try:
for filename in glob.glob(files_folder+"\*csv"):
tablename = filename.replace(files_folder, "").replace('\\', "").replace(".csv", "")
input_file = open(filename, "r")
columns = input_file.readline().strip().split(",")
create_query = 'DROP TABLE IF EXISTS ' + config["staging_schema"] + "." + tablename + "; \n"
create_query +='CREATE TABLE ' + config["staging_schema"] + "." + tablename + " ( "
for column in columns:
create_query += column + " VARCHAR, \n "
create_query = create_query[:-4]
create_query += ");"
engine.execute(text(create_query).execution_options(autocommit=True))
print(tablename + " table created")
except:
print("Error at uploading tables")
I am trying to read a file which contains a list of table_names and I want to execute a simple query:
SELECT *
FROM $TABLE_NAME
from each SQL Server database.
The results of which I need to store in a separate .csv file.
Can you please help how to achieve this?
You have to read data from server and write into csv:
get data from sql:
import pyodbc
import csv
mydb = pyodbc.connect("Driver={SQL Server Native Client 11.0};"
"Server=Server;"
"Database=Database;"
"uid=username;pwd=password")
cursor = mydb.cursor()
sql = """SELECT * FROM $TABLE_NAME"""
cursor.execute(sql)
row = cursor.fetchall()
write data into csv:
with open('test.csv', 'w', newline= '') as f:
a = csv.writer(f, delimiter=',')
a.writerow(["Header 1", "Header 2"]) ## etc
a.writerows(row)
Give this code a try.
import pyodbc
import csv
# SQL Server Connection settings
conn = pyodbc.connect("Driver={SQL Server Native Client 11.0};"
"Server=server;"
"Database=dbName;"
"uid=User;pwd=password"
"Trusted_Connection=yes;")
cursor = conn.cursor()
inputFile= open("absolute_inputfile_path","w+")
outputDataLocation="absolute_outputfile_path"
# Reading inout file line by line, assuming each line is a table name
line = inputFile.readline()
while line:
tableName = line
line = f.readline()
query = "SELECT * FROM " + str(tableName)
# Read query data
cursor.execute(query)
rows = cursor.fetchall()
# Write to File as CSV
fileWriter = open(outputDataLocation + "/" + str(tableName), 'w')
myFile = csv.writer(fileWriter)
myFile.writerows(rows)
fileWriter.close()
inputFile.close()
I'm new to Python, and my task is to import the csv to mysql database. I have this sample values inside my csv file:
SHA1,VSDT,TRX
005c41fc0f8580f51644493fcbaa0d2d468312c3,(WIN32 (EXE 7-2)),Ransom.Win32.TRX.XXPE50FFF027,
006ea7ce2768fa208ec7dfbf948bffda9da09e4e,WIN32 EXE 2-2,TROJ.Win32.TRX.XXPE50FFF027,
My problem here is, how can I remove "( " and ")" only at the start and end point of string at the second column before importing to database?
I have this code to import the csv
import csv
import mysql.connector
file = open(fullPath, 'rb')
csv_data = csv.reader(file)
mycursor = mydb.cursor()
cursor = mydb.cursor()
for row in csv_data:
cursor.execute('INSERT INTO jeremy_table_test(sha1,vsdt,trendx)'
'VALUES(%s, %s, %s)',[(row[0]),(row[1]),(row[2]))
mydb.commit()
cursor.close()
print("Done")
Skip the row when you read it in, rather than when you write it.
with open(fullPath, 'rb') as file:
csv_data = csv.reader(file)
next(csv_data)
mycursor = mydb.cursor()
cursor = mydb.cursor()
for row in csv_data:
cursor.execute('INSERT INTO jeremy_table_test(sha1,vsdt,trendx)'
'VALUES(%s, %s, %s)',[(row[0]),(row[1]),(row[2]))
mydb.commit()
cursor.close()
print("Done")
MySQL LOAD DATA tool can probably do what you want here. Here is what the LOAD DATA call might look like:
LOAD DATA INFILE 'path/to/rb'
INTO TABLE jeremy_table_test
FIELDS TERMINATED BY ','
LINES TERMINATED BY '\r\n' -- or '\n'
IGNORE 1 LINES
(sha1, #var1, trendx)
SET vsdt = TRIM(TRAILING ')' FROM TRIM(LEADING '(' FROM #var1));
To make this call from your Python code, you may try something like this:
query = "LOAD DATA INFILE 'path/to/rb' INTO TABLE jeremy_table_test FIELDS TERMINATED BY ',' LINES TERMINATED BY '\r\n' IGNORE 1 LINES (sha1, #var1, trendx) SET vsdt = TRIM(TRAILING ')' FROM TRIM(LEADING '(' FROM #var1))"
cursor.execute(query)
connection.commit()
I am running SQL query from python API and want to collect data in Structured(column-wise data under their header).CSV format.
This is the code so far I have.
sql = "SELECT id,author From researches WHERE id < 20 "
cursor.execute(sql)
data = cursor.fetchall()
print (data)
with open('metadata.csv', 'w', newline='') as f_handle:
writer = csv.writer(f_handle)
header = ['id', 'author']
writer.writerow(header)
for row in data:
writer.writerow(row)
Now the data is being printed on the console but not getting in .CSV file this is what I am getting as output:
What is that I am missing?
Here is a simple example of what you are trying to do:
import sqlite3 as db
import csv
# Run your query, the result is stored as `data`
with db.connect('vehicles.db') as conn:
cur = conn.cursor()
sql = "SELECT make, style, color, plate FROM vehicle_vehicle"
cur.execute(sql)
data = cur.fetchall()
# Create the csv file
with open('vehicle.csv', 'w', newline='') as f_handle:
writer = csv.writer(f_handle)
# Add the header/column names
header = ['make', 'style', 'color', 'plate']
writer.writerow(header)
# Iterate over `data` and write to the csv file
for row in data:
writer.writerow(row)
import pandas as pd
import numpy as np
from sqlalchemy import create_engine
from urllib.parse import quote_plus
params = quote_plus(r'Driver={SQL Server};Server=server_name; Database=DB_name;Trusted_Connection=yes;')
engine = create_engine("mssql+pyodbc:///?odbc_connect=%s" % params)
sql_string = '''SELECT id,author From researches WHERE id < 20 '''
final_data_fetch = pd.read_sql_query(sql_string, engine)
final_data_fetch.to_csv('file_name.csv')
Hope this helps!
with mysql - export csv with mysqlclient library - utf8
import csv
import MySQLdb as mariadb;
import sys
tablelue="extracted_table"
try:
conn = mariadb.connect(
host="127.0.0.1",
port=3306,
user="me",
password="mypasswd",
database="mydb")
cur = conn.cursor()
instruction="show columns from " + tablelue
cur.execute(instruction)
myresult = cur.fetchall()
work=list()
i=0
for x in myresult:
work.append(x[0])
i=i+1
wsql = "SELECT * FROM " + tablelue
cur.execute(wsql)
wdata = cur.fetchall()
# Create the csv file
fichecrit=tablelue+".csv"
with open(fichecrit, 'w', newline='', encoding="utf8") as f_handle:
writer = csv.writer(f_handle,delimiter=";")
# Add the header/column names
header = work
writer.writerow(header)
# Iterate over `data` and write to the csv file
for row in wdata:
writer.writerow(row)
conn.close()
except Exception as e:
print(f"Error: {e}")
sys.exit(0)
You can dump all results to the csv file without looping:
data = cursor.fetchall()
...
writer.writerows(data)
I am reading very large tables (~3 times my RAM) from SQL Server and writing them as .csv in chunks. But in the .csv files the column names are missing. What am I doing wrong?
My code is as follows:
import pandas as pd
import pyodbc
cnxn = pyodbc.connect('''Driver={SQL Server}; Server=myServer; > Database=myDb''')
cursor = cnxn.cursor()
#Extracting the naems of all the tables in the database
cursor.execute("select * from information_schema.tables")
tables = cursor.fetchall()
cursor.close()
#Reading all the tables and saving them chunk by chunk as csv
counter = 1
for t in tables:
print("Currently working on table Number {0} - {1}".format(counter, t[2]))
cursor = cnxn.cursor()
cursor.execute("Select * from {0}".format(t[2]))
file_name = "{0}.csv".format(t[2])
f = open(file_name, 'w')
# Get data in batches
while True:
# Read the data
df = pd.DataFrame(cursor.fetchmany(1000))
# We are done if there are no data
if len(df) == 0:
break
# Let's write to the file
else:
df.to_csv(f, header=False)
counter += 1
f.close()
cursor.close()
cnxn.close()