Jupyter - Using psycogp2 in Python and concatenating problems - python

Im trying to concatenate the date and code entity in order to read a database, but I can't. Here is my code. Its comment a line where my code works but im not concating the date and code entity. Please, I want to help me changing my code.
# PAQUETES
import os
import pandas as pd
import psycopg2 as pg2
entidadinput="00022"
fechainput="202202"
tabla="admcrcd.v_rcd_anexo6"
variables="*"
userRCD=os.getenv('JUPYTERHUB_USER')
con = pg2.connect(user=userRCD,
password="post",
host="172.XX.ABC.EF",
port="3456",
database="BDCreditos")
with con:
cur=con.cursor() #vesel
#cur.execute("SELECT " + variables + " from " + tabla + " WHERE fecha='202202'"+" AND cdg_emp='00022'") # This works
cur.execute("SELECT " + variables + " from " + tabla + " WHERE fecha= '" + fechainput +"' AND cdg_emp='" + entidadinput + "'") # This doesnt work
#cur.execute("SELECT * from admcrcd.v_rcd_anexo6 WHERE fecha=202204")
version = cur.fetchone()[0]
print(version)
BD = cur.fetchall()
#for row in rows:
# print(f"{row[0]} {row[1]} {row[2]}")
con.close()

First, the second line should import os. I regenerated both connection strings and the only difference I found is that the string not working has one more space before fecha=, which should not cause the problem normally. But otherwise, the two strings are the same. You can try that (code is as below).
cur.execute("SELECT " + variables + " from " + tabla + " WHERE fecha='" + fechainput +"' AND cdg_emp=" + entidadinput) # This does not work

Related

Write SQL script out of pandas values

I have a pd df that looks like this:
tab_name
col_name
CV_TAB1
TAB1_COL1
CV_TAB1
TAB1_COL2
CV_TAB1
TAB1_COL3
CV_TAB2
TAB2_COL1
CV_TAB2
TAB2_COL2
what I need is a python script that generates as many SQL scripts as are the different tab_names WITHOUT the CV_ prefix (in the example, 2 scripts) that create a view for every table with associated columns WITHOUT the TABX_ prefix. So:
'CREATE WIEV TAB1 AS (
SELECT
TAB1_COL1 AS COL1,
TAB1_COL2 AS COL2,
TAB1_COL3 AS COL3
FROM CV_TAB1);'
And another one for TAB2.
Note that in the FROM clause the table name is WITH the CV_.
I've written the code that creates the script:
script = ""
i = 0
list_of_scripts = list()
for t in list_of_table_names:
if t['name'][:3] == 'CV_':
table_name = t['name'][3:] # Removing CV_ from table name
script += "DROP VIEW IF EXISTS " + str(table_name) + ";\n\n" + "CREATE VIEW "+ str(table_name) + " AS ( " + "\n" + "SELECT\n" # First part of SQL script
for c in list_of_columns_out_of_pd_df:
if c['columnName'][:len(table_name)] == str(table_name): # Check for the TABX_ prefix
column_name = c['columnName'][len(table_name)+1:]
if i == 0:
script += str(c['columnName']) + " AS " + str(column_name) + "\n"
i+=1
else:
script += "," + str(c['columnName']) + " AS " + str(column_name) + "\n"
else:
if i == 0:
script += str(c['columnName']) + " AS " + str(c['columnName']) + "\n"
i+=1
else:
script += "," + str(c['columnName']) + " AS " + str(c['columnName']) + "\n"
i = 0
script += "FROM " + str(t['name']) + ");"
list_of_scripts.append(script)
script = ""
what I am missing are the list_of_table_names and the list_of_columns_out_of_pd_df variables
Thanks!
Here is a way using groupby.
l = []
for tb, cols in df.groupby('tab_name')['col_name']:
# aggregate the cols with the wanted format
_s = ',\n'.join([f'{col} as {col.split("_")[-1]}' for col in cols])
# create the full query
s = f'''CREATE VIEW {tb.split("_")[-1]} AS (\nSELECT\n{_s}\nFROM {tb});'''
l.append(s)
print(l[0])
# CREATE VIEW TAB1 AS (
# SELECT
# TAB1_COL1 as COL1,
# TAB1_COL2 as COL2,
# TAB1_COL3 as COL3
# FROM CV_TAB1);
print(l[1])
# CREATE VIEW TAB2 AS (
# SELECT
# TAB2_COL1 as COL1,
# TAB2_COL2 as COL2
# FROM CV_TAB2);

CX_Oracle CTE execution

I built a relation data model in Oracle and now creating a GUI using Python. I need a SQL statement to execute in my IDE but get a cx_Oracle.DatabaseError: ORA-00936: missing expression error message. This is a CTE that runs fine in TOAD and when I remove the CTE and put in a simple SQl statement it executes fine.
I can build a view in my DB and do a select * from but I don't want to go that way.
I'm new to Python so I'm sure there is a better way to do this.
import cx_Oracle
con = cx_Oracle.connect('Example', 'Example', "Example")
cur = con.cursor()
statement = ("with r1 as (" +
" select " +
" r.PARENT_ITEM_id, " +
" D.SC_ID, " +
" --F.TONS" +
" SUM(ROUND(F.TONS*Pic_Distro*2000*s.stk_lvl_mult)) as Stocking_Lvl" +
" from PIC_DISTRO_TBL D" +
" Left Join Part_Velocity_TBL P on (P.item_ID = D.Item_ID and D.SC_ID = P.SC_ID)" +
" Left Join Forecast_TBL F on (D.Bucket_ID = F.Bucket_ID and D.SC_ID =F.SC_ID)" +
" left join Stock_lvl_tbl S on (S.Velocity_id = P.VELOCITY_ID)" +
" left join item_tbl I on (i.item_ID = D.ITEM_ID)" +
" left join parent_item_tbl R on (r.PARENT_ITEM_id = i.PARENT_ITEM_id)" +
" Where F.MTH = '4'" +
" and F.YEAR = '2017'" +
" and P.Velocity_id in ('A','B','C')" +
" and D.SC_ID in ('01','02')" +
" -- and SUM(ROUND(F.TONS*Pic_Distro*2000*s.stk_lvl_mult)) > 0" +
" Group by " +
" r.PARENT_ITEM_id, D.SC_ID " +
" Order by " +
" D.SC_ID DESC, Stocking_lvl DESC" +
")," +
"R2 as (" +
"select r.Parent_Item_ID, o.SC_ID, " +
"coalesce(sum(avail_wt), 0) as Avail_Wt" +
" from" +
" open_inv_tbl O" +
" left join item_tbl I on (i.item_ID = o.ITEM_ID)" +
" left join parent_item_tbl R on (r.PARENT_ITEM_id = i.PARENT_ITEM_id)" +
" Where r.Parent_item_ID is not null" +
" Group by r.Parent_Item_ID,o.SC_ID)" +
"select " +
" r1.PARENT_ITEM_id, " +
" R1.SC_ID, R1.Stocking_Lvl , " +
" coalesce(R2.Avail_wt, 0 ) as Avail_Wt, " +
" coalesce(R2.Avail_wt/R1.Stocking_Lvl, 0) as Precantage" +
" From R1" +
" left join R2 on (R1.parent_item_id = R2.parent_item_id and R1.Sc_ID = R2.Sc_ID) " +
" Where R1.Stocking_lvl > '0' " +
" Order by SC_id Desc, Stocking_Lvl Desc)" )
cur.arraysize = 2000
cur.execute(statement)
Python offers you multi-line strings, when wrapped in triple quotes.
Try a execute a single string and review your query being correct.
statement = """
with r1 as (
select
r.PARENT_ITEM_id,
D.SC_ID,
--F.TONS
...
"""
cur.execute(statement)

os.path.getsize Invalid syntax error

First thing first, code is list below,
import sys
import os
import latexmake
import mysql.connector
conn = mysql.connector.connect(user='root',password='oilwell',host='localhost',database='sqlpush1')
with conn:
mycursor = conn.cursor()
mycursor=execute("SELECT DATE,oil,gas,oilprice,gasprice,totrev FROM results WHERE DATE BETWEEN '2011-01-01' AND '2027-12-01'")
rows = mycursor.fetchall()
a.write("\\documentclass{standalone}\\usepackage{booktabs}\n\n\\usepackage{siunitx}\r \n\
\r\n\\begin{document}\r\n\\begin{tabular}{ccS[table-format = 5.2]} \\\\ \\toprule\r")
a.write("Date & Oil & Gas & Oil price & Gas price & Total Revenue \\\\ \\midrule \r")
for row in rows:
a = open("testtest.tex", "w")
a.write("" + str(row[0]) + " & " + str(row[1]) + " & " + str(row[2]) + " & " + str(row[3]) + " & " + str(row[4]) + " & " + str(row[5]) + " \\\\ \r")
a.write("\\bottomrule \\end{tabular}\r\\end{document}")
a.close
print os.path.getsize("testtest.tex")
os.system('latexmk.py -q testtest.tex')
mycursor.close()
conn.close()
after run IDLE, an error called "Invalid Syntax" pop up on line "os.path.getsize"
I did import os at very beginning, and I don't know why.
Appreciated your kind help.
Regards,
Cheng
It's not the os.path.getsize that's the problem, it's the print. You're using Python 3 and print is now a function, and therefore needs parentheses.
print(os.path.getsize("testtest.tex"))
Also, it'd be nice (though admittedly, you'd never notice in this script) if you actually called close rather than just referencing it:
a.close()

Python/MySQL - LOAD DATA LOCAL INFILE

I am using the mysql connector for Python and I'm trying to run the following SQL statement via Python (Windows) - It's a .csv file:
sql1 = ('SET GLOBAL local_infile = "ON";')
cursor.execute(sql1)
sql2 = ('LOAD DATA LOCAL INFILE "' + path[1:-1] + '" INTO TABLE mytable COLUMNS TERMINATED BY "," LINES TERMINATED BY "\\r\\n" (COL0, COL1, COL2, COL3, COL4, COL5, COL6) SET COL7 = "'some_data'";')
cursor.execute(sql2)
but when I try to execute I receive the following exception:
1148 (42000): The used command is not allowed with this MySQL version
If I try to execute LOAD DATA LOCAL INFILE on mysql console, everything runs fine.
Load Data Infile is disabled by default with Connector/Python
while creating the connection set LOCAL_FILES client flag like this:
from mysql.connector.constants import ClientFlag
conn = mysql.connector.connect(...., client_flags=[ClientFlag.LOCAL_FILES])
There are a lot of security issues with LOAD DATA, so the server is really picky. Are you logging in to localhost, not the public IP of the server? Often one IP will be granted LOAD DATA, but the other won't.
See the fine manual
You could iterate through each line of the file, inserting each as a row. This would be easy since you already mentioned each column is delineated by , and each row is delineated by newlines.
For example, assuming your table mytable had 8 string columns, (COL0 to COL7):
input_file = open(path[1:-1], 'r')
#Loop through the lines of the input file, inserting each as a row in mytable
for line_of_input_file in input_file:
values_from_file = line_of_input_file.split(',', 1) #get the columns from the line read from the file
if(len(values_from_file) == 7): #ensure that 7 columns accounted for on this line of the file
sql_insert_row = "INSERT INTO mytable VALUES (" + values_from_file[0] + "," + values_from_file[1] + "," + values_from_file[2] + "," + values_from_file[3] + "," + values_from_file[4] + "," + values_from_file[5] + "," + values_from_file[6] + "," + some_data + ");"
cursor.execute(sql_insert_row)
input_file.close()
With the MySQLdb driver:
import MySQLdb
from MySQLdb.constants import CLIENT
then along with other arguments to MySQLdb.connect() , pass client_flag=CLIENT.LOCAL_FILES
Discovered by studying the source, and then trying it out.

Add a column in the middle of a row from a .csv file with python

Hello I have a python script which changes a timestamp column in a .csv file from dot notation to "date time TSQL" notation:
One row looks like this before executing the code:
send,2007.10.04.10.11.11.669,Server,Data,Client,TYPE=STP,Length=329,Cnt=11
after executing the code it looks like this:
send,2007-10-04 10:11:11.669,Server,Data,Client,TYPE=STP,Length=329,Cnt=11
I want to append the same time in the new format after the first time column, that it looks like this:
send,2007-10-04 10:11:11.669,2007-10-04 10:11:11.669,Server,Data,Client,TYPE=STP,Length=329,Cnt=11
Here is the Script:
import csv
cr = csv.reader(open("ActualTrace_01 - short2Times.csv", "rb"))
output = csv.writer(open("GermanygoalInputFormatActualTrace_01 - short.csv", "wb"))
for row in cr:
dateTimeContentsSend = row[1].split(".")
finishSend = dateTimeContentsSend[0] + "-" + dateTimeContentsSend[1] + "-" + dateTimeContentsSend[2] + " " + dateTimeContentsSend[3] + ":"
finishSend+= dateTimeContentsSend[4] + ":" + dateTimeContentsSend[5] + "." + dateTimeContentsSend[6]
row[1] = finishSend
output.writerow(row)
All Threads here were not useful and if you just say row[1] = finishSend + "," + finishSend
it makes it in row[1] with quotes like this
send,"2007-10-04 10:11:11.669,2007-10-04 10:11:11.684",Server,Data,Client,TYPE=STP,Length=329,Cnt=11
Are you after (just after row[1] = finishSend)?
row.insert(2, row[1])

Categories