I am a bit of a newbie and I'm stuck with this postgres insert step.
My challenge is I am pulling a Dict from a json that is stored in a list and I am trying to pull the values from the dict and save it to a postgres DB.
any help on how to write this up correctly would be appreciated
Here is the connection string for the DB under the page break line is the code used for the db insert.
import psycopg2
'''DATABASE CONNECTION SETTINGS'''
def dbconnect():
"""Function returns settings for db connection."""
dbauth = psycopg2.connect("dbname='databse' user='username' \
host='dbhost' password='password'")
return dbauth
def weatherupdate(dbauth, list):
connection = dbauth
try:
connection
except:
print "I am unable to connect to the database"
conn = connection
cursor = conn.cursor()
l01 = list[0]['state_time_zone']
l02 = list[0]['time_zone']
l03 = list[0]['product_name']
l04 = list[0]['state']
l05 = list[0]['refresh_message']
l06 = list[0]['name']
l11 = list[1]['swell_period']
l12 = list[1]['lat']
l13 = list[1]['lon']
l14 = list[1]['cloud_oktas']
l15 = list[1]['gust_kt']
l16 = list[1]['history_product']
l17 = list[1]['local_date_time']
l18 = list[1]['cloud']
l19 = list[1]['cloud_type']
l110 = list[1]['swell_height']
l111 = list[1]['wmo']
l112 = list[1]['wind_dir']
l113 = list[1]['weather']
l114 = list[1]['wind_spd_kt']
l115 = list[1]['rain_trace']
l116 = list[1]['aifstime_utc']
l117 = list[1]['press_tend']
l118 = list[1]['press']
l119 = list[1]['vis_km']
l120 = list[1]['sea_state']
l121 = list[1]['air_temp']
l122 = list[1]['cloud_base_m']
l123 = list[1]['cloud_type_id']
l124 = list[1]['swell_dir_worded']
l125 = list[1]['sort_order']
query = "INSERT INTO weather (state_time_zone, time_zone, product_name, state, refresh_message, name, swell_period, lat, lon, cloud_oktas, gust_kt, history_product, local_date_time, cloud, cloud_type, swell_height, wmo, wind_dir, weather, wind_spd_kt, rain_trace, aifstime_utc, press_tend, press, vis_km, sea_state, air_temp, cloud_base_m, cloud_type_id, swell_dir_worded, sort_order ) VALUES (l01, l02, l03, l04, l05, l06, l11, l12, l13, l14, l15, l16, l17, l18, l19, l110, l111, l112, l113, l114, l115, l116, l117, l118, l119, l120, l121, l122, l123, l124, l125);"
cursor.execute(query)
conn.commit()
weatherupdate(dbconnect(), getweather())
When i run the code it throws this error:
Traceback (most recent call last):
File "weatherDb.py", line 57, in <module>
weatherupdate(dbconnect(), getweather())
File "weatherDb.py", line 53, in weatherupdate
cursor.execute(query)
psycopg2.ProgrammingError: column "l01" does not exist
LINE 1: ...d_type_id, swell_dir_worded, sort_order ) VALUES (l01, l02, ...
Im sure this is incorrect so any help and direction would be great.
Thanks in advance.
query = """INSERT INTO weather (state_time_zone, time_zone, product_name, [SNIP])
VALUES (%s, %s, %s, [SNIP] ) """
cursor.execute(query, (l01, l02, l03 [SNIP])
Related
I have tried multiple solutions and way around to solve this issue, probably something is still I am missing.
I want to insert a list of values to my database. Here is what I am doing -
import psycopg2
import pandas as pd
Region = [
"Region1",
"Region2",
]
qa = "endpoint1"
def insert_many(data_list):
"""Add data to the table."""
insert_query = """INSERT INTO pbi_forecast(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
conn = None
try:
conn = psycopg2.connect(
database='db',
user='user',
host='host1',
port=5432,
password=pw
)
cur = conn.cursor()
cur.executemany(insert_query, data_list)
conn.commit()
cur.close()
except(Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
# connect prod2
conn_prod2 = psycopg2.connect(
database='db',
user='user',
host='host2',
port=5432,
password=pw
)
cur_prod2 = conn_prod2.cursor()
for re in region:
sql_prod2_select = f"select * from pbi_forecast where \
run_date >= '2022-04-20 00:00:00'\
and run_date <= '2022-04-22 00:00:00'\
and region = '{re}' ;"
cur_prod2.execute(sql_prod2_select)
forecast = pd.DataFrame(cur_prod2.fetchall())
data_list = [list(row) for row in forecast.itertuples(index=False)]
insert_many(data_list)
I am getting integer out of range error when running it. If I restrict the insert record to somewhere 50 records it works but when running it without any limit it throws this error.
Thanks in advance.
I have a code script that get group id and run some wmi commands for this groups.
I have several wmi commands like get ip, get computer manufacturer, get computer username and so on. I want to save this outputs in database. Like after get ip, get database and write it on ip column. And the same things for other parameters like manufacturer and username.
How can i do it? Im so confused because in my for loop I run all the commands in order.
query = f"select wmic_id from group_wmic where group_id = '{group_id}'"
cursor = connection.cursor(dictionary=True)
cursor.execute(query)
wmic_ids = cursor.fetchall()
for row in wmic_ids:
wmic_id = row["wmic_id"]
query = f"select command_line from wmic_commands where id = '{wmic_id}'"
cursor = connection.cursor(dictionary=True)
cursor.execute(query)
command_line = cursor.fetchall()
for data in command_line:
cmd_line = data['command_line']
output = wmi.query(cmd_line)
result = json.dumps(output)
mySql_insert_query = """INSERT INTO wmi_inventory (wmi_data, date)
VALUES (%s, %s) """
now = datetime.now()
dt_string = now.strftime("%Y-%m-%d-%H:%M:%S")
record = (result, dt_string)
cursor.execute(mySql_insert_query, record)
connection.commit()
For being more clear,
I have commands like:
SELECT UserName FROM Win32_ComputerSystem
which gives me the username of computer. or SELECT Manufacturer, Model FROM Win32_ComputerSystem it gives me the manufacturer.
In this part of my code
query = f"select command_line from wmic_commands where id = '{wmic_id}'"
cursor = connection.cursor(dictionary=True)
cursor.execute(query)
command_line = cursor.fetchall()
**for data in command_line:
cmd_line = data['command_line']
output = wmic.query(cmd_line)**
result = json.dumps(output)
mySql_insert_query = """INSERT INTO wmi_inventory (wmi_data, date)
VALUES (%s, %s) """
now = datetime.now()
dt_string = now.strftime("%Y-%m-%d-%H:%M:%S")
record = (result, dt_string)
cursor.execute(mySql_insert_query, record)
connection.commit()
I execute all these queries in my for loop. And save the output as a json data in my database. But i want to save outputs in to the database like when i execute the username commands, save it in to username column.
import sqlite3
import traceback
from time import sleep
import mysql.connector
def check_user(user_id):
conn = mysql.connector.connect(host='localhost', database='online', user='root1', password='rootRRR111_')
cur = conn.cursor()
cur.execute('CREATE TABLE IF NOT EXISTS online(id INT, last_online_date TEXT)')
conn.commit()
select = "SELECT * FROM online WHERE id = %s LIMIT 0, 1"
result = cur.execute(select, (user_id,))
if result is None:
insert = ('INSERT INTO online (id, last_online_date) VALUES (%s, %s)')
cur.reset()
cur.execute(insert, (user_id, online_time))
conn.commit()
def update_online_status(user_id, online_time):
conn = mysql.connector.connect(host='localhost', database='online', user='root1', password='rootRRR111_')
cursor = conn.cursor()
select = 'SELECT last_online_date FROM online WHERE id = %s'
result = cursor.execute(select, (user_id,))
old_online = result
online_time = f'{old_online},{online_time}'
cursor.reset()
cursor.execute('UPDATE online SET last_online_date = %s WHERE id = %s', (online_time, user_id))
conn.commit()
app = Client("my_account")
app.start()
while True:
try:
with open('ids.ini', 'r') as file:
users = file.read().splitlines()
for user in users:
result = app.get_users(user)
user_id = result['id']
if result['status'] == 'offline':
unix_timestamp = float(result['last_online_date'])
local_timezone = tzlocal.get_localzone()
local_time = datetime.fromtimestamp(unix_timestamp, local_timezone)
online_time = local_time.strftime("%Y/%m/%d %H:%M:%S")
elif result['status'] == 'online':
now = datetime.now()
online_time = now.strftime("%Y/%m/%d %H:%M:%S")
check_user(user_id)
update_online_status(user_id, online_time)
# sleep(300)
except Exception:
traceback.print_exc()
continue
app.stop()
I am writing a program that would read the online status of a user in telegram.
Instead of writing online to an existing user, a huge number of identical rows appear in the database.
Example:
Table with repetitions
When I try to fix something, there are a lot of errors.
mysql.connector.errors.programmingerror: not all parameters were used in the sql statement
mysql.connector.errors.internalerror: unread result found
and other...
Pls help!!
My query returns no error but doesn't commit to my database.
import mysql.connector
from datetime import date
def ImportKey():
testsite_array = []
converted_list = []
cnx = mysql.connector.connect(user='USER', password='PASSWORD', host='HOST', database='DATABASE')
cursor = cnx.cursor()
keyType = input("Valid types: Day, Month, Life:\n > ")
if keyType == "Day":
with open('Keys.txt') as my_file:
for line in my_file:
testsite_array.append(line)
for element in testsite_array:
converted_list.append(element.strip())
sql_query = "INSERT INTO `DailyK`(`keyDaily`) VALUES (%s)"
cursor.execute(sql_query, (converted_list[0], ))
cnx.commit()
cursor.close()
cnx.close()
Why does this code not work? It returns no error and from what I can see there is nothing wrong with it. Help would be much appreciated.
I would like to write a function that return a pandas DataFrame via an SQL query.
I have 4 agruments, the last 2 are the 'query' itself and the 'date_of_report'.
This is the query structure:
practice_query = """select * from <TABLE_NAME> b
where b.VALUE_DATE = TO_DATE(:date_of_report,'yyyy-mm-dd') """
My funtion:
import time
import cx_Oracle
import pandas as pd
def my_query(username: str = "USERNAME",
password: str = "PASSWORD",
query: str = "",
date_of_report: str = "") -> pd.DataFrame:
start = time.perf_counter()
# create connection ot EDW tables
dsn = cx_Oracle.makedsn("....", \
1112, \
service_name = "...")
conn = cx_Oracle.connect(username, password, dsn, encoding="UTF-8")
# create cursor
cur = conn.cursor()
cur.execute(query, date_of_report)
col_names = [row[0] for row in cur.description]
mydata = pd.DataFrame(cur.fetchall())
mydata.columns = col_names
end = time.perf_counter() - start
print("Run time: {:.3f}s".format(end))
return mydata
My function call:
mydata = edw_query(query = practice_query, date_of_report = "2020-09-30")
This is my error message:
DatabaseError Traceback (most recent call last)
<ipython-input-22-0ecdc646fe92> in <module>
----> 1 mydata = edw_query(query = practice_query, date_of_report = '"2020-09-30"')
<ipython-input-8-c7875fbe0d2a> in edw_query(username, password, query, date_of_report)
36 cur = conn.cursor()
37
---> 38 cur.execute(query, date_of_report)
39 col_names = [row[0] for row in cur.description]
40 mydata = pd.DataFrame(cur.fetchall())
DatabaseError: ORA-01036: illegal variable name/number
Could you help find the correct syntax? I've tried date ':date_of_report' in the SQL text where date_of_report = "2020-09-30", I've also tried :date_of_report where date_of_report = '"2020-09-30"', but nothing seems to be working.
I've found these but could not figure out a way:
https://cx-oracle.readthedocs.io/en/latest/user_guide/bind.html
https://www.techonthenet.com/oracle/functions/to_date.php
Many thanks:
Roland
Change the execute bind parameter to:
cur.execute(query, [date_of_report])
or
cur.execute(query, date_of_report=date_of_report)
See the cx_Oracle manual section Using Bind Variables. If you have suggestions on improving this doc, let us know.