Related
i wanted to insert some data into my sql but having trouble because there is alot of columns so i would have to write alot of parameters after VALUE. I have a table with all attributes from the json file and a player_id which i add myself
#Gamelogs for players and Teams
import requests
import json
import psycopg2
# Connect to your postgres DB
conn = psycopg2.connect("dbname=NBA user=postgres password=********")
# Open a cursor to perform database operations
cur = conn.cursor()
cur.execute('CREATE TABLE player_logs("player_id" int,"GameId" int,"Date" int,"Team" VARCHAR(10),"Opponent" VARCHAR(10),"Minutes" int,"Arc3Assists" int,"Arc3FGA" int,"Arc3Frequency" int,"AssistPoints" int,"Assists" int,"AtRimAssists" int,"AtRimFG3AFrequency" int,"Avg2ptShotDistance" int,"Avg3ptShotDistance" int,"BadPassOutOfBoundsTurnovers" int,"BadPassSteals" int,"BadPassTurnovers" int,"Corner3FGA" int,"Corner3Frequency" int,"DeadBallTurnovers" int,"DefArc3ReboundPct" int,"DefFGReboundPct" int,"DefPoss" int,"DefRebounds" int,"DefThreePtReboundPct" int,"DefThreePtRebounds" int,"EfgPct" int,"FG2A" int,"FG2M" int,"FG3A" int,"FG3APct" int,"FTA" int,"Fg2Pct" int,"FirstChancePoints" int,"Fouls" int,"FoulsDrawn" int,"FtPoints" int,"LiveBallTurnoverPct" int,"LiveBallTurnovers" int,"LongMidRangeAccuracy" int,"LongMidRangeAssists" int,"LongMidRangeFGA" int,"LongMidRangeFGM" int,"LongMidRangeFrequency" int,"Loose Ball Fouls" int,"LostBallTurnovers" int,"NonHeaveArc3FGA" int,"OffFGReboundPct" int,"OffPoss" int,"OffRebounds" int,"OffShortMidRangeReboundPct" int,"OffTwoPtReboundPct" int,"OffTwoPtRebounds" int,"OnDefRtg" int,"OnOffRtg" int,"PenaltyArc3FGA" int,"PenaltyArc3Frequency" int,"PenaltyDefPoss" int,"PenaltyEfgPct" int,"PenaltyFG2A" int,"PenaltyFG2M" int,"PenaltyFG3A" int,"PenaltyFg2Pct" int,"PenaltyOffPoss" int,"PenaltyOffPossExcludingTakeFouls" int,"PenaltyOffPossPct" int,"PenaltyPoints" int,"PenaltyPointsExcludingTakeFouls" int,"PenaltyPointsPct" int,"PenaltyShotQualityAvg" int,"PenaltyTsPct" int,"PenaltyTurnovers" int,"Period2Fouls2Minutes" int,"Period3Fouls3Minutes" int,"PlusMinus" int,"Points" int,"PtsUnassisted2s" int,"Rebounds" int,"SecondChanceOffPoss" int,"SelfOReb" int,"SelfORebPct" int,"ShootingFouls" int,"ShootingFoulsDrawnPct" int,"ShortMidRangeAccuracy" int,"ShortMidRangeAssists" int,"ShortMidRangeFGA" int,"ShortMidRangeFGM" int,"ShortMidRangeFrequency" int,"ShortMidRangeOffReboundedPct" int,"ShotQualityAvg" int,"Steals" int,"ThreePtAssists" int,"TotalPoss" int,"TsPct" int,"Turnovers" int,"TwoPtAssists" int,"TwoPtShootingFoulsDrawn" int,"TwoPtShootingFoulsDrawnPct" int,"UnblockedLongMidRangeAccuracy" int,"UnblockedShortMidRangeAccuracy" int,"Usage" int,"Arc3Accuracy" int,"Arc3FGM" int,"Arc3PctAssisted" int,"Assisted2sPct" int,"Assisted3sPct" int,"AtRimAccuracy" int,"AtRimFGA" int,"AtRimFGM" int,"AtRimFrequency" int,"AtRimOffReboundedPct" int,"AtRimPctBlocked" int,"Blocked2s" int,"BlockedShortMidRange" int,"Blocks" int,"BlocksRecoveredPct" int,"Corner3Assists" int,"DefAtRimReboundPct" int,"DefLongMidRangeReboundPct" int,"DefShortMidRangeReboundPct" int,"DefTwoPtReboundPct" int,"DefTwoPtRebounds" int,"FG2APctBlocked" int,"FG3M" int,"Fg2aBlocked" int,"Fg3Pct" int,"LongMidRangeOffReboundedPct" int,"LostBallSteals" int,"NonHeaveArc3Accuracy" int,"NonHeaveArc3FGM" int,"NonHeaveFg3Pct" int,"NonPutbacksAssisted2sPct" int,"NonShootingFoulsDrawn" int,"NonShootingPenaltyNonTakeFoulsDrawn" int,"OffLongMidRangeReboundPct" int,"Offensive Fouls Drawn" int,"PenaltyArc3Accuracy" int,"PenaltyArc3FGM" int,"PenaltyAtRimAccuracy" int,"PenaltyAtRimFGA" int,"PenaltyAtRimFGM" int,"PenaltyAtRimFrequency" int,"PenaltyFG3M" int,"PenaltyFg3Pct" int,"PenaltyFtPoints" int,"PtsAssisted2s" int,"PtsAssisted3s" int,"PtsPutbacks" int,"PtsUnassisted3s" int,"RecoveredBlocks" int,"SecondChanceArc3FGA" int,"SecondChanceArc3Frequency" int,"SecondChanceEfgPct" int,"SecondChanceFG2A" int,"SecondChanceFG2M" int,"SecondChanceFG3A" int,"SecondChanceFg2Pct" int,"SecondChancePoints" int,"SecondChancePointsPct" int,"SecondChanceShotQualityAvg" int,"SecondChanceTsPct" int,"ShortMidRangePctAssisted" int,"ShortMidRangePctBlocked" int,"ThreePtShootingFoulsDrawn" int,"ThreePtShootingFoulsDrawnPct" int,"UnblockedArc3Accuracy" int,"UnblockedAtRimAccuracy" int,"OffArc3ReboundPct" int,"OffThreePtReboundPct" int,"OffThreePtRebounds" int,"Offensive Fouls" int,"Corner3Accuracy" int,"Corner3FGM" int,"ThreePtOffReboundedPct" int,"UnblockedCorner3Accuracy" int,"DefFTReboundPct" int,"FTDefRebounds" int,"Technical Free Throw Trips" int,"BlockedAtRim" int,"LostBallOutOfBoundsTurnovers" int,"OffAtRimReboundPct" int,"BlockedLongMidRange" int,"Charge Fouls Drawn" int,"LongMidRangePctAssisted" int,"NonShootingPenaltyNonTakeFouls" int,"SecondChanceTurnovers" int,"Travels" int,"SecondChanceAtRimFGA" int,"SecondChanceAtRimFrequency" int,"Clear Path Fouls" int,"DefCorner3ReboundPct" int,"HeaveAttempts" int,"LongMidRangePctBlocked" int,"2pt And 1 Free Throw Trips" int,"AtRimPctAssisted" int,"Period3Fouls4Minutes" int,"Period4Fouls4Minutes" int,"Charge Fouls" int,"Loose Ball Fouls Drawn" int,"PeriodOTFouls4Minutes" int,"SecondChanceAtRimAccuracy" int,"SecondChanceAtRimFGM" int,"PenaltyCorner3FGA" int,"PenaltyCorner3Frequency" int,"Corner3PctAssisted" int,"SecondChanceFtPoints" int,"OffCorner3ReboundPct" int,"SecondChanceArc3Accuracy" int,"SecondChanceArc3FGM" int,"SecondChanceFG3M" int,"SecondChanceFg3Pct" int,"3pt And 1 Free Throw Trips" int,"Defensive 3 Seconds Violations" int,"Period4Fouls5Minutes" int,"StepOutOfBoundsTurnovers" int,"Period1Fouls2Minutes"int)')
x = 'https://api.pbpstats.com/get-all-players-for-league/nba'
headers = {'user-agent': 'Chrome/88.0.4324.190'}
jsonData1 = requests.get(x, headers=headers).json() # Player id and name
EntityId = json.loads(json.dumps(jsonData1)[12:-1])
SeasonType = {'R':'Regular+Season','P':'Playoff+Season','A':'All'}
EntityType = {'P':'Player','T':'Team'}
Season = {
'2008-09',
'2009-10',
'2010-11',
'2011-12',
'2012-13',
'2013-14',
'2014-15',
'2015-16',
'2016-17',
'2017-18',
'2018-19',
'2019-20',
'2020-21'
}
def log (S:Season,ST:SeasonType,EI:EntityId,ET:EntityType):
url = 'https://api.pbpstats.com/get-game-logs/nba'
payload = {
'Season': S,
'SeasonType': ST,
'EntityId': EI,
'EntityType': ET
}
r = requests.get(url, headers=headers, params=payload).json()
if r == {'error': 'no results'} :
return()
else :
for c in r['multi_row_table_data']:
j = {'Player_id':EI}
c.update(j)
cur.execute('INSERT INTO player_log (Player_id,GameId,Date,Team,Opponent,Minutes,Arc3Assists,Arc3FGA,Arc3Frequency,AssistPoints,Assists,AtRimAssists,AtRimFG3AFrequency,Avg2ptShotDistance,Avg3ptShotDistance,BadPassOutOfBoundsTurnovers,BadPassSteals,BadPassTurnovers,Corner3FGA,Corner3Frequency,DeadBallTurnovers,DefArc3ReboundPct,DefFGReboundPct,DefPoss,DefRebounds,DefThreePtReboundPct,DefThreePtRebounds,EfgPct,FG2A,FG2M,FG3A,FG3APct,FTA,Fg2Pct,FirstChancePoints,Fouls,FoulsDrawn,FtPoints,LiveBallTurnoverPct,LiveBallTurnovers,LongMidRangeAccuracy,LongMidRangeAssists,LongMidRangeFGA,LongMidRangeFGM,LongMidRangeFrequency,Loose_Ball_Fouls,LostBallTurnovers,NonHeaveArc3FGA,OffFGReboundPct,OffPoss,OffRebounds,OffShortMidRangeReboundPct,OffTwoPtReboundPct,OffTwoPtRebounds,OnDefRtg,OnOffRtg,PenaltyArc3FGA,PenaltyArc3Frequency,PenaltyDefPoss,PenaltyEfgPct,PenaltyFG2A,PenaltyFG2M,PenaltyFG3A,PenaltyFg2Pct,PenaltyOffPoss,PenaltyOffPossExcludingTakeFouls,PenaltyOffPossPct,PenaltyPoints,PenaltyPointsExcludingTakeFouls,PenaltyPointsPct,PenaltyShotQualityAvg,PenaltyTsPct,PenaltyTurnovers,Period2Fouls2Minutes,Period3Fouls3Minutes,PlusMinus,Points,PtsUnassisted2s,Rebounds,SecondChanceOffPoss,SelfOReb,SelfORebPct,ShootingFouls,ShootingFoulsDrawnPct,ShortMidRangeAccuracy,ShortMidRangeAssists,ShortMidRangeFGA,ShortMidRangeFGM,ShortMidRangeFrequency,ShortMidRangeOffReboundedPct,ShotQualityAvg,Steals,ThreePtAssists,TotalPoss,TsPct,Turnovers,TwoPtAssists,TwoPtShootingFoulsDrawn,TwoPtShootingFoulsDrawnPct,UnblockedLongMidRangeAccuracy,UnblockedShortMidRangeAccuracy,Usage,Arc3Accuracy,Arc3FGM,Arc3PctAssisted,Assisted2sPct,Assisted3sPct,AtRimAccuracy,AtRimFGA,AtRimFGM,AtRimFrequency,AtRimOffReboundedPct,AtRimPctBlocked,Blocked2s,BlockedShortMidRange,Blocks,BlocksRecoveredPct,Corner3Assists,DefAtRimReboundPct,DefLongMidRangeReboundPct,DefShortMidRangeReboundPct,DefTwoPtReboundPct,DefTwoPtRebounds,FG2APctBlocked,FG3M,Fg2aBlocked,Fg3Pct,LongMidRangeOffReboundedPct,LostBallSteals,NonHeaveArc3Accuracy,NonHeaveArc3FGM,NonHeaveFg3Pct,NonPutbacksAssisted2sPct,NonShootingFoulsDrawn,NonShootingPenaltyNonTakeFoulsDrawn,OffLongMidRangeReboundPct,Offensive_Fouls_Drawn,PenaltyArc3Accuracy,PenaltyArc3FGM,PenaltyAtRimAccuracy,PenaltyAtRimFGA,PenaltyAtRimFGM,PenaltyAtRimFrequency,PenaltyFG3M,PenaltyFg3Pct,PenaltyFtPoints,PtsAssisted2s,PtsAssisted3s,PtsPutbacks,PtsUnassisted3s,RecoveredBlocks,SecondChanceArc3FGA,SecondChanceArc3Frequency,SecondChanceEfgPct,SecondChanceFG2A,SecondChanceFG2M,SecondChanceFG3A,SecondChanceFg2Pct,SecondChancePoints,SecondChancePointsPct,SecondChanceShotQualityAvg,SecondChanceTsPct,ShortMidRangePctAssisted,ShortMidRangePctBlocked,ThreePtShootingFoulsDrawn,ThreePtShootingFoulsDrawnPct,UnblockedArc3Accuracy,UnblockedAtRimAccuracy,OffArc3ReboundPct,OffThreePtReboundPct,OffThreePtRebounds,Offensive_Fouls,Corner3Accuracy,Corner3FGM,ThreePtOffReboundedPct,UnblockedCorner3Accuracy,DefFTReboundPct,FTDefRebounds,Technical_Free_Throw_Trips,BlockedAtRim,LostBallOutOfBoundsTurnovers,OffAtRimReboundPct,BlockedLongMidRange,Charge_Fouls_Drawn,LongMidRangePctAssisted,NonShootingPenaltyNonTakeFouls,SecondChanceTurnovers,Travels,SecondChanceAtRimFGA,SecondChanceAtRimFrequency,Clear_Path_Fouls,DefCorner3ReboundPct,HeaveAttempts,LongMidRangePctBlocked,"2pt_And_1_Free_Throw_Trips",AtRimPctAssisted,Period3Fouls4Minutes,Period4Fouls4Minutes,Charge_Fouls,Loose_Ball_Fouls_Drawn,PeriodOTFouls4Minutes,SecondChanceAtRimAccuracy,SecondChanceAtRimFGM,PenaltyCorner3FGA,PenaltyCorner3Frequency,Corner3PctAssisted,SecondChanceFtPoints,OffCorner3ReboundPct,SecondChanceArc3Accuracy,SecondChanceArc3FGM,SecondChanceFG3M,SecondChanceFg3Pct,"3pt_And_1_Free_Throw_Trips",Defensive_3_Seconds_Violations,Period4Fouls5Minutes,StepOutOfBoundsTurnovers,Period1Fouls2Minutes) VALUES',
c)
return()
y=log('2020-21','Regular+Season','101108','Player')
conn.commit()
conn.close()
cur.close()
So was wondering if i could insert the data so it matched with the key and the column name. So the table and dict isnt order the same way either if it makes a difference.
This is fairly simple to do by adopting two helper libraries: pandas and preql.
You can use pandas to load the json into a single dataframe, and then use preql to import it into the database.
Here is runnable code demonstrating how to do it:
import requests
import pandas as pd
from preql import Preql
headers = {'user-agent': 'Chrome/88.0.4324.190'}
def log(S,ST,EI,ET):
url = 'https://api.pbpstats.com/get-game-logs/nba'
payload = {
'Season': S,
'SeasonType': ST,
'EntityId': EI,
'EntityType': ET
}
r = requests.get(url, headers=headers, params=payload).json()
if r == {'error': 'no results'} :
return
else:
return [{'Player_id':EI, **d} for d in r['multi_row_table_data']]
rows=log('2020-21','Regular+Season','101108','Player')
df = pd.DataFrame.from_dict(rows)
print("Dataframe shape:", df.shape) # (50, 218)
p = Preql() # For postgres use: p = Preql("postgres://user:pass#server")
p.import_pandas(my_table=df)
print('SQL columns:', p('count(columns(my_table))')) # 219 - includes id
print('SQL rows:', p('count(my_table)')) # 50
Note that this code example is currently using Python's built-in Sqlite, but you can easily make it work with postgres by providing Preql with the postgres URL, as the comment shows.
Install them with pip install pandas preql-lang
Might not be the best solution, but I wrote a small function that just converts to SQL statement.
def insert_into_table_query(table, data):
col_names = list(data.keys())
task = tuple(data.values())
col_str = ', '.join(str(item) for item in col_names)
col_str = '(' + col_str + ')'
value_str = ', '.join('%s' for item in task)
value_str = '(' + value_str + ');'
sql = 'INSERT INTO {tn} '.format(tn=table) + col_str + ' VALUES ' + value_str
return (sql, task)
I am trying to use a returning into clause with cx_oracle (version 7.3) to grab the ids generated by a sequence in one of my tables. However I am not getting the value of the sequence field that I am expecting. I want the value of an_ash_s.nextval
The call to my function looks like this:
self.insert_into_columns_with_return('AN_SHIPMENT', payload.shipment_columns, payload.shipment_rows, id_sequence='an_ash_s.nextval')
where payload.shipment_columns looks like
['ASH_ID', 'ASH_AJ_ID', 'ASH_CD_PROCESS_STATUS', 'ASH_PROCESS_ID', 'ASH_SHIPMENT_KEY', 'ASH_ORG_OPERATIONAL_ID', 'ASH_SEED_EQUIP', 'ASH_SEED_EQUIP_CODE', 'ASH_SHIP_DATE', 'ASH_SHIP_DATE_DSP', 'ASH_SHIP_DIRECTION', 'ASH_SHIP_DIRECTION_DSP', 'ASH_FREIGHT_TERMS', 'ASH_FREIGHT_TERMS_DSP', 'ASH_WEIGHT', 'ASH_WEIGHT_MEASURE', 'ASH_HAZ_FLAG', 'ASH_CREATE_DATE', 'ASH_PACKAGE_COUNT', 'ASH_SHIPPING_CLASS_ID', 'ASH_SHIPPING_CLASS_TYPE', 'ASH_ORG_CONSIGNOR_ID', 'ASH_ORG_CONSIGNOR_NAME', 'ASH_LOC_ORIG_ID', 'ASH_LOC_ORIG_COUNTRY_ID', 'ASH_LOC_ORIG_STATE_CODE', 'ASH_LOC_ORIG_CITY', 'ASH_LOC_ORIG_POSTAL_CODE', 'ASH_ORG_CONSIGNEE_ID', 'ASH_ORG_CONSIGNEE_NAME', 'ASH_LOC_DEST_ID', 'ASH_LOC_DEST_COUNTRY_ID', 'ASH_LOC_DEST_STATE_CODE', 'ASH_LOC_DEST_CITY', 'ASH_LOC_DEST_POSTAL_CODE', 'ASH_ERROR_MESSAGE', 'ASH_USE_CURRENT_DATE']
And Payload Shipment rows looks like:
[[310, '5', None, 'Test', '*ISP_CLT', 'LTL', 'LTL', datetime.date(2019, 4, 15), '03/15/2019', 'I', 'Inbound', 'P', 'Pre-Paid', 3000, 'LB', 'N', datetime.date(2019, 3, 24), None, '70', None, None, None, '241144', 'US', 'GA', 'ANYTOWN', '25451', None, None, '12345', 'US', 'VA', 'BANKS', '45678', None, 'N']]
Based on the feedback I have received I have modified my function to look like this:
def insert_into_columns_with_return(self, table_name, columns, rows, id_sequence=None):
arrstr = rows
col_str = ''
for col_id in range(1, len(columns) + 1):
col_str += columns[col_id - 1]
if col_id < len(columns):
col_str += ', '
with self.conn.cursor() as cur:
intCol = cur.var(int)
childIdVar = cur.var(int, arraysize=len(arrstr))
cur.setinputsizes(None, childIdVar)
if(id_sequence == None):
sql = "INSERT INTO {table_name} ({column_names}) VALUES (:arrstr) RETURNING ASH_ID INTO :intCol"
sql = sql.format(table_name=table_name, column_names =col_str)
elif (id_sequence != None):
sql = "INSERT INTO {table_name} ({column_names}) VALUES ( {id_sequence}, :arrstr ) RETURNING ASH_ID INTO :intCol"
sql = sql.format(table_name=table_name, column_names=col_str, id_sequence=id_sequence)
cur.executemany(sql, [tuple(x) for x in arrstr])
for ix, stri in enumerate(arrstr):
print("IDs Str", stri, "is", childIdVar.getvalue(ix))
self.conn.commit()
However I am now getting an error cx_Oracle.DatabaseError: ORA-01036: illegal variable name/number
I thought the goal was to feed execute many a list of tuples but I think it does not accept it.
As your code is not complete and usable, I tried the following bit of code, taken from the doc here
cursor = con.cursor()
intCol = cursor.var(int)
arrstr=[ ("First" ),
("Second" ),
("Third" ),
("Fourth" ),
("Fifth" ),
("Sixth" ),
("Seventh" ) ]
print("Adding rows", arrstr)
print(intCol.getvalue())
childIdVar = cursor.var(int, arraysize=len(arrstr))
cursor.setinputsizes(None, childIdVar)
cursor.executemany("insert into treturn values (tret.nextval, :arrstr) returning c1 into :intCol",
[(i,) for i in arrstr])
for ix, stri in enumerate(arrstr):
print("IDs Str", stri, "is",
childIdVar.getvalue(ix))
and I get this output
Adding rows ['First', 'Second', 'Third', 'Fourth', 'Fifth', 'Sixth', 'Seventh']
None
IDs Str First is [24]
IDs Str Second is [25]
IDs Str Third is [26]
IDs Str Fourth is [27]
IDs Str Fifth is [28]
IDs Str Sixth is [29]
IDs Str Seventh is [30]
i'm a newbie in python3. My homework is create a Sqlite database include 10 tables, each table contains 50 columns, each columns contains 1000 rows, data is randomly generated using Python. I have almost done.
My code :
import sqlite3
conn = sqlite3.connect('testmydb.db')
cur = conn.cursor()
for table_number in range(1,11):
cur.execute('''CREATE TABLE table''' + str(table_number) + '''(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)''')
listOfColumns = ("column0",)
for column_number in range(1,49):
newColumn = ("column" + str(column_number),)
listOfColumns = listOfColumns + newColumn
for column_number in listOfColumns:
cur.execute('''ALTER TABLE table''' + str(table_number) + ''' ADD COLUMN %s TEXT''' % column_number)
conn.commit()
cur.close()
conn.close()
Now i want to insert 1000 row into 1 colums but i'm confusing when i wanted create a for loop more. Can anyone suggest me ?
The following should do what you want (see comments)
import sqlite3
import random
conn = sqlite3.connect('testmydb.db')
cur = conn.cursor()
#Only need to do this once
listOfColumns = ("column0",)
bindMarkers = ",?" #ADDED to allow values to be bound will be ?,?,?,?, ........ 49 ?
for column_number in range(1, 49):
newColumn = ("column" + str(column_number),)
listOfColumns = listOfColumns + newColumn
bindMarkers = bindMarkers + ",?"
for table_number in range(1,11):
cur.execute("DROP TABLE IF EXISTS table" + str(table_number)) #make it rerunnable
cur.execute('''CREATE TABLE IF NOT EXISTS table''' + str(table_number) + '''(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT)''')
for column_number in listOfColumns:
cur.execute('''ALTER TABLE table''' + str(table_number) + ''' ADD COLUMN %s TEXT''' % column_number)
# The INSERT statement note null means that the id will be automatically generated
insertsql = "INSERT INTO table" + str(table_number) + " VALUES(null" + bindMarkers + ")"
#print the INSERT SQL (just the once)
if table_number == 1:
print(insertsql)
for row_number in range(1,1001):
# Generate a list of 49 random values
listOfRandomValues =[random.randint(1, 999999999999) for i in range(49)]
cur.execute(insertsql,listOfRandomValues) # insert the row
# extract the first 5 rows an print each row
cursor = cur.execute("SELECT * FROM table" + str(table_number) + " LIMIT 5")
result = "row in table table" + str(table_number) + " Data is "
for row in cursor:
print(row)
conn.commit()
cur.close()
conn.close()
This will produce output like (first line is the first INSERT statement/SQL) :-
INSERT INTO table1 VALUES(null,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
(1, '208968800970', '673486951978', '416011320117', '257739455602', '161014001387', '66915092142', '192394825558', '894946418178', '147479449787', '429768915009', '343072031065', '312483697033', '38240897968', '179592184222', '517690986147', '401721693004', '760956848808', '787028914225', '658523299261', '923606731801', '740090529164', '169600507787', '441903806645', '82302358448', '250921627878', '542116452618', '998918595471', '775548995005', '733089506549', '957054106540', '449321507524', '798501631292', '409382414444', '945602662286', '706232454927', '930118739979', '691405693853', '201175361297', '513975533346', '16690109599', '592944414377', '948709328664', '490207084748', '406188522423', '799744354342', '474761616653', '314527920015', '94102072722', '912028741567')
(2, '172875509043', '126844020427', '423436418690', '973535472434', '171412421537', '693479106176', '909004577995', '920911700813', '605955273811', '325652512054', '94057263900', '45907520985', '64928934172', '301130729226', '103229253943', '114469347031', '551553752113', '626314462779', '22617947251', '997836163264', '585793592332', '620096766798', '565760327235', '348031514661', '871589505728', '58320228377', '179652288652', '977988196994', '742110712624', '201181530463', '816248034687', '22951611374', '723154858722', '289036915539', '997272483698', '61348539011', '977373908399', '668284539899', '55348735729', '263726052214', '662603583920', '790720286573', '487793507420', '883073500835', '519633722649', '383008255347', '30563959610', '617324332661', '89956476106')
(3, '253567041183', '70027774987', '535230659770', '191267720449', '791090949115', '399626615217', '649492276413', '594283985270', '983353743022', '713002984294', '982490173135', '109850128623', '571489216078', '900560015434', '729185220526', '362712800267', '619582132251', '990925729743', '144006421433', '790742578660', '64886161120', '266462916556', '89211644675', '941650491818', '878437527129', '827767387129', '899754797443', '280555144440', '623469334050', '882001652568', '395198811620', '393149546360', '509545198950', '534252806675', '582802496697', '674715538387', '748829323303', '296068248515', '573789396002', '84015250035', '963083904856', '677426863455', '173505995385', '569976297792', '643158854425', '273191627696', '676364784545', '536715691007', '678846958313')
(4, '575055534615', '179094408882', '418242646417', '258767847915', '533305509121', '800410396430', '416643709991', '453093467839', '352906227023', '711478657972', '542560050616', '477511637703', '464619274323', '438591712313', '293891594997', '638717557413', '796607432824', '845617819673', '682479247215', '687662681530', '682910774205', '547150987433', '645097550529', '781225444825', '498491871793', '280308928866', '386747319120', '175187502068', '554032903538', '906897892968', '847200546291', '724824936579', '257524554306', '341479642174', '628478037881', '41911000836', '487139622046', '698641404274', '300203051807', '321147725978', '201308004931', '324554566932', '54668008952', '799888599714', '544776279131', '851164639529', '1118079080', '993554994315', '97774308420')
(5, '263377483252', '535276579958', '434436394255', '235123585872', '886866465625', '83437890933', '546739192349', '832929945092', '889303183895', '517501283515', '386452334064', '437005515113', '567305852696', '254940127493', '158473804439', '714105412308', '887616841407', '873758857265', '59024734698', '495085412255', '757296111012', '438130715784', '661863799528', '370244296694', '559859930401', '409259131854', '72716791778', '900054227569', '897455645761', '254989679831', '46456169823', '597888422562', '581408791663', '191438417130', '468539979785', '998729241595', '596707251066', '731997835957', '432001941801', '351970232680', '602771773558', '793033654396', '205236245465', '547142878108', '973842386021', '742055066627', '455501634405', '130419180039', '870186517783')
(1, '472841964440', '177094420514', '859773622393', '943573354468', '909606787130', '278659426379', '129796913302', '67857238168', '104155180296', '581639712382', '451184580063', '917433785632', '226959780068', '190462507493', '256274613979', '919674630928', '976702823134', '121337013780', '254022515917', '293782992065', '903483153770', '147697931939', '279062893088', '553519369139', '962433270653', '640822114280', '816716757345', '999707836592', '697963179054', '104305203866', '735705858863', '617083342099', '262076004375', '797912340506', '205887749382', '576489282235', '705096989440', '670969562520', '649164826831', '311493582872', '760367591190', '749686855909', '819181100789', '466265188300', '304292298579', '420782152623', '854335337149', '916391611738', '964274785687')
(2, '621325506597', '776006955683', '137683264810', '351906945610', '682429690372', '965366508605', '666337420753', '453325880143', '70778770818', '103682937480', '868216544504', '229703959756', '41004116292', '507097353534', '871910281669', '251530835311', '836500603189', '601460038094', '897559700303', '681312522817', '161143454247', '553960203443', '777460295192', '458302954528', '977754347041', '892360041754', '681995024692', '248485864749', '348381577064', '450879805019', '650777503736', '353872867221', '97506344721', '747237255889', '455629065944', '861413783175', '214743871915', '77511793017', '621196858622', '825422146350', '489409477723', '908004452720', '238639741015', '426722798842', '980323652543', '561628376666', '838205614824', '784039262073', '949055065484')
(3, '736008123891', '923934389646', '546159245294', '429258073881', '583372466354', '50804206500', '273716995212', '733988654121', '788160350686', '749598895287', '551751993459', '916986772574', '622366294456', '687624270621', '185660393899', '329963428664', '928661078668', '875765821125', '754653923243', '151547845857', '248763933358', '636547599095', '87140063802', '267688269107', '224477253917', '641792646340', '59046381016', '103443043545', '485267444040', '387215340714', '268223896307', '480068950182', '225811319773', '492031230630', '502916805016', '514567127425', '178032451267', '750288734257', '825600642728', '641081438590', '207022050440', '902457228778', '115373751089', '348372424350', '768147081429', '715162751738', '210598155420', '196905259558', '873091126544')
(4, '560125266801', '378302831641', '471084702841', '679900688640', '201624340251', '909766550240', '687623074376', '116508086811', '217573740193', '378086229046', '466649195230', '932285473013', '648745964471', '968517127245', '748917121449', '224930472692', '698734544540', '793428186573', '153336974374', '24843476682', '42926459163', '503345524005', '116363947828', '524399560588', '238188045685', '3353134402', '97245283198', '780904780984', '768226492682', '337351478339', '761762114083', '4108216481', '715457129140', '718946387960', '808632491477', '283509135313', '750631442686', '302040053814', '354520401885', '30869550070', '831081853310', '317334330124', '175699898404', '316762996417', '144843539429', '647890863625', '500905345131', '686585819856', '439083530058')
(5, '786320993918', '418227705376', '222672045565', '50994821164', '445050766070', '655740733971', '144925180595', '178456995314', '968483620704', '217344736719', '659133382247', '699130444999', '645737723689', '211418136852', '977174813693', '404005933734', '416012774264', '498694089898', '286235598876', '105048705716', '745323502156', '22320974963', '287621972357', '484051431377', '677832782489', '175141638805', '652237666867', '633826915005', '826792363302', '181964153730', '549735148579', '820006084751', '622355043852', '615716362152', '337022948655', '280970738440', '264064973515', '550249406679', '912858473551', '542805313957', '43397863679', '257720759974', '189160263335', '265086252271', '692156831796', '860245023055', '769544988002', '856033591981', '865669688852')
(1, '29773154022', '105812125224', '923886735040', '494040618517', '406872772654', '964605045362', '483548207268', '222657267987', '728533595865', '427758006630', '250839721516', '246117222632', '625392752778', '372756660516', '276521371279', '677307428516', '434498176501', '757867858941', '568841625163', '315224423736', '939706907834', '567757610656', '977473375050', '476473505693', '921117900131', '344700573908', '350627473109', '569315794206', '780528101292', '957322180230', '952406583209', '435610932961', '463449885730', '174468401098', '916963726643', '193968348451', '297427605119', '481930164885', '685603984144', '543719297225', '612929787721', '475021539217', '176642603133', '74400339089', '95276914071', '808000358479', '79312180687', '502877681225', '659274942719')
..........
i still don't get what is variable bindMarkers and if table_number == 1: print(insertsql)
First the 2nd the line if table_number == 1: print(insertsql)
Is just printing out the INSERT statement to show what it looks like. It was just included for that and is not necessary. BUT, it's useful to know what it looks like to explain the ? placeholder and binding values.
So the INSERT statement is along the lines of
INSERT INTO tablex VALUES(null,?,? ....... (49 ?'s)
tablex where x represents 1-10
First null as per the comment allows SQLite to generate a unique value for the id column.
Each ? is a placeholder and will be replaced by a bound value. This technique prevents SQL injection.
bindMarkers is just a string that is generated with 1 ? per column so it's a string of 49 ?'s (easier than typing VALUES(null,?,?,?,?,? .....) and also more flexible/adaptable if the column number were to change).
You see that the line listOfRandomValues =[random.randint(1, 999999999999) for i in range(49)] creates a List of 49 random values each will be used to replace a single ? (the first value replaces the first ?, the second value replaces the second ? and so on).
This is considered better practice than building a statement along the lines of
INSERT INTO tablex VALUES(null,'208968800970', '673486951978', '416011320117', '257739455602', '161014001387', '66915092142', '192394825558', '894946418178', '147479449787', '429768915009', '343072031065', '312483697033', '38240897968', '179592184222', '517690986147', '401721693004', '760956848808', '787028914225', '658523299261', '923606731801', '740090529164', '169600507787', '441903806645', '82302358448', '250921627878', '542116452618', '998918595471', '775548995005', '733089506549', '957054106540', '449321507524', '798501631292', '409382414444', '945602662286', '706232454927', '930118739979', '691405693853', '201175361297', '513975533346', '16690109599', '592944414377', '948709328664', '490207084748', '406188522423', '799744354342', '474761616653', '314527920015', '94102072722', '912028741567')
The statement itself is shorter (i.e 1 ? instead of 12 digits) and therefore less likely to cause issues with limits.
I have a function named "search_suggestion" that takes search parameter and pass into MySQL then a result is appended into an empty list "Suggestion" inside a function below
def search_suggestion(self,search,limit=25):
"""This method takes the parameter search return the search suggestion of employees in database"""
cursor = None
suggestions = []
try:
cursor = kasaa()
cursor.execute(
'''
SELECT ospos_people.first_name,ospos_people.last_name
FROM ospos_employees
INNER JOIN ospos_people ON ospos_employees.person_id = ospos_people.person_id
WHERE ospos_employees.deleted = 0 AND ospos_people.first_name LIKE %s OR ospos_people.last_name LIKE %s
OR ospos_people.phone_number LIKE %s OR ospos_people.email LIKE %s
ORDER BY ospos_people.first_name ASC LIMIT %s
''',(search,search,search,search,limit)
)
row = cursor.fetchall()
for ro in row:
suggestions.append(ro["first_name"]+ " " + ro["last_name"])
print(suggestions)
except Exception as e:
print(e)
finally:
cursor.close()
what am expecting is a list like ['alkhadil Issa', 'john Magufuli'] a one single list
instead am getting two list.
[alkhadil Issa']
['alkhadil Issa' 'john Magufuli']
I have try to check if len(suggestions) < 1: before append ro["first_name"] but am not getting what i want. What is the most efficient way of doing this, any patient you can afford on my learning journey i would appreciate
I replicated your problem by manually creating an output similar to what cursor.fetchall() returns according to you.
>>> dic1 = {'first_name': 'Abdallah', 'last_name': 'Abdillah'}
>>> dic2 = {'first_name': 'Joseph', 'last_name': 'Magufuli'}
>>> row = [dic1, dic2]
>>> row
[{'first_name': 'Abdallah', 'last_name': 'Abdillah'}, {'first_name': 'Joseph', 'last_name': 'Magufuli'}]
Assuming cursor.fetchall() returns something similar to the list above your code should work fine:
>>> suggestions = []
>>> for r in row:
... suggestions.append(r['first_name'] + " " + r['last_name'])
... print(suggestions)
...
['Abdallah Abdillah']
['Abdallah Abdillah', 'Joseph Magufuli']
If that is not the case, then your problem is your cursor.fetchall() result.
Edit:
I just realized your problem is getting 2 lists. Please be aware that your print statement is inside the for loop, so each time a value is added to the list, the resulting list is printed. If you only want to print the list in the end, just add the print statement after the loop ends:
So, instead of:
>>> for dic in row:
... suggestions.append(dic['first_name'] + " " + dic['last_name'])
... print(suggestions)
...
['Abdallah Abdillah']
['Abdallah Abdillah', 'Joseph Magufuli']
Place the print outside of the loop:
>>> for r in row:
... suggestions.append(r['first_name'] + " " + r['last_name'])
...
>>> print(suggestions)
['Abdallah Abdillah', 'Joseph Magufuli']
Below is a small subset of the data I'm working with. I can format the data any way I please. The data within the variable 'dc' is made up of the values 'id1' and 'id2'. What I want to do is be able to issue one SELECT statement for all of the values I have in 'dc'. For some reason, no matter what I try in the 'cursor.execute' statement or within the 'format_strings' variable I can't seem to get the proper code to be able to pass two variables to MySQL.
Comments/suggestions on how to format the data ('dc') or code to perform one SELECT statement would be very helpful.
results = ()
dc = ['103,4770634', '42,427752', '64,10122045', '42,13603629', '42,25516425', '103,2748102', '42,1966402', '42,30262834', '42,6667711', '18,13737683', '42,28921168', '42,26076925', '103,3733654', '42,23313527', '64,3307344', '103,3973533', '42,6360982', '48,11846077', '103,3775309', '64,10122050', '42,1965119', '103,4265810', '103,3971645', '103,4962583', '103,689615', '42,22834366', '103,761655', '95,1184', '64,9594482', '42,22855603', '48,8654764', '103,4226756', '42,23366982', '103,3897036', '42,11339650', '101,6369', '42,25830920', '103,5009291', '42,29238961', '59,6299475', '42,22931663', '42,25839056', '43,11864458', '43,41346192', '103,4261645', '42,3747082', '103,4795050', '42,9417503', '103,4245623', '42,61431911']
try:
format_strings = ','.join(['%s%s'] * len(dc))
cursor.execute("SELECT * FROM tbl1 WHERE id1=(%s) AND id2=(%s)" % format_strings, (dc))
res = cursor.fetchall()
results = results + res
except Exception, e:
print e
UPDATE
Taking what #lecumia and #beroe posted below I came up with the following, not as elegant and probably not super efficient but it works.
results = ()
id1 = []
id2 = []
dc = ['103,4770634', '42,427752', '64,10122045', '42,13603629', '42,25516425']
for d in dc:
id1.append(d.split(',')[0])
id2.append(d.split(',')[1])
try:
sql = "SELECT * FROM DomainEmails WHERE email_id IN (%s) AND domain_id IN (%s)"
in_id1 = "'" + "', '".join(id1) + "'"
in_id2 = "'" + "', '".join(id2) + "'"
sql = sql % (in_id1, in_id2)
cursor.execute(sql)
res = cursor.fetchall()
results = results + res
except Exception, e:
print e
Actual Query
SELECT * FROM tbl1 WHERE id1 IN ('103', '42', '64', '42', '42') AND id2 IN ('4770634', '427752', '10122045', '13603629', '25516425')
Query Results
These match what I was expecting:
{'id1': 42L, 'id2': 427752L, 'firstseen': datetime.date(2010, 5, 6)}
{'id1': 42L, 'id2': 427752L, 'firstseen': datetime.date(2011, 5, 2)}
{'id1': 42L, 'id2': 13603629L, 'firstseen': datetime.date(2011, 3, 21)}
{'id1': 42L, 'id2': 13603629L, 'firstseen': datetime.date(2011, 4, 17)}
based on
Executing "SELECT ... WHERE ... IN ..." using MySQLdb
results = ()
dc = ['103,4770634', '42,427752', '64,10122045', '42,13603629', '42,25516425', '103,2748102', '42,1966402', '42,30262834', '42,6667711', '18,13737683', '42,28921168', '42,26076925', '103,3733654', '42,23313527', '64,3307344', '103,3973533', '42,6360982', '48,11846077', '103,3775309', '64,10122050', '42,1965119', '103,4265810', '103,3971645', '103,4962583', '103,689615', '42,22834366', '103,761655', '95,1184', '64,9594482', '42,22855603', '48,8654764', '103,4226756', '42,23366982', '103,3897036', '42,11339650', '101,6369', '42,25830920', '103,5009291', '42,29238961', '59,6299475', '42,22931663', '42,25839056', '43,11864458', '43,41346192', '103,4261645', '42,3747082', '103,4795050', '42,9417503', '103,4245623', '42,61431911']
try:
sql = "SELECT * FROM tbl1 WHERE id1 in (%s) AND id2 in (%s)"
in_ids = ', '.join(map(lambda x: '%s', dc))
in_ids = in_ids % tuple(dc)
sql = sql % (in_ids, in_ids)
cursor.execute(sql)
res = cursor.fetchall()
results = results + res
except Exception, e:
print e
Results
SELECT * FROM tbl1 WHERE id1 in (103,4770634, 42,427752, 64,10122045, 42,13603629, 42,25516425, 103,2748102, 42,1966402, 42,30262834, 42,6667711, 18,13737683, 42,28921168, 42,26076925, 103,3733654, 42,23313527, 64,3307344, 103,3973533, 42,6360982, 48,11846077, 103,3775309, 64,10122050, 42,1965119, 103,4265810, 103,3971645, 103,4962583, 103,689615, 42,22834366, 103,761655, 95,1184, 64,9594482, 42,22855603, 48,8654764, 103,4226756, 42,23366982, 103,3897036, 42,11339650, 101,6369, 42,25830920, 103,5009291, 42,29238961, 59,6299475, 42,22931663, 42,25839056, 43,11864458, 43,41346192, 103,4261645, 42,3747082, 103,4795050, 42,9417503, 103,4245623, 42,61431911) AND id2 in (103,4770634, 42,427752, 64,10122045, 42,13603629, 42,25516425, 103,2748102, 42,1966402, 42,30262834, 42,6667711, 18,13737683, 42,28921168, 42,26076925, 103,3733654, 42,23313527, 64,3307344, 103,3973533, 42,6360982, 48,11846077, 103,3775309, 64,10122050, 42,1965119, 103,4265810, 103,3971645, 103,4962583, 103,689615, 42,22834366, 103,761655, 95,1184, 64,9594482, 42,22855603, 48,8654764, 103,4226756, 42,23366982, 103,3897036, 42,11339650, 101,6369, 42,25830920, 103,5009291, 42,29238961, 59,6299475, 42,22931663, 42,25839056, 43,11864458, 43,41346192, 103,4261645, 42,3747082, 103,4795050, 42,9417503, 103,4245623, 42,61431911)