I am downloading 15Y of data (daily close) for 5 stocks ('A','AAP','AAPL','ABBV','ABC'). The issue is that I got some repetitions. No issue for the first one ,'A', I got the right amount of data. For the second one,'AAP', I have twice the right number of rows, it seems the data were downloaded twice. Same issue for the last 3 stocks for which I have three times the right number of rows. I have attached a screenshot showing the size of the csv files, these files should have the same size if everything was fine.
I suspect that the issue comes from the 10 seconds pause after calling reqHistoricalData; it may be too long. How could I avoid having duplicated rows and how to pause the right amount of time (not too long and not too short)?
import pandas as pd
import datetime as dt
import time
import collections
import threading
import os
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
from ibapi.common import BarData
path = r"D:\trading\data\debug\\"
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
self.data=collections.defaultdict(list)
def nextValidId(self, orderId: int):
super().nextValidId(orderId)
self.nextorderId = orderId
print('The next valid order id is: ', self.nextorderId)
def error(self, reqId, errorCode, errorString):
super().error(reqId, errorCode, errorString)
print("Error. Id:", reqId, "Code:", errorCode, "Msg:", errorString)
def historicalData(self, reqId:int, bar:BarData):
self.data["date"].append(bar.date)
self.data["close"].append(bar.close)
self.df = pd.DataFrame.from_dict(self.data)
tickers = ["A","AAP","AAPL","ABBV","ABC"]
def run_loop():
app.run()
app = IBapi()
app.connect("127.0.0.1", 7496, 5)
app.nextorderId = None
# Start the socket in a thread
api_thread = threading.Thread(target=run_loop, daemon=True)
api_thread.start()
# Check if the API is connected via orderid
while True:
if isinstance(app.nextorderId, int):
print('connected')
break
else:
print('waiting for connection')
time.sleep(1)
n_id = app.nextorderId
for ticker in tickers:
contract = Contract()
contract.symbol = ticker
contract.secType = "STK"
contract.exchange = "SMART"
contract.currency = "USD"
app.reqHistoricalData(n_id, contract, "","15 Y", "1 day", "TRADES", 1, 1, False, [])
time.sleep(10)
app.df.to_csv(path + ticker + ".csv")
n_id = n_id + 1
app.disconnect()
You don't clear the list in between requests.
def historicalData(self, reqId:int, bar:BarData):
# just keeps adding data to list
self.data["date"].append(bar.date)
self.data["close"].append(bar.close)
# makes a new dataframe on every single bar
self.df = pd.DataFrame.from_dict(self.data)
In the historicalDataEnd method you can make a dataframe and save it to a file. Make a dict of tickers and reqId's so you know which ticker is finished.
You should still have a 10 second delay in between calls for pacing but do not count on data being returned within 10 seconds. If it doesn't arrive, you will get an empty file (or in your case, all the previous tickers data, which seems to have happened with ABC).
Your duplicates come every Friday. You make a request for, say Friday (1st iteration) and in the next 2 iterations (which are Saturday and Sunday) the API returns data from the first possible trading day (last Friday). Otherwise 5 seconds is enough time to wait.
Related
I'm new in programming, sorry if somwhere make simplemistakes or do not get something.
I try to use IBApi for build script.
Main point - request historical data continuously(one time in 15 sec for last week timerange, as example), convert it to dataframe and make further calculations.
But i can't do that as guide shows, it does not work that way.
I guess there is a way to do it simply, as with ib_insync.
To get code example, how i can request bars data from IB using IBApi and convert to dataframe
import time
from ibapi.client import *
from ibapi.wrapper import *
from ibapi.contract import Contract
import pandas as pd
from datetime import timedelta
import mplfinance as mpf
class TestApp(EClient, EWrapper):
# put number of candles for plot(check timerange in "self.reqHistoricalData" - must be bigger than this value
cnadles_plot = 10000
# Simply shift hours forward+ or back-
hours_change = -5
def __init__(self):
EClient.__init__(self, self)
def nextValidId(self, orderId: int):
mycontract = Contract()
mycontract.symbol = "TSLA"
mycontract.secType = "STK"
mycontract.exchange = "SMART"
mycontract.currency = "USD"
self.reqMarketDataType(4)
self.reqMktData(orderId, mycontract, "", 0, 0, [])
self.histbars = []
self.reqHistoricalData(orderId, mycontract, "20221010-15:00:00", "5 D", "1 min", "TRADES", 0, 1, 0, [])
def historicalData(self, reqId: int, bar: BarData):
bardict = {"HistoricalData": reqId, "Date": bar.date, "Open": bar.open, "High": bar.high, "Low": bar.low,
"Close": bar.close, "Volume": bar.volume, "Count": bar.barCount}
self.histbars.append(bardict)
def historicalDataEnd(self, reqId: int, start: str, end: str):
print(f"End of request")
print(f"Start: {start}, End {end}")
df = pd.DataFrame.from_records(self.histbars)
df["Date"] = df["Date"].str.split().str[:2].str.join(' ')
df["Date"] = pd.to_datetime(df["Date"])
df["Date"] = df["Date"] + timedelta(hours=self.hours_change)
df.set_index("Date", inplace=True)
df["Volume"] = pd.to_numeric(df["Volume"])
def vwap(df):
high = df.High.values
low = df.Low.values
vol = df.Volume.values
return df.assign(vwap=((high + low) / 2 * vol).cumsum() / vol.cumsum())
df = df.groupby(df.index.date, group_keys=False).apply(vwap)
print(df.tail(self.cnadles_plot))
print(df.dtypes)
apdict = mpf.make_addplot(df['vwap'])
mpf.plot(df, type="candle", volume=True,tight_layout=True,show_nontrading=True, addplot=apdict)
app = TestApp()
app.connect("127.0.0.1", 7496, 1000)
while True:
app.run()
time.sleep(20)
Firstly check out
https://interactivebrokers.github.io/tws-api/historical_limitations.html and
https://interactivebrokers.github.io/tws-api/historical_bars.html#hd_request
for the limitations of data requests and available bar sizes.
Depending on exactly what timeframes and quantity of data you require you may need to introduce delays into the code for rate limitations.
I never use python, the code works but is most likely low quality, it should get you started with IBApi.
Assuming you want 15sec bars...
import time
from ibapi.client import *
from ibapi.wrapper import *
from ibapi.contract import Contract
from datetime import datetime, timedelta
next_req_id = 1
bars_recieved = 0
req_date = datetime.now() + timedelta(days=-9)
req_end_date = datetime.now() + timedelta(days=-8)
mycontract = Contract()
mycontract.symbol = "TSLA"
mycontract.secType = "STK"
mycontract.exchange = "SMART"
mycontract.currency = "USD"
class TestApp(EClient, EWrapper):
def __init__(self):
EClient.__init__(self, self)
def nextValidId(self, orderId: int):
# We get here after connected and ready, but do not need orderId because this is not an order.
global next_req_id
# 14400 S is 4hours, the maximum duration allowed for the 15 sec bar size.
self.reqHistoricalData(next_req_id, mycontract, format(req_date, "%Y%m%d-%H:%M:%S"), "14400 S", "15 secs", "TRADES", 0, 1, 0, [])
#each request has its own ID
next_req_id += 1
def historicalData(self, reqId: int, bar: BarData):
global bars_recieved
bars_recieved += 1
#print(f"Recieved: {bar.date} {bar.open} {bar.high} {bar.low} {bar.close}")
def historicalDataEnd(self, reqId: int, start: str, end: str):
global next_req_id
global req_date
global bars_recieved
print(f"Recieved: {bars_recieved} bars from {start}, End {end}")
bars_recieved = 0
next_req_id += 1
# 4 hours - should match requested duration, in this case 14400 seconds
req_date += timedelta(hours=4)
# exit when all bars are recieved
if req_date > req_end_date:
app.disconnect()
time.sleep(2)
quit()
# A timed event would be better, but longer durations will need to delay for either soft or hard rate limit
time.sleep(2)
self.reqHistoricalData(next_req_id, mycontract, format(req_date, "%Y%m%d-%H:%M:%S"), "14400 S", "15 secs", "TRADES", 0, 1, 0, [])
app = TestApp()
app.connect("127.0.0.1", 7496, 1000)
app.run()
Pay particular attention to this (from 1st link)
Pacing Violations for Small Bars (30 secs or less)... occurs whenever one or more of the following restrictions is not observed:
Making identical historical data requests within 15 seconds.
Making six or more historical data requests for the same Contract, Exchange and Tick Type within two seconds.
Making more than 60 requests within any ten minute period.
At this time Historical Data Limitations for barSize = "1 mins" and
greater have been lifted. However, please use caution when requesting
large amounts of historical data or sending historical data requests
too frequently. Though IB has lifted the "hard" limit, we still
implement a "soft" slow to load-balance client requests vs. server
response. Requesting too much historical data can lead to throttling
and eventual disconnect of the API client. If a request requires more
than several minutes to return data, it would be best to cancel the
request using the IBApi.EClient.cancelHistoricalData function.
i am new to Interactive brokers API, wondering if there is any way to get quote of NIFTY50 index, as i can see examples of NIFTY FUTURES, etc but not on index price quote of NIFTY50 or BANKNIFTY,
need help with some example, thanks,
sample on on getting quote of NIFTY OPTIONS,
from locale import currency
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
from ibapi.ticktype import TickTypeEnum
from ibapi.common import *
import threading
import time
def get_close_quote(symbol, secType='OPT', exchange='NSE', currency='INR'):
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
def historicalData(self, reqId, bar):
print(f'Time: {bar.date} Close: {bar.close}')
def tickPrice(self, reqId, tickType, price, attrib):
if tickType == TickTypeEnum.LAST or tickType == TickTypeEnum.DELAYED_LAST:
self.last = price;
self.disconnect()
def run_loop():
app.run()
app = IBapi()
app.connect('127.0.0.1', 7496, 123)
#Start the socket in a thread
api_thread = threading.Thread(target=run_loop, daemon=True)
api_thread.start()
time.sleep(1) #Sleep interval to allow time for connection to server
#Create contract object
contract = Contract()
contract.localSymbol = symbol
contract.secType = secType
contract.exchange = exchange
contract.currency = currency
app.reqMktData(100, contract, "", False, False, None)
time.sleep(2) #sleep to allow enough time for data to be returned
app.disconnect()
return(app.last)
I am not familiar with Interactive brokers API but from digging on their website I did find those useful commands. You are welcome to check more than provided in this answer.
If you are trying to get a different type of security, as per the documentation, you would change the contract.secType to one of the following:
The security's type: STK - stock (or ETF) OPT - option FUT - future
IND - index FOP - futures option CASH - forex pair BAG - combo WAR -
warrant BOND- bond CMDTY- commodity NEWS- news FUND- mutual fund.
As I understood, you also want to get the actual price as part of the request for example last price is tick id = 4 as seen on the documentation of different types, and in general only have to request for specific ticks.
In reqMktData the genericTickList (3rd parameter) can be used to specify extra ticks to monitor, like demonstrated in documentation:
# Requesting RTVolume (Time & Sales) and shortable generic ticks
self.reqMktData(1004, ContractSamples.USStockAtSmart(), "233,236", False, False, [])
Where "233, 236" are the specified ticks to get from the website.
Also when checking the contract documentation, you can acquire the Strike price from the contract object.
I was able to piece together a script from IB's documentation/examples and forums on this site. I am getting the output I want for a single symbol, however, if I use a list of stocks, I cannot figure out a way to pass the ticker symbol through to the DF output file. My workaround was to create a dictionary that uses the list sequence (see below) however the output from IB's api changes slightly each time rendering the symbols mostly pointless. The list i am using below normally has 20+ names but may change, i cut it down to make it easier to view.
#Brian/and or other developers, if there is a way to create either a unique ID/sequence for each symbol call and stamp it to data that is brought back, i can then utilize a dictionary to apply the symbol. In the other forum, you passed in a line where n_id = n_id +1, if that can be applied and is linked to each specific call which is done in order of the list, then that could work?
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
import pandas as pd
import threading
import time
from datetime import timedelta
import datetime
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
self.data = [] #Initialize variable to store candle
def historicalData(self, reqId, bar):
#print(f'Time: {bar.date} Close: {bar.close} Volume: {bar.volume}',reqId)
self.data.append([bar.date, bar.close, bar.volume, reqId])
def run_loop():
app.run()
app = IBapi()
app.connect('127.0.0.1', 7496, 123)
#Start the socket in a thread
api_thread = threading.Thread(target=run_loop, daemon=True)
api_thread.start()
time.sleep(1) #Sleep interval to allow time for connection to server
symbols = ['SPY','MSFT','GOOG','AAPL','QQQ','IWM','TSLA']
for sym in symbols:
contract = Contract()
contract.symbol = str(sym)
contract.secType = "STK"
contract.exchange = "SMART"
contract.currency = "USD"
#contract.primaryExchange = "ISLAND"
app.reqHistoricalData(1, contract, "", "1 D", "10 mins", "ADJUSTED_LAST", 1, 2, False, [])
time.sleep(5) #sleep to allow enough time for data to be returned
df = pd.DataFrame(app.data, columns=['DateTime', 'ADJUSTED_LAST','Volume','reqId'])
df['DateTime'] = pd.to_datetime(df['DateTime'],unit='s') #,unit='s')
df['Count'] = df.groupby('DateTime').cumcount()+1
sym_dict = {1:'SPY',2:'MSFT',3:'GOOG',4:'AAPL',5:'QQQ',6:'IWM',7:'TSLA'}
df['Ticker'] = df['Count'].map(sym_dict)
print(df)
#edit, adding in #Brian's detail:
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
import pandas as pd
import time
from datetime import timedelta
import datetime
start = datetime.datetime.utcnow()
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
self.data = []
def error(self, reqId, errorCode, errorString):
print("Error. Id: " , reqId, " Code: " , errorCode , " Msg: " , errorString)
def historicalData(self, reqId, bar):
self.data.append([bar.date, bar.close, bar.volume, sym_dict[reqId]])
print("HistoricalData. ReqId:", sym_dict[reqId], "BarData.", bar)
# include this callback to track progress and maybe disconnectwhen all are finished
def historicalDataEnd(self, reqId: int, start: str, end: str):
print("finished", sym_dict[reqId])
def run_loop():
app.run()
app = IBapi()
app.connect('127.0.0.1', 7496, 123)
# you should wait for nextValidId instead of sleeping, what if it takes more than 1 second? #john: how do i do this?
time.sleep(5) #john: how do i do this? wait for nextValidId?
symbols = ['SPY','MSFT','GOOG','AAPL','QQQ','IWM','TSLA']
reqId = 1
sym_dict = {}
for sym in symbols:
contract = Contract()
contract.symbol = str(sym)
sym_dict[reqId] = sym
contract.secType = "STK"
contract.exchange = "SMART"
contract.currency = "USD"
#contract.primaryExchange = "ISLAND" # you may need this for msft
app.reqHistoricalData(reqId, contract, "", "1 D", "10 mins", "ADJUSTED_LAST", 1, 2, False, [])
reqId += 1
time.sleep(5)
df = pd.DataFrame(app.data, columns=['DateTime', 'ADJUSTED_LAST','Volume','sym'])
df['DateTime'] = pd.to_datetime(df['DateTime'],unit='s') #,unit='s')
df = df.set_index(['sym','DateTime']).sort_index()
print(df)
app.disconnect()
You just need to maintain a dict of reqId and symbol.
I'm not sure that one DataFrame is the best way to store your data but if you do then set a multi index. Decide how much data you want and how you're going to store it on disk and then decide on a data structure. I suggest csv for speed or sqlite for simplicity. Pandas can handle either.
I deleted your comments and added some of my own.
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
import pandas as pd
import threading
import time
from datetime import timedelta
import datetime
# I added this code to get fake data, works wtihout tws running
from ibapi.common import BarData
from random import random
start = datetime.datetime.utcnow()
def fake_data(reqId, ib):
last = reqId*10
for i in range(60, 0, -10):
bar = BarData();
bar.date = start - timedelta(minutes=i)
last += random() - 0.5
bar.close = last
bar.volume = reqId * 1000
ib.historicalData(reqId, bar)
ib.historicalDataEnd(reqId,"","")
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
self.data = []
#always include this for important messages, also turn on api logging in TWS/IBG
def error(self, reqId, errorCode, errorString):
print("Error. Id: " , reqId, " Code: " , errorCode , " Msg: " , errorString)
def historicalData(self, reqId, bar):
self.data.append([bar.date, bar.close, bar.volume, sym_dict[reqId]])
# include this callback to track progress and maybe disconnectwhen all are finished
def historicalDataEnd(self, reqId: int, start: str, end: str):
print("finished", sym_dict[reqId])
def run_loop():
app.run()
app = IBapi()
app.connect('127.0.0.1', 7496, 123)
# threading is needed only if you plan to interact after run is called
# this is a good way if you use a ui like jupyter
api_thread = threading.Thread(target=run_loop, daemon=True)
api_thread.start()
# you should wait for nextValidId instead of sleeping, what if it takes more than 1 second?
time.sleep(1)
symbols = ['SPY','MSFT','GOOG','AAPL','QQQ','IWM','TSLA']
reqId = 1
sym_dict = {}
for sym in symbols:
contract = Contract()
contract.symbol = str(sym)
sym_dict[reqId] = sym
contract.secType = "STK"
contract.exchange = "SMART"
contract.currency = "USD"
#contract.primaryExchange = "ISLAND" # you may need this for msft
#app.reqHistoricalData(reqId, contract, "", "1 D", "10 mins", "ADJUSTED_LAST", 1, 2, False, [])
fake_data(reqId, app)
reqId += 1
#now you need to sleep(10) to make sure you don't get a pacing error for too many requests
# don't sleep, use historicalDataEnd to know when finished
time.sleep(5)
df = pd.DataFrame(app.data, columns=['DateTime', 'ADJUSTED_LAST','Volume','sym'])
df['DateTime'] = pd.to_datetime(df['DateTime'],unit='s')
#make an index and sort
df = df.set_index(['sym','DateTime']).sort_index()
# now you can use the indexes
print(df.loc[("SPY","2021")])
#don't forget to disconnect somewhere or the clientId will still be in use
This is the code attempting to retrieve AAPL delayed stock price via Interactive Broker (IB) TWS.
However, none of data is retrieved.
As you can see, app.reqMarketDataType(3) has been called to set delayed data. (3 is delay)
I've logged into demo account in IB TWS and ensured "Enable ActiveX and Socket client" is selected.
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
import threading
import time
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
def tickPrice(self, reqId, tickType, price, attrib):
if tickType == 2 and reqId == 1:
print('The current ask price is: ', price)
def run_loop():
app.run()
app = IBapi()
app.connect('127.0.0.1', 7497, 123)
#Start the socket in a thread
api_thread = threading.Thread(target=run_loop, daemon=True)
api_thread.start()
time.sleep(1) #Sleep interval to allow time for connection to server
#Create contract object
apple_contract = Contract()
apple_contract.symbol = 'AAPL'
apple_contract.secType = 'STK'
apple_contract.exchange = 'SMART'
apple_contract.currency = 'USD'
#Request Market Data
app.reqMarketDataType(3)
app.reqMktData(1, apple_contract, '', False, False, [])
time.sleep(10) #Sleep interval to allow time for incoming price data
app.disconnect()
The code works ok. The problem is you don't print unless you get tickType 2(real time ask), delayed ask is tickType 67.
https://interactivebrokers.github.io/tws-api/tick_types.html
I want to use IBAPI to get historical stock data. My code does not work and returns nothings. Can anybody help me edit the code? Thanks
from ibapi import client
from ibapi import wrapper
import datetime
from ibapi.contract import Contract
from ibapi.common import BarData
# ! [socket_init]
class App(wrapper.EWrapper,client.EClient):
def __init__(self):
wrapper.EWrapper.__init__(self)
client.EClient.__init__(self, wrapper=self)
#Build a sample contract
contract = Contract();
contract.symbol = "9005.T";
contract.secType = "STK";
contract.currency = "JPY";
contract.exchange = "SMART";
app = App()
app.connect(host='localhost',port=7497, clientId=3)
print(app.isConnected())
queryTime = (datetime.datetime.today() - datetime.timedelta(days=180)).strftime("%Y%m%d %H:%M:%S")
print(app.reqHistoricalData(4102, contract, queryTime,"1 M", "1 day", "MIDPOINT", 1, 1, False, []))
queryTime = (datetime.datetime.today() - datetime.timedelta(days=180)).strftime("%Y%m%d %H:%M:%S")
print(app.historicalData(4102,BarData))
OUTPUT:
True
None
None
I tried to read the source code. But I found it is quite difficult to understand for me. Other posts showed the answer that appears not relevant to the newest api version.
Here's what I would do.
class App(wrapper.EWrapper,client.EClient): I would only subclass EClient if I wanted to override any of it's methods. I don't use python very much but it's like that in other languages.
Inside the class App you need to override the methods you are interested in like historicalData
After app.connect you must call app.run() to start it's message reader thread. Once that thread takes control it will block in your program so you must do program flow asynchronously.
I will put numbers in comments so you see the flow.
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
from ibapi.common import BarData
import datetime
class MyWrapper(EWrapper):
def nextValidId(self, orderId:int):
#4 first message received is this one
print("setting nextValidOrderId: %d", orderId)
self.nextValidOrderId = orderId
#5 start requests here
self.start()
def historicalData(self, reqId:int, bar: BarData):
#7 data is received for every bar
print("HistoricalData. ReqId:", reqId, "BarData.", bar)
def historicalDataEnd(self, reqId: int, start: str, end: str):
#8 data is finished
print("HistoricalDataEnd. ReqId:", reqId, "from", start, "to", end)
#9 this is the logical end of your program
app.disconnect()
print("finished")
def error(self, reqId, errorCode, errorString):
# these messages can come anytime.
print("Error. Id: " , reqId, " Code: " , errorCode , " Msg: " , errorString)
def start(self):
queryTime = (datetime.datetime.today() - datetime.timedelta(days=180)).strftime("%Y%m%d %H:%M:%S")
fx = Contract()
fx.secType = "CASH"
fx.symbol = "USD"
fx.currency = "JPY"
fx.exchange = "IDEALPRO"
#6 request data, using fx since I don't have Japanese data
app.reqHistoricalData(4102, fx, queryTime,"1 M", "1 day", "MIDPOINT", 1, 1, False, [])
app = EClient(MyWrapper()) #1 create wrapper subclass and pass it to EClient
app.connect("127.0.0.1", 7497, clientId=123) #2 connect to TWS/IBG
app.run() #3 start message thread