We Keep Coding

Backtrader 'File Not Found Error [ERRNO 2]'. Error with sample code from Backtrader Github page

I ran across this error when trying to run the backtrader sample setup code found on github. After doing some research I found that the error probably stems from the Yahoo Finance API being out of date or no longer compatible with the backtrader package. I plan on using online data feeds for the backtesting I want to do, so does anyone know how I can fix the yahoo data feed issue? Some online sources suggested digging into the source code. I already tried those suggestions with no avail.
from datetime import datetime
import backtrader as bt
class SmaCross(bt.SignalStrategy):
def __init__(self):
sma1, sma2 = bt.ind.SMA(period=10), bt.ind.SMA(period=30)
crossover = bt.ind.CrossOver(sma1, sma2)
self.signal_add(bt.SIGNAL_LONG, crossover)
cerebro = bt.Cerebro()
cerebro.addstrategy(SmaCross)
data0 = bt.feeds.YahooFinanceData(dataname='AAPL', fromdate=datetime(2011, 1, 1),
todate=datetime(2012, 12, 31))
cerebro.adddata(data0)
cerebro.run()
cerebro.plot()
Error Stack:
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
<ipython-input-2-4abfaa589128> in <module>
12 cerebro.adddata(data0)
13
---> 14 cerebro.run()
15 cerebro.plot()
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/cerebro.py in run(self, **kwargs)
1125 # let's skip process "spawning"
1126 for iterstrat in iterstrats:
-> 1127 runstrat = self.runstrategies(iterstrat)
1128 self.runstrats.append(runstrat)
1129 if self._dooptimize:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/cerebro.py in runstrategies(self, iterstrat, predata)
1208 if self._exactbars < 1: # datas can be full length
1209 data.extend(size=self.params.lookahead)
-> 1210 data._start()
1211 if self._dopreload:
1212 data.preload()
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feed.py in _start(self)
201
202 def _start(self):
--> 203 self.start()
204
205 if not self._started:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feeds/yahoo.py in start(self)
354
355 # Prepared a "path" file - CSV Parser can take over
--> 356 super(YahooFinanceData, self).start()
357
358
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feeds/yahoo.py in start(self)
92
93 def start(self):
---> 94 super(YahooFinanceCSVData, self).start()
95
96 if not self.params.reverse:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feed.py in start(self)
672 else:
673 # Let an exception propagate to let the caller know
--> 674 self.f = io.open(self.p.dataname, 'r')
675
676 if self.p.headers:
FileNotFoundError: [Errno 2] No such file or directory: 'AAPL'

This was an issue that popped up this week. We have fixed it on Backtrader2 here. Backtrader2 was created to keep urgent bug fixes up to date, since the original backtrader is a closed repo now.
If you wish, there is only one line change. You can adjust your local backtrader file to match this: https://github.com/backtrader2/backtrader/pull/67/files
In file backtrader/feeds/yahoo.py, you will find the following code at line 269:
crumb = None
sess = requests.Session()
Add the following code on the next line:
sess.headers['User-Agent'] = 'backtrader'

Wahoo TICKR X .fit file reading/parsing and analysis in Python

Not sure if I can post a question like this here so please redirect me if I'm in the wrong place.
I've bought a Wahoo TICKR X to monitor my heart rate during exercise. Also I would like to get more familiar with python so i decided I would like do do the analyses of my heart rate myself in python instead of in the wahoo app. I thought this would also give more freedom in the choice of visualization, testing etc.
I've recorded my heart rate for 5 minutes or so and exported the .fit file. However I can't even find a suitable library to read the .fit file. Can anyone recommend a library that works with .fit file from wahoo?
I'm using ubuntu, anaconda, python 3.7
import pyfits
# Load the FITS file into the program
hdulist = pyfits.open('/home/bradmin/Downloads/2020-03-26.fit')
# Load table data as tbdata
tbdata = hdulist[1].data
OSError Traceback (most recent call last)
<ipython-input-3-a970e2cd9dee> in <module>
2
3 # Load the FITS file into the program
----> 4 hdulist = pyfits.open('/home/bradmin/Downloads/2020-03-26.fit')
5
6 # Load table data as tbdata
~/anaconda3/lib/python3.7/site-packages/pyfits/hdu/hdulist.py in fitsopen(name, mode, memmap, save_backup, **kwargs)
122 raise ValueError('Empty filename: %s' % repr(name))
123
--> 124 return HDUList.fromfile(name, mode, memmap, save_backup, **kwargs)
125
126
~/anaconda3/lib/python3.7/site-packages/pyfits/hdu/hdulist.py in fromfile(cls, fileobj, mode, memmap, save_backup, **kwargs)
264
265 return cls._readfrom(fileobj=fileobj, mode=mode, memmap=memmap,
--> 266 save_backup=save_backup, **kwargs)
267
268 #classmethod
~/anaconda3/lib/python3.7/site-packages/pyfits/hdu/hdulist.py in _readfrom(cls, fileobj, data, mode, memmap, save_backup, **kwargs)
853 # raise and exception
854 if mode in ('readonly', 'denywrite') and len(hdulist) == 0:
--> 855 raise IOError('Empty or corrupt FITS file')
856
857 # initialize/reset attributes to be used in "update/append" mode
OSError: Empty or corrupt FITS file
link to the file: https://wetransfer.com/downloads/6d054a5d52899aefcb1bcd22bda92ba120200326161849/b9831a
EDIT
I've tried this now but i get an error:
import fitdecode
src_file = "/home/bradmin/Downloads/2020-03-26.fit"
with fitdecode.FitReader(src_file) as fit:
for frame in fit:
# The yielded frame object is of one of the following types:
# * fitdecode.FitHeader
# * fitdecode.FitDefinitionMessage
# * fitdecode.FitDataMessage
# * fitdecode.FitCRC
if isinstance(frame, fitdecode.FitDataMessage):
# Here, frame is a FitDataMessage object.
# A FitDataMessage object contains decoded values that
# are directly usable in your script logic.
print(frame.name)
file_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
developer_data_id
field_description
field_description
field_description
field_description
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-7-e8d95d3087dc> in <module>
2
3 with fitdecode.FitReader(src_file) as fit:
----> 4 for frame in fit:
5 # The yielded frame object is of one of the following types:
6 # * fitdecode.FitHeader
~/anaconda3/lib/python3.7/site-packages/fitdecode/reader.py in __iter__(self)
191
192 def __iter__(self):
--> 193 yield from self._read_next()
194
195 #property
~/anaconda3/lib/python3.7/site-packages/fitdecode/reader.py in _read_next(self)
298 assert self._header
299
--> 300 record = self._read_record()
301 if not record:
302 break
~/anaconda3/lib/python3.7/site-packages/fitdecode/reader.py in _read_record(self)
443 self._add_dev_data_id(message)
444 elif message.mesg_type.mesg_num == profile.MESG_NUM_FIELD_DESCRIPTION:
--> 445 self._add_dev_field_description(message)
446
447 return message
~/anaconda3/lib/python3.7/site-packages/fitdecode/reader.py in _add_dev_field_description(self, message)
780 base_type_id = message.get_field('fit_base_type_id').raw_value
781 field_name = message.get_field('field_name').raw_value
--> 782 units = message.get_field('units').raw_value
783
784 try:
~/anaconda3/lib/python3.7/site-packages/fitdecode/records.py in get_field(self, field_name_or_num, idx)
188 raise KeyError(
189 f'field "{field_name_or_num}" (idx #{idx}) not found in ' +
--> 190 f'message "{self.name}"')
191
192 def get_fields(self, field_name_or_num):
KeyError: 'field "units" (idx #0) not found in message "field_description"'

The format seems to be this FIT format. pyfits is for an entirely different format, it seems.
The article above refers to a gpsbabel tool, which you could use to convert the FIT file to something more interoperable and usable, e.g. GPX (an XML-based format that's easy to parse).
Or, of course, if you want a pure-Python solution, you can port the FIT format reading bits from gpsbabel to Python use the fitdecode library.

YQLResponseMalformedError: Response malformed on yahoo_finance?

yahoo_finance's get_historical function has been working fine for me until yesterday.
Now it just keeps giving me YQLResponseMalformedError every time I try to call get_historical. Any ideas? I literally did nothing new with the code.
from yahoo_finance import Share
yahoo = Share('YHOO')
yahoo.refresh()
yahoo.get_historical('2014-04-26', '2014-04-29')
---------------------------------------------------------------------------
YQLResponseMalformedError Traceback (most recent call last)
<ipython-input-8-fbc20f9b6d26> in <module>()
----> 1 yahoo.get_historical('2014-04-26', '2014-04-29')
/Users/anjian/anaconda/lib/python2.7/site-packages/yahoo_finance-1.4.0-py2.7.egg/yahoo_finance/__init__.pyc in get_historical(self, start_date, end_date)
340 try:
341 query = self._prepare_query(table='historicaldata', startDate=s, endDate=e)
--> 342 result = self._request(query)
343 if isinstance(result, dict):
344 result = [result]
/Users/anjian/anaconda/lib/python2.7/site-packages/yahoo_finance-1.4.0-py2.7.egg/yahoo_finance/__init__.pyc in _request(self, query)
123 raise YQLQueryError(response['error']['description'])
124 except KeyError:
--> 125 raise YQLResponseMalformedError()
126 else:
127 if self._is_error_in_results(results):
YQLResponseMalformedError: Response malformed.

Answered here: Python and Yahoo finance weird "YQLQueryError(response['error']['description'])" get_historical

AzureException: Incorrect padding. Just following Azure tutorial

I'm trying to learn how to use the Azure Table Storage service. I'm following this tutorial: https://learn.microsoft.com/en-us/azure/storage/storage-python-how-to-use-table-storage and I am simply copy pasting the code into a jupyter notebook. I've setup a storage account and am successfully using the blob storage. Also from a notebook.
Code from the tutorial:
from azure.storage.table import TableService, Entity
table_service = TableService(account_name='myaccount', account_key='mykey')
table_service.create_table('tasktable')
When I run the last line I get the following error and I'm not sure what I am doing wrong to cause it
---------------------------------------------------------------------------
Error Traceback (most recent call last)
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
205 _add_date_header(request)
--> 206 self.authentication.sign_request(request)
207
/usr/local/lib/python3.5/site-packages/azure/storage/_auth.py in sign_request(self, request)
96
---> 97 self._add_authorization_header(request, string_to_sign)
98
/usr/local/lib/python3.5/site-packages/azure/storage/_auth.py in _add_authorization_header(self, request, string_to_sign)
50 def _add_authorization_header(self, request, string_to_sign):
---> 51 signature = _sign_string(self.account_key, string_to_sign)
52 auth_string = 'SharedKey ' + self.account_name + ':' + signature
/usr/local/lib/python3.5/site-packages/azure/storage/_common_conversion.py in _sign_string(key, string_to_sign, key_is_base64)
87 if key_is_base64:
---> 88 key = _decode_base64_to_bytes(key)
89 else:
/usr/local/lib/python3.5/site-packages/azure/storage/_common_conversion.py in _decode_base64_to_bytes(data)
77 data = data.encode('utf-8')
---> 78 return base64.b64decode(data)
79
/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/base64.py in b64decode(s, altchars, validate)
89 raise binascii.Error('Non-base64 digit found')
---> 90 return binascii.a2b_base64(s)
91
Error: Incorrect padding
During handling of the above exception, another exception occurred:
AzureException Traceback (most recent call last)
<ipython-input-17-192b23ba629f> in <module>()
----> 1 table_service.create_table('tasktable')
/usr/local/lib/python3.5/site-packages/azure/storage/table/tableservice.py in create_table(self, table_name, fail_on_exist, timeout)
520 if not fail_on_exist:
521 try:
--> 522 self._perform_request(request)
523 return True
524 except AzureHttpError as ex:
/usr/local/lib/python3.5/site-packages/azure/storage/table/tableservice.py in _perform_request(self, request, parser, parser_args, operation_context)
1087 def _perform_request(self, request, parser=None, parser_args=None, operation_context=None):
1088 _update_storage_table_header(request)
-> 1089 return super(TableService, self)._perform_request(request, parser, parser_args, operation_context)
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
264 sleep(retry_interval)
265 else:
--> 266 raise ex
267 finally:
268 # If this is a location locked operation and the location is not set,
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
240 if sys.version_info >= (3,):
241 # Automatic chaining in Python 3 means we keep the trace
--> 242 raise AzureException(ex.args[0])
243 else:
244 # There isn't a good solution in 2 for keeping the stack trace
AzureException: Incorrect padding

As a summary, the issue was caused by the variable name for account key within some mistake. Accroding to the error information Error: Incorrect padding, as #Scovetta said, it seems to be not BASE64 encoding. Some changes for the key like missing the last = symbol or adding more = symbol will cause the error. And the length of correct account key of Azure Storage is 88.

Shameless necro after putting "incorrect padding azure" in my favorite search engine.
Turns out I was passing arguments like so : --account-key "$ACCOUNT_KEY", and Azure didn't understand the quotes.
Since it's supposed to be base64 encoded, all characters in there should be shell-safe, so if your input is OK there shouldn't be any problem putting it like that.

Python call script still failing

I've spent hours trying to run this script, but for some reason it simply cannot function properly. 'Sex' is now a global install, so it calls that fine, but something else seemingly fails. The thing that looks like it fails is the thing it's supposed to create, unfortunately. I even added a simple print statement after the def statement, and it wouldn't print, it just fails automatically. Any insight?
Code:
def objmask(inimgs, inwhts, thresh1='20.0', thresh2='2.0', tfdel=True,
xceng=3001., yceng=3001., outdir='.', tmpdir='tmp'):
print "c"
# initial detection of main galaxy with SExtractor for re-centering purposes
if outdir!='.':
if not os.path.exists(outdir):
os.makedirs(outdir)
print inimgs
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
for c in range(np.size(inimgs)):
print 'Creating Aperture Run:', c
subprocess.call(['sex',inimgs[c],'-c','/home/vidur/se_files/gccg.sex',
'-CATALOG_NAME','/home/vidur/se_files/tmp'+str(c)+'.cat',
'-PARAMETERS_NAME','/home/vidur/se_files/gccg_ell.param',
'-FILTER_NAME','/home/vidur/se_files/gccg.conv',
'-STARNNW_NAME','/home/vidur/se_files/gccg.nnw',
'-CHECKIMAGE_TYPE','APERTURES',
'-VERBOSE_TYPE','QUIET',
'-DETECT_THRESH',thresh1,
'-ANALYSIS_THRESH',thresh2,
'-WEIGHT_IMAGE',inwhts[c]],shell=True
)
Error:
---------------------------------------------------------------------------
IOError Traceback (most recent call last)
/home/vidur/se_files/<ipython-input-2-bbc58f9e134a> in <module>()
----> 1 fetch_swarp2.objmask(['sciPHOTOf105w0.fits'],['whtPHOTOf105w0.fits'])
/home/vidur/se_files/fetch_swarp2.pyc in objmask(inimgs, inwhts, thresh1, thresh2, tfdel, xceng, yceng, outdir, tmpdir)
116 secat=asciitable.read('./se_files/_tmp_seobj'+str(c)+'.cat',
117 names=['flux','ferr','xmin','ymin','xmax','ymax',
--> 118 'xc','yc','cxx','cyy','cxy'])
119 robj = np.sqrt((secat['xc']-xceng)**2.0+(secat['yc']-yceng)**2.0)
120 rmin = (robj==np.min(robj))
/usr/local/lib/python2.7/dist-packages/asciitable-0.8.0-py2.7.egg/asciitable/ui.pyc in read(table, numpy, guess, **kwargs)
129 guess = _GUESS
130 if guess:
--> 131 dat = _guess(table, new_kwargs)
132 else:
133 reader = get_reader(**new_kwargs)
/usr/local/lib/python2.7/dist-packages/asciitable-0.8.0-py2.7.egg/asciitable/ui.pyc in _guess(table, read_kwargs)
173 try:
174 reader = get_reader(**guess_kwargs)
--> 175 dat = reader.read(table)
176 # When guessing impose additional requirements on column names and number of cols
177 bads = [" ", ",", "|", "\t", "'", '"']
/usr/local/lib/python2.7/dist-packages/asciitable-0.8.0-py2.7.egg/asciitable/core.pyc in read(self, table)
839 self.header.data = self.data
840
--> 841 self.lines = self.inputter.get_lines(table)
842 self.data.get_data_lines(self.lines)
843 self.header.get_cols(self.lines)
/usr/local/lib/python2.7/dist-packages/asciitable-0.8.0-py2.7.egg/asciitable/core.pyc in get_lines(self, table)
155 table = table.read()
156 elif '\n' not in table and '\r' not in table + '':
--> 157 table = open(table, 'r').read()
158 lines = table.splitlines()
159 except TypeError:
IOError: [Errno 2] No such file or directory: './se_files/_tmp_seobj0.cat'
The most relevant portion of the error seems to be the end.
P.S. I run Ubuntu 12.04 32-bit