We Keep Coding

Python lzma unable to load joblib

I have a scikit learn pipeline that I serialize using:
with lzma.open('outputs/baseModel_LR.joblib',"wb") as f:
dill.dump(pipeline, f)
When I try to open the file and load the pipeline using:
with lzma.open('outputs/baseModel_LR.joblib',"rb") as f:
model = dill.load(f)
it gives error:
---------------------------------------------------------------------------
EOFError Traceback (most recent call last)
somePath/notebooks/test.ipynb Cell 5 in <cell line: 1>()
1 with lzma.open('outputs/baseModel_LR.joblib',"rb") as f:
----> 2 model = dill.load(f)
3 model
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dill/_dill.py:373, in load(file, ignore, **kwds)
367 def load(file, ignore=None, **kwds):
368 """
369 Unpickle an object from a file.
370
371 See :func:`loads` for keyword arguments.
372 """
--> 373 return Unpickler(file, ignore=ignore, **kwds).load()
File /anaconda/envs/azureml_py38/lib/python3.8/site-packages/dill/_dill.py:646, in Unpickler.load(self)
645 def load(self): #NOTE: if settings change, need to update attributes
--> 646 obj = StockUnpickler.load(self)
647 if type(obj).__module__ == getattr(_main_module, '__name__', '__main__'):
648 if not self._ignore:
649 # point obj class to main
File /anaconda/envs/azureml_py38/lib/python3.8/lzma.py:200, in LZMAFile.read(self, size)
194 """Read up to size uncompressed bytes from the file.
...
100 "end-of-stream marker was reached")
101 else:
102 rawblock = b""
**EOFError: Compressed file ended before the end-of-stream marker was reached**
Has anyone faced this problem and solved it? I use lzma because otherwise the joblib size is 27GB and with lzma its just 20MB

Upload large file (>100 MB) directly to github with pygithub

I am using pyGitHub to upload files to my repo, however some of the files are so large that the server connection times out. My code to upload/overwrite a file from a folder is:
def commit(folder):
foldername = folder.split("/")[-1]
onlyfiles = [f for f in listdir(folder) if isfile(join(folder, f))]
repo = g.get_repo(user.login+"/My-repo")
all_files = []
contents = repo.get_contents("")
while contents:
file_content = contents.pop(0)
if file_content.type == "dir":
contents.extend(repo.get_contents(file_content.path))
else:
file = file_content
all_files.append(str(file).replace('ContentFile(path="','').replace('")',''))
body = '''
Line 1: Message
Line 2: Sample Text
Line 3: yet another line
'''
for i in onlyfiles:
print(i)
input_file = open(folder + "/" + i, "rb")
data = input_file.read()
input_file.close()
if not(f"{foldername}/{i}" in all_files):
repo.create_file(f"{foldername}/{i}", "Created building data", data)
else:
file = repo.get_contents(f"{foldername}/{i}")
repo.update_file(file.path, "Updated information", data, file.sha)
This code works for files <25mb, but for larger ones I get the error:
---------------------------------------------------------------------------
GithubException Traceback (most recent call last)
<ipython-input-9-7d41473c81a0> in <module>()
79
80
---> 81 commit(str("/content/"+dirname))
3 frames
<ipython-input-9-7d41473c81a0> in commit(folder)
72 input_file.close()
73 if not(f"{foldername}/{i}" in all_files):
---> 74 repo.create_file(f"{foldername}/{i}", "Created building data", data)
75 else:
76 file = repo.get_contents(f"{foldername}/{i}")
/usr/local/lib/python3.7/dist-packages/github/Repository.py in create_file(self, path, message, content, branch, committer, author)
2091 "PUT",
2092 f"{self.url}/contents/{urllib.parse.quote(path)}",
-> 2093 input=put_parameters,
2094 )
2095
/usr/local/lib/python3.7/dist-packages/github/Requester.py in requestJsonAndCheck(self, verb, url, parameters, headers, input)
353 return self.__check(
354 *self.requestJson(
--> 355 verb, url, parameters, headers, input, self.__customConnection(url)
356 )
357 )
/usr/local/lib/python3.7/dist-packages/github/Requester.py in __check(self, status, responseHeaders, output)
376 output = self.__structuredFromJson(output)
377 if status >= 400:
--> 378 raise self.__createException(status, responseHeaders, output)
379 return responseHeaders, output
380
GithubException: 502 {"message": "Server Error"}
I am aware that the file upload limit for github is 25MB, but apparently files up to 100MB can be uploaded via the command line. How would I upload files larger than this to GitHub using pyGitHub? The file is zipped, so it really is as small as it can be, but is still ~150MB. Is this doable? If not, is there a way to reference a larger file in github which I can upload elsewhere? I am using Google Colab in case anyone is wondering.

Backtrader 'File Not Found Error [ERRNO 2]'. Error with sample code from Backtrader Github page

I ran across this error when trying to run the backtrader sample setup code found on github. After doing some research I found that the error probably stems from the Yahoo Finance API being out of date or no longer compatible with the backtrader package. I plan on using online data feeds for the backtesting I want to do, so does anyone know how I can fix the yahoo data feed issue? Some online sources suggested digging into the source code. I already tried those suggestions with no avail.
from datetime import datetime
import backtrader as bt
class SmaCross(bt.SignalStrategy):
def __init__(self):
sma1, sma2 = bt.ind.SMA(period=10), bt.ind.SMA(period=30)
crossover = bt.ind.CrossOver(sma1, sma2)
self.signal_add(bt.SIGNAL_LONG, crossover)
cerebro = bt.Cerebro()
cerebro.addstrategy(SmaCross)
data0 = bt.feeds.YahooFinanceData(dataname='AAPL', fromdate=datetime(2011, 1, 1),
todate=datetime(2012, 12, 31))
cerebro.adddata(data0)
cerebro.run()
cerebro.plot()
Error Stack:
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
<ipython-input-2-4abfaa589128> in <module>
12 cerebro.adddata(data0)
13
---> 14 cerebro.run()
15 cerebro.plot()
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/cerebro.py in run(self, **kwargs)
1125 # let's skip process "spawning"
1126 for iterstrat in iterstrats:
-> 1127 runstrat = self.runstrategies(iterstrat)
1128 self.runstrats.append(runstrat)
1129 if self._dooptimize:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/cerebro.py in runstrategies(self, iterstrat, predata)
1208 if self._exactbars < 1: # datas can be full length
1209 data.extend(size=self.params.lookahead)
-> 1210 data._start()
1211 if self._dopreload:
1212 data.preload()
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feed.py in _start(self)
201
202 def _start(self):
--> 203 self.start()
204
205 if not self._started:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feeds/yahoo.py in start(self)
354
355 # Prepared a "path" file - CSV Parser can take over
--> 356 super(YahooFinanceData, self).start()
357
358
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feeds/yahoo.py in start(self)
92
93 def start(self):
---> 94 super(YahooFinanceCSVData, self).start()
95
96 if not self.params.reverse:
~/opt/anaconda3/lib/python3.8/site-packages/backtrader/feed.py in start(self)
672 else:
673 # Let an exception propagate to let the caller know
--> 674 self.f = io.open(self.p.dataname, 'r')
675
676 if self.p.headers:
FileNotFoundError: [Errno 2] No such file or directory: 'AAPL'

This was an issue that popped up this week. We have fixed it on Backtrader2 here. Backtrader2 was created to keep urgent bug fixes up to date, since the original backtrader is a closed repo now.
If you wish, there is only one line change. You can adjust your local backtrader file to match this: https://github.com/backtrader2/backtrader/pull/67/files
In file backtrader/feeds/yahoo.py, you will find the following code at line 269:
crumb = None
sess = requests.Session()
Add the following code on the next line:
sess.headers['User-Agent'] = 'backtrader'

Azure CLI ML Throws TypeError: __init__() takes 2 positional arguments but 3 were given

I'm attempting to follow this tutorial and am getting the following error. I'm working in jupyter notebook with python 3. I am trying to build a recommendation engine using Azure tools and the Microsoft doc I attached.
TypeError: init() takes 2 positional arguments but 3 were given
I have attached an image of code from jupyter notebook to demonstrate proper formatting
I've tried solutions to roll back my Azure Python SDK using Pip. I run pip install --upgrade azureml-sdk and all looks good. Thanks very much for any and all help!
The code appears below:
print(workspace_name)
ws = Workspace.create(
name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
location=location,
exist_ok=True
)
The full error appears below:
'''
TypeError Traceback (most recent call last)
<ipython-input-11-25e04e55f419> in <module>
5 resource_group=resource_group,
6 location=location,
----> 7 exist_ok=True
8 )
~\.conda\envs\reco_pyspark\lib\site-packages\azureml\core\workspace.py in create(name, auth, subscription_id, resource_group, location, create_resource_group, sku, friendly_name, storage_account, key_vault, app_insights, container_registry, cmk_keyvault, resource_cmk_uri, hbi_workspace, default_cpu_compute_target, default_gpu_compute_target, private_endpoint_config, private_endpoint_auto_approval, exist_ok, show_output)
437
438 if location:
--> 439 available_locations = _available_workspace_locations(subscription_id, auth)
440 available_locations = [x.lower().replace(' ', '') for x in available_locations]
441 location = location.lower().replace(' ', '')
~\.conda\envs\reco_pyspark\lib\site-packages\azureml\core\workspace.py in _available_workspace_locations(subscription_id, auth)
1556 if not auth:
1557 auth = InteractiveLoginAuthentication()
-> 1558 return _commands.available_workspace_locations(auth, subscription_id)
~\.conda\envs\reco_pyspark\lib\site-packages\azureml\_project\_commands.py in available_workspace_locations(auth, subscription_id)
334 :rtype: list[str]
335 """
--> 336 response = auth._get_service_client(ResourceManagementClient, subscription_id).providers.get(
337 "Microsoft.MachineLearningServices")
338 for resource_type in response.resource_types:
~\.conda\envs\reco_pyspark\lib\site-packages\azureml\core\authentication.py in _get_service_client(self, client_class, subscription_id, subscription_bound, base_url)
150 return _get_service_client_using_arm_token(self, client_class, subscription_id,
151 subscription_bound=subscription_bound,
--> 152 base_url=base_url)
153
154 def signed_session(self, session=None):
~\.conda\envs\reco_pyspark\lib\site-packages\azureml\core\authentication.py in _get_service_client_using_arm_token(auth, client_class, subscription_id, subscription_bound, base_url)
1620 else:
1621 # converting subscription_id, which is string, to string because of weird python 2.7 errors.
-> 1622 client = client_class(adal_auth_object, str(subscription_id), base_url=base_url)
1623 return client
1624
TypeError: __init__() takes 2 positional arguments but 3 were given
'''

Have you tried using the Workspace.from_config() and the config.json method? Check out this doc page on how the set up your environment.

AzureException: Incorrect padding. Just following Azure tutorial

I'm trying to learn how to use the Azure Table Storage service. I'm following this tutorial: https://learn.microsoft.com/en-us/azure/storage/storage-python-how-to-use-table-storage and I am simply copy pasting the code into a jupyter notebook. I've setup a storage account and am successfully using the blob storage. Also from a notebook.
Code from the tutorial:
from azure.storage.table import TableService, Entity
table_service = TableService(account_name='myaccount', account_key='mykey')
table_service.create_table('tasktable')
When I run the last line I get the following error and I'm not sure what I am doing wrong to cause it
---------------------------------------------------------------------------
Error Traceback (most recent call last)
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
205 _add_date_header(request)
--> 206 self.authentication.sign_request(request)
207
/usr/local/lib/python3.5/site-packages/azure/storage/_auth.py in sign_request(self, request)
96
---> 97 self._add_authorization_header(request, string_to_sign)
98
/usr/local/lib/python3.5/site-packages/azure/storage/_auth.py in _add_authorization_header(self, request, string_to_sign)
50 def _add_authorization_header(self, request, string_to_sign):
---> 51 signature = _sign_string(self.account_key, string_to_sign)
52 auth_string = 'SharedKey ' + self.account_name + ':' + signature
/usr/local/lib/python3.5/site-packages/azure/storage/_common_conversion.py in _sign_string(key, string_to_sign, key_is_base64)
87 if key_is_base64:
---> 88 key = _decode_base64_to_bytes(key)
89 else:
/usr/local/lib/python3.5/site-packages/azure/storage/_common_conversion.py in _decode_base64_to_bytes(data)
77 data = data.encode('utf-8')
---> 78 return base64.b64decode(data)
79
/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/base64.py in b64decode(s, altchars, validate)
89 raise binascii.Error('Non-base64 digit found')
---> 90 return binascii.a2b_base64(s)
91
Error: Incorrect padding
During handling of the above exception, another exception occurred:
AzureException Traceback (most recent call last)
<ipython-input-17-192b23ba629f> in <module>()
----> 1 table_service.create_table('tasktable')
/usr/local/lib/python3.5/site-packages/azure/storage/table/tableservice.py in create_table(self, table_name, fail_on_exist, timeout)
520 if not fail_on_exist:
521 try:
--> 522 self._perform_request(request)
523 return True
524 except AzureHttpError as ex:
/usr/local/lib/python3.5/site-packages/azure/storage/table/tableservice.py in _perform_request(self, request, parser, parser_args, operation_context)
1087 def _perform_request(self, request, parser=None, parser_args=None, operation_context=None):
1088 _update_storage_table_header(request)
-> 1089 return super(TableService, self)._perform_request(request, parser, parser_args, operation_context)
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
264 sleep(retry_interval)
265 else:
--> 266 raise ex
267 finally:
268 # If this is a location locked operation and the location is not set,
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
240 if sys.version_info >= (3,):
241 # Automatic chaining in Python 3 means we keep the trace
--> 242 raise AzureException(ex.args[0])
243 else:
244 # There isn't a good solution in 2 for keeping the stack trace
AzureException: Incorrect padding

As a summary, the issue was caused by the variable name for account key within some mistake. Accroding to the error information Error: Incorrect padding, as #Scovetta said, it seems to be not BASE64 encoding. Some changes for the key like missing the last = symbol or adding more = symbol will cause the error. And the length of correct account key of Azure Storage is 88.

Shameless necro after putting "incorrect padding azure" in my favorite search engine.
Turns out I was passing arguments like so : --account-key "$ACCOUNT_KEY", and Azure didn't understand the quotes.
Since it's supposed to be base64 encoded, all characters in there should be shell-safe, so if your input is OK there shouldn't be any problem putting it like that.