We Keep Coding

How to connect to HDFS Cluster using snakebite-py3

I am trying to connect to an HDFS Cluster using python code, library(snakebite-py3) and I see that when I set use_sasl to True I am getting the following error:
Code Snippet:
from snakebite.client import Client
client = Client(host='hostname', port=8020,
effective_user='user', use_sasl=True)
for x in client.ls(['/']):
print(x,"\n")
Error:
---------------------------------------------------------------------------
GSSError Traceback (most recent call last)
<ipython-input-21-62c8b8df16ea> in <module>
2 from snakebite.client import Client
3
----> 4 client = Client(host='hostname',port=8020, effective_user='user', use_sasl=True)
5
6 for x in client.ls(['/test_abha']): print(x,"\n")
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\client.py in __init__(self, host, port, hadoop_version, use_trash, effective_user, use_sasl, hdfs_namenode_principal, sock_connect_timeout, sock_request_timeout, use_datanode_hostname)
126 self.hdfs_namenode_principal = hdfs_namenode_principal
127 self.service_stub_class = client_proto.ClientNamenodeProtocol_Stub
--> 128 self.service = RpcService(self.service_stub_class, self.port, self.host, hadoop_version,
129 effective_user,self.use_sasl, self.hdfs_namenode_principal,
130 sock_connect_timeout, sock_request_timeout)
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\service.py in __init__(self, service_stub_class, port, host, hadoop_version, effective_user, use_sasl, hdfs_namenode_principal, sock_connect_timeout, sock_request_timeout)
30
31 # Setup the RPC channel
---> 32 self.channel = SocketRpcChannel(host=self.host, port=self.port, version=hadoop_version,
33 effective_user=effective_user, use_sasl=use_sasl,
34 hdfs_namenode_principal=hdfs_namenode_principal,
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\channel.py in __init__(self, host, port, version, effective_user, use_sasl, hdfs_namenode_principal, sock_connect_timeout, sock_request_timeout)
193 raise FatalException("Kerberos libs not found. Please install snakebite using 'pip install snakebite[kerberos]'")
194
--> 195 kerberos = Kerberos()
196 self.effective_user = effective_user or kerberos.user_principal()
197 else:
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\kerberos.py in __init__(self)
41 class Kerberos:
42 def __init__(self):
---> 43 self.credentials = gssapi.Credentials(usage='initiate')
44
45 def user_principal(self):
C:\ProgramData\Anaconda3\lib\site-packages\gssapi\creds.py in __new__(cls, base, token, name, lifetime, mechs, usage, store)
61 base_creds = rcred_imp_exp.import_cred(token)
62 else:
---> 63 res = cls.acquire(name, lifetime, mechs, usage,
64 store=store)
65 base_creds = res.creds
C:\ProgramData\Anaconda3\lib\site-packages\gssapi\creds.py in acquire(cls, name, lifetime, mechs, usage, store)
134
135 if store is None:
--> 136 res = rcreds.acquire_cred(name, lifetime,
137 mechs, usage)
138 else:
gssapi/raw/creds.pyx in gssapi.raw.creds.acquire_cred()
GSSError: Major (851968): Unspecified GSS failure. Minor code may provide more information, Minor (39756044): Credential cache is empty
Please kindly suggest, thank you.

How to onnect to a LDAP server using Python (Version 3.8.8)

I need to connect to a LDAP server using Python (Version 3.8.8) and I've already tried to replicate some examples such as those showed in the link bellow but none of them are working for me. I either get (TypeError: iter() returned non-iterator of type 'NoneType') or (PyAsn1Error: Attempted "__iter__" operation on ASN.1 schema object). It seems like ldap3 and python-ldap used in the examples were not updated to work with python 3.8.8. I would be very pleased if anyone could give me a real example of another existing library to connect a LPDAP server or help me with this issue.
(1) https://sixfeetup.com/blog/new-ldap3-python-ldap-library
(2) https://ldap3.readthedocs.io/en/latest/tutorial_intro.html
(3) https://stackoverflow.com/questions/58907026/ldap3-bind-failed-when-cn-and-displayname-are-different
my test error for the example in link 3:
from ldap3 import Server, Connection
server = Server('ipa.demo1.freeipa.org')
conn = Connection(server)
conn.bind()
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
ipython-input-4-470896b147da> in <module>
2 server = Server('ipa.demo1.freeipa.org')
3 conn = Connection(server)
----> 4 conn.bind()
C:\ProgramData\Anaconda3\lib\site-packages\ldap3\core\connection.py in bind(self, read_server_info, controls)
418 if log_enabled(PROTOCOL):
419 log(PROTOCOL, 'anonymous BIND request <%s> sent via <%s>', bind_request_to_dict(request), self)
--> 420 response = self.post_send_single_response(self.send('bindRequest', request, controls))
421 elif self.authentication == SIMPLE:
422 if log_enabled(PROTOCOL):
C:\ProgramData\Anaconda3\lib\site-packages\ldap3\strategy\sync.py in post_send_single_response(self, message_id)
120 Returns the result message or None
121 """
--> 122 responses, result = self.get_response(message_id)
123 self.connection.result = result
124 if result['type'] == 'intermediateResponse': # checks that all responses are intermediates (there should be only one)
C:\ProgramData\Anaconda3\lib\site-packages\ldap3\strategy\base.py in get_response(self, message_id, timeout)
296 if self._outstanding and message_id in self._outstanding:
297 while timeout >= 0: # waiting for completed message to appear in responses
--> 298 responses = self._get_response(message_id)
299 if not responses:
300 sleep(RESPONSE_SLEEPTIME)
C:\ProgramData\Anaconda3\lib\site-packages\ldap3\strategy\sync.py in _get_response(self, message_id)
166 log(EXTENDED, 'ldap message received via <%s>:%s', self.connection, format_ldap_message(ldap_resp, '<<'))
167 if int(ldap_resp['messageID']) == message_id:
--> 168 dict_response = self.decode_response(ldap_resp)
169 ldap_responses.append(dict_response)
170 if dict_response['type'] not in ['searchResEntry', 'searchResRef', 'intermediateResponse']:
C:\ProgramData\Anaconda3\lib\site-packages\ldap3\strategy\base.py in decode_response(self, ldap_message)
401 if message_type == 'bindResponse':
402 if not bytes(component['matchedDN']).startswith(b'NTLM'): # patch for microsoft ntlm authentication
--> 403 result = bind_response_to_dict(component)
404 else:
405 result = sicily_bind_response_to_dict(component)
C:\ProgramData\Anaconda3\lib\site-packages\ldap3\operation\bind.py in bind_response_to_dict(response)
116 'dn': str(response['matchedDN']),
117 'message': str(response['diagnosticMessage']),
--> 118 'referrals': referrals_to_list(response['referral']),
119 'saslCreds': bytes(response['serverSaslCreds']) if response['serverSaslCreds'] is not None else None}
120
C:\ProgramData\Anaconda3\lib\site-packages\ldap3\protocol\convert.py in referrals_to_list(referrals)
42
43 def referrals_to_list(referrals):
---> 44 return [str(referral) for referral in referrals if referral] if referrals else None
45
46
C:\ProgramData\Anaconda3\lib\site-packages\pyasn1\type\base.py in __bool__(self)
572 else:
573 def __bool__(self):
--> 574 return bool(self.components)
575
576 #property
C:\ProgramData\Anaconda3\lib\site-packages\pyasn1\type\univ.py in components(self)
1958 def components(self):
1959 return [self._componentValues[idx]
-> 1960 for idx in sorted(self._componentValues)]
1961
1962 def clear(self):
TypeError: iter() returned non-iterator of type 'NoneType'

It was a problem of library version... I ran the "pip install ldap3 upgrade" command and it worked fine

pywren and twint - Tweet download

The following code takes a username and scrapes their twitter history from a given date
import pandas as pd
import twint
import pywren
def scrape_user(username):
c = twint.Config()
c.Username = username
c.Lang = 'es'
c.Since = '2021-04-28'
c.Hide_output = True
c.Pandas = True
twint.run.Search(c)
return twint.storage.panda.Tweets_df
When I run the function, I get the intended result i.e., a Pandas dataframe e.g., scrape_user("DeLaCalleHum"). However, when I use pywren (on even a single username)
pwex = pywren.default_executor()
futures = pwex.map(scrape_user, "DeLaCalleHum")
tweet_list = pywren.get_all_results(futures)
I get this error.
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
<ipython-input-31-15f9e00ead75> in <module>
----> 1 wc_list = pywren.get_all_results(futures)
~/macs30123/lib/python3.7/site-packages/pywren/wren.py in get_all_results(fs)
117 """
118 wait(fs, return_when=ALL_COMPLETED)
--> 119 return [f.result() for f in fs]
~/macs30123/lib/python3.7/site-packages/pywren/wren.py in <listcomp>(.0)
117 """
118 wait(fs, return_when=ALL_COMPLETED)
--> 119 return [f.result() for f in fs]
~/macs30123/lib/python3.7/site-packages/pywren/future.py in result(self, timeout, check_only, throw_except, storage_handler)
146 if self._state == JobState.error:
147 if throw_except:
--> 148 raise self._exception
149 else:
150 return None
OSError: [Errno 28] No space left on device
What am I doing wrong? I would appreciate any help.

After some time I found the answer. I can automatically parallelize such function calls in PyWren as long as I add the ComprehendFullAccess policy to my pywren_exec_role_1 role in IAM

AzureException: Incorrect padding. Just following Azure tutorial

I'm trying to learn how to use the Azure Table Storage service. I'm following this tutorial: https://learn.microsoft.com/en-us/azure/storage/storage-python-how-to-use-table-storage and I am simply copy pasting the code into a jupyter notebook. I've setup a storage account and am successfully using the blob storage. Also from a notebook.
Code from the tutorial:
from azure.storage.table import TableService, Entity
table_service = TableService(account_name='myaccount', account_key='mykey')
table_service.create_table('tasktable')
When I run the last line I get the following error and I'm not sure what I am doing wrong to cause it
---------------------------------------------------------------------------
Error Traceback (most recent call last)
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
205 _add_date_header(request)
--> 206 self.authentication.sign_request(request)
207
/usr/local/lib/python3.5/site-packages/azure/storage/_auth.py in sign_request(self, request)
96
---> 97 self._add_authorization_header(request, string_to_sign)
98
/usr/local/lib/python3.5/site-packages/azure/storage/_auth.py in _add_authorization_header(self, request, string_to_sign)
50 def _add_authorization_header(self, request, string_to_sign):
---> 51 signature = _sign_string(self.account_key, string_to_sign)
52 auth_string = 'SharedKey ' + self.account_name + ':' + signature
/usr/local/lib/python3.5/site-packages/azure/storage/_common_conversion.py in _sign_string(key, string_to_sign, key_is_base64)
87 if key_is_base64:
---> 88 key = _decode_base64_to_bytes(key)
89 else:
/usr/local/lib/python3.5/site-packages/azure/storage/_common_conversion.py in _decode_base64_to_bytes(data)
77 data = data.encode('utf-8')
---> 78 return base64.b64decode(data)
79
/usr/local/Cellar/python3/3.5.1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/base64.py in b64decode(s, altchars, validate)
89 raise binascii.Error('Non-base64 digit found')
---> 90 return binascii.a2b_base64(s)
91
Error: Incorrect padding
During handling of the above exception, another exception occurred:
AzureException Traceback (most recent call last)
<ipython-input-17-192b23ba629f> in <module>()
----> 1 table_service.create_table('tasktable')
/usr/local/lib/python3.5/site-packages/azure/storage/table/tableservice.py in create_table(self, table_name, fail_on_exist, timeout)
520 if not fail_on_exist:
521 try:
--> 522 self._perform_request(request)
523 return True
524 except AzureHttpError as ex:
/usr/local/lib/python3.5/site-packages/azure/storage/table/tableservice.py in _perform_request(self, request, parser, parser_args, operation_context)
1087 def _perform_request(self, request, parser=None, parser_args=None, operation_context=None):
1088 _update_storage_table_header(request)
-> 1089 return super(TableService, self)._perform_request(request, parser, parser_args, operation_context)
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
264 sleep(retry_interval)
265 else:
--> 266 raise ex
267 finally:
268 # If this is a location locked operation and the location is not set,
/usr/local/lib/python3.5/site-packages/azure/storage/storageclient.py in _perform_request(self, request, parser, parser_args, operation_context)
240 if sys.version_info >= (3,):
241 # Automatic chaining in Python 3 means we keep the trace
--> 242 raise AzureException(ex.args[0])
243 else:
244 # There isn't a good solution in 2 for keeping the stack trace
AzureException: Incorrect padding

As a summary, the issue was caused by the variable name for account key within some mistake. Accroding to the error information Error: Incorrect padding, as #Scovetta said, it seems to be not BASE64 encoding. Some changes for the key like missing the last = symbol or adding more = symbol will cause the error. And the length of correct account key of Azure Storage is 88.

Shameless necro after putting "incorrect padding azure" in my favorite search engine.
Turns out I was passing arguments like so : --account-key "$ACCOUNT_KEY", and Azure didn't understand the quotes.
Since it's supposed to be base64 encoded, all characters in there should be shell-safe, so if your input is OK there shouldn't be any problem putting it like that.

pysd library ParseError

I'm using a library called pysd to translate vensim files to Python, but when I try to do it (library functions) I get a parse error but don't understand what it means.
This is my log.
ParseError Traceback (most recent call last)
<ipython-input-1-9b0f6b9bac1f> in <module>()
1 get_ipython().magic(u'pylab inline')
2 import pysd
----> 3 model = pysd.read_vensim('201520_1A_Volare_Ev.Tecnica.itmx')
/Library/Python/2.7/site-packages/pysd/pysd.pyc in read_vensim(mdl_file)
45 """
46 from .vensim2py import translate_vensim
---> 47 py_model_file = translate_vensim(mdl_file)
48 model = PySD(py_model_file)
49 model.mdl_file = mdl_file
/Library/Python/2.7/site-packages/pysd/vensim2py.pyc in translate_vensim(mdl_file)
651 for section in file_sections:
652 if section['name'] == 'main':
--> 653 model_elements += get_model_elements(section['string'])
654
655 # extract equation components
/Library/Python/2.7/site-packages/pysd/vensim2py.pyc in get_model_elements(model_str)
158 """
159 parser = parsimonious.Grammar(model_structure_grammar)
--> 160 tree = parser.parse(model_str)
161
162 class ModelParser(parsimonious.NodeVisitor):
/Library/Python/2.7/site-packages/parsimonious/grammar.pyc in parse(self, text, pos)
121 """
122 self._check_default_rule()
--> 123 return self.default_rule.parse(text, pos=pos)
124
125 def match(self, text, pos=0):
/Library/Python/2.7/site-packages/parsimonious/expressions.pyc in parse(self, text, pos)
108
109 """
--> 110 node = self.match(text, pos=pos)
111 if node.end < len(text):
112 raise IncompleteParseError(text, node.end, self)
/Library/Python/2.7/site-packages/parsimonious/expressions.pyc in match(self, text, pos)
125 node = self.match_core(text, pos, {}, error)
126 if node is None:
--> 127 raise error
128 return node
129
ParseError: Rule 'escape_group' didn't match at '' (line 1, column 20243).

.itmx is an iThink extension, which unfortunately PySD doesn't support (yet). In the future, we'll work out a conversion pathway that lets you bring these in.