I am trying to use https://github.com/dowjones/dj-dna-streams-python/tree/master/dnaStreaming . It's a package to receive news streams from Dow Jones. When I try to "listen" to the streams I receive the following error:
---------------------------------------------------------------------------
ImportError Traceback (most recent call last)
<ipython-input-36-372f4305a9e1> in <module>()
1 while True:
----> 2 listener.listen(callback, maximum_messages=4, subscription_id=subscription_id)
~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dj_dna_streaming_python-1.0.10-py3.5.egg\dnaStreaming\listener.py in listen(self, on_message_callback, maximum_messages, subscription_id)
21 def listen(self, on_message_callback, maximum_messages=DEFAULT_UNLIMITED_MESSAGES, subscription_id=None):
22 limit_pull_calls = not (maximum_messages == self.DEFAULT_UNLIMITED_MESSAGES)
---> 23 pubsub_client = pubsub_service.get_client(self.config)
24
25 subscription_id = subscription_id if subscription_id is not None else self.config.subscription()
~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dj_dna_streaming_python-1.0.10-py3.5.egg\dnaStreaming\services\pubsub_service.py in get_client(config)
7
8
----> 9 def get_client(config):
10 streaming_credentials = credentials_service.fetch_credentials(config)
11 credentials = authentication_service.get_authenticated_oauth_credentials(streaming_credentials)
~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dj_dna_streaming_python-1.0.10-py3.5.egg\dnaStreaming\services\credentials_service.py in fetch_credentials(config)
11 response = _get_requests().get(config.credentials_uri(), headers=headers)
12
---> 13 streaming_credentials_string = json.loads(response.text)['data']['attributes']['streaming_credentials']
14
15 return json.loads(streaming_credentials_string)
~\AppData\Local\Continuum\Anaconda3\lib\site-packages\requests\models.py in text(self)
824
825 self._content_consumed = True
--> 826 # don't need to release the connection; that's been handled by urllib3
827 # since we exhausted the data.
828 return self._content
~\AppData\Local\Continuum\Anaconda3\lib\site-packages\requests\models.py in apparent_encoding(self)
694 is **not** a check to see if the response code is ``200 OK``.
695 """
--> 696 try:
697 self.raise_for_status()
698 except HTTPError:
~\AppData\Local\Continuum\Anaconda3\lib\site-packages\requests\packages\chardet\__init__.py in detect(aBuf)
ImportError: cannot import name 'universaldetector'
I understand that the key part is that I can't import universaldetector. Any idea why is that? I have seen this answer but can't really relate to my problem. I have upgraded chardet and requests.
I am on Python3 and win . Executing code in Jupyter Notebook.
Related
I am trying to connect to an HDFS Cluster using python code, library(snakebite-py3) and I see that when I set use_sasl to True I am getting the following error:
Code Snippet:
from snakebite.client import Client
client = Client(host='hostname', port=8020,
effective_user='user', use_sasl=True)
for x in client.ls(['/']):
print(x,"\n")
Error:
---------------------------------------------------------------------------
GSSError Traceback (most recent call last)
<ipython-input-21-62c8b8df16ea> in <module>
2 from snakebite.client import Client
3
----> 4 client = Client(host='hostname',port=8020, effective_user='user', use_sasl=True)
5
6 for x in client.ls(['/test_abha']): print(x,"\n")
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\client.py in __init__(self, host, port, hadoop_version, use_trash, effective_user, use_sasl, hdfs_namenode_principal, sock_connect_timeout, sock_request_timeout, use_datanode_hostname)
126 self.hdfs_namenode_principal = hdfs_namenode_principal
127 self.service_stub_class = client_proto.ClientNamenodeProtocol_Stub
--> 128 self.service = RpcService(self.service_stub_class, self.port, self.host, hadoop_version,
129 effective_user,self.use_sasl, self.hdfs_namenode_principal,
130 sock_connect_timeout, sock_request_timeout)
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\service.py in __init__(self, service_stub_class, port, host, hadoop_version, effective_user, use_sasl, hdfs_namenode_principal, sock_connect_timeout, sock_request_timeout)
30
31 # Setup the RPC channel
---> 32 self.channel = SocketRpcChannel(host=self.host, port=self.port, version=hadoop_version,
33 effective_user=effective_user, use_sasl=use_sasl,
34 hdfs_namenode_principal=hdfs_namenode_principal,
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\channel.py in __init__(self, host, port, version, effective_user, use_sasl, hdfs_namenode_principal, sock_connect_timeout, sock_request_timeout)
193 raise FatalException("Kerberos libs not found. Please install snakebite using 'pip install snakebite[kerberos]'")
194
--> 195 kerberos = Kerberos()
196 self.effective_user = effective_user or kerberos.user_principal()
197 else:
C:\ProgramData\Anaconda3\lib\site-packages\snakebite\kerberos.py in __init__(self)
41 class Kerberos:
42 def __init__(self):
---> 43 self.credentials = gssapi.Credentials(usage='initiate')
44
45 def user_principal(self):
C:\ProgramData\Anaconda3\lib\site-packages\gssapi\creds.py in __new__(cls, base, token, name, lifetime, mechs, usage, store)
61 base_creds = rcred_imp_exp.import_cred(token)
62 else:
---> 63 res = cls.acquire(name, lifetime, mechs, usage,
64 store=store)
65 base_creds = res.creds
C:\ProgramData\Anaconda3\lib\site-packages\gssapi\creds.py in acquire(cls, name, lifetime, mechs, usage, store)
134
135 if store is None:
--> 136 res = rcreds.acquire_cred(name, lifetime,
137 mechs, usage)
138 else:
gssapi/raw/creds.pyx in gssapi.raw.creds.acquire_cred()
GSSError: Major (851968): Unspecified GSS failure. Minor code may provide more information, Minor (39756044): Credential cache is empty
Please kindly suggest, thank you.
The following code takes a username and scrapes their twitter history from a given date
import pandas as pd
import twint
import pywren
def scrape_user(username):
c = twint.Config()
c.Username = username
c.Lang = 'es'
c.Since = '2021-04-28'
c.Hide_output = True
c.Pandas = True
twint.run.Search(c)
return twint.storage.panda.Tweets_df
When I run the function, I get the intended result i.e., a Pandas dataframe e.g., scrape_user("DeLaCalleHum"). However, when I use pywren (on even a single username)
pwex = pywren.default_executor()
futures = pwex.map(scrape_user, "DeLaCalleHum")
tweet_list = pywren.get_all_results(futures)
I get this error.
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
<ipython-input-31-15f9e00ead75> in <module>
----> 1 wc_list = pywren.get_all_results(futures)
~/macs30123/lib/python3.7/site-packages/pywren/wren.py in get_all_results(fs)
117 """
118 wait(fs, return_when=ALL_COMPLETED)
--> 119 return [f.result() for f in fs]
~/macs30123/lib/python3.7/site-packages/pywren/wren.py in <listcomp>(.0)
117 """
118 wait(fs, return_when=ALL_COMPLETED)
--> 119 return [f.result() for f in fs]
~/macs30123/lib/python3.7/site-packages/pywren/future.py in result(self, timeout, check_only, throw_except, storage_handler)
146 if self._state == JobState.error:
147 if throw_except:
--> 148 raise self._exception
149 else:
150 return None
OSError: [Errno 28] No space left on device
What am I doing wrong? I would appreciate any help.
After some time I found the answer. I can automatically parallelize such function calls in PyWren as long as I add the ComprehendFullAccess policy to my pywren_exec_role_1 role in IAM
I'm trying to authenticate to Atlas with the atlasapi. I'm using my google account and get the error ErrAtlasUnauthorized: Authentication is required with the below method. Is google auth supported or am I doing something wrong?
from atlasapi.atlas import Atlas
auth = Atlas("foo#google.com","<password>","<groupId>")
clusters = auth.Clusters.get_all_clusters
print (clusters())
full trace:
ErrAtlasUnauthorized Traceback (most recent call last)
<ipython-input-61-d69a101fdf69> in <module>
1 clusters = auth.Clusters.get_all_clusters
----> 2 print (clusters())
C:\...\atlasapi\atlas.py in get_all_clusters(self, pageNum, itemsPerPage, iterable)
129
130 uri = Settings.api_resources["Clusters"]["Get All Clusters"] % (self.atlas.group, pageNum, itemsPerPage)
--> 131 return self.atlas.network.get(Settings.BASE_URL + uri)
132
133 def get_single_cluster(self, cluster: str) -> dict:
C:\...\atlasapi\network.py in get(self, uri)
144 logger.debug("Auth information = {} {}".format(self.user, self.password))
145
--> 146 return self.answer(r.status_code, r.json())
147
148 except Exception:
C:\...\atlasapi\network.py in answer(self, c, details)
68 raise ErrAtlasBadRequest(c, details)
69 elif c == Settings.UNAUTHORIZED:
---> 70 raise ErrAtlasUnauthorized(c, details)
71 elif c == Settings.FORBIDDEN:
72 raise ErrAtlasForbidden(c, details)
ErrAtlasUnauthorized: Authentication is required
The API access keys are your User/Password.
I was trying to import ecoinvent 3.5 cutoff to a project using brightway, with the following:
if 'ecoinvent 3.5 cutoff' not in databases:
ei35cutofflink=r"H:\Data\ecoinvent 3.5_cutoff_lci_ecoSpold02\datasets"
ei35cutoff=SingleOutputEcospold2Importer(ei35cutofflink, 'ecoinvent 3.5 cutoff')
ei35cutoff.apply_strategies()
ei35cutoff.statistics()
ei35cutoff.write_database()
But I got the following error. It looks like the issue is not that related to brightway, but rather multiprocessing or pickle? I don't understand what the error message means.
---------------------------------------------------------------------------
MaybeEncodingError Traceback (most recent call last)
<ipython-input-4-f9acb2bc0c84> in <module>
1 if 'ecoinvent 3.5 cutoff' not in databases:
2 ei35cutofflink=r"H:\Data\ecoinvent 3.5_cutoff_lci_ecoSpold02\datasets"
----> 3 ei35cutoff=SingleOutputEcospold2Importer(ei35cutofflink, 'ecoinvent 3.5 cutoff')
4 ei35cutoff.apply_strategies()
5 ei35cutoff.statistics()
C:\miniconda3_py37\envs\ab\lib\site-packages\bw2io\importers\ecospold2.py in __init__(self, dirpath, db_name, extractor, use_mp, signal)
63 start = time()
64 try:
---> 65 self.data = extractor.extract(dirpath, db_name, use_mp=use_mp)
66 except RuntimeError as e:
67 raise MultiprocessingError('Multiprocessing error; re-run using `use_mp=False`'
C:\miniconda3_py37\envs\ab\lib\site-packages\bw2io\extractors\ecospold2.py in extract(cls, dirpath, db_name, use_mp)
91 ) for x in filelist
92 ]
---> 93 data = [p.get() for p in results]
94 else:
95 pbar = pyprind.ProgBar(len(filelist), title="Extracting ecospold2 files:", monitor=True)
C:\miniconda3_py37\envs\ab\lib\site-packages\bw2io\extractors\ecospold2.py in <listcomp>(.0)
91 ) for x in filelist
92 ]
---> 93 data = [p.get() for p in results]
94 else:
95 pbar = pyprind.ProgBar(len(filelist), title="Extracting ecospold2 files:", monitor=True)
C:\miniconda3_py37\envs\ab\lib\multiprocessing\pool.py in get(self, timeout)
655 return self._value
656 else:
--> 657 raise self._value
658
659 def _set(self, i, obj):
MaybeEncodingError: Error sending result: '<multiprocessing.pool.ExceptionWithTraceback object at 0x000001D257C55358>'. Reason: 'TypeError("can't pickle lxml.etree._ListErrorLog objects")'```
Use can use use_mp=False to get a sense of what the actual error is (instead of the error not being pickle-able, and this raising a separate errror). In this case I think you have a problem with the data folder, which you can solve by deleting it and downloading or extracting it again.
I'm trying to open the Sage's notebook, but it isn't working.
I have no idea where this error came from, because the notebook was working this week. I guess it just popped up out of nowhere.
The message's error is:
sage: notebook()
---------------------------------------------------------------------------
EnvironmentError Traceback (most recent call last)
<ipython-input-4-3728cb3d7c7d> in <module>()
----> 1 notebook()
/home/jerome/opt/SageMath/src/sage/misc/lazy_import.pyx in sage.misc.lazy_import.LazyImport.__call__ (/home/jerome/opt/SageMath/src/build/cythonized/sage/misc/lazy_import.c:3634)()
384 True
385 """
--> 386 return self._get_object()(*args, **kwds)
387
388 def __repr__(self):
/home/jerome/opt/SageMath/src/sage/misc/lazy_import.pyx in sage.misc.lazy_import.LazyImport._get_object (/home/jerome/opt/SageMath/src/build/cythonized/sage/misc/lazy_import.c:2241)()
244 elif self._at_startup and not startup_guard:
245 print('Option ``at_startup=True`` for lazy import {0} not needed anymore'.format(self._name))
--> 246 self._object = getattr(__import__(self._module, {}, {}, [self._name]), self._name)
247 alias = self._as_name or self._name
248 if self._deprecation is not None:
/home/jerome/opt/SageMath/local/lib/python2.7/site-packages/sagenb/notebook/notebook_object.py in <module>()
15 import time, os, shutil, signal, tempfile
16
---> 17 import notebook as _notebook
18
19 import run_notebook
/home/jerome/opt/SageMath/local/lib/python2.7/site-packages/sagenb/notebook/notebook.py in <module>()
33
34 # Sage libraries
---> 35 from sagenb.misc.misc import (pad_zeros, cputime, tmp_dir, load, save,
36 ignore_nonexistent_files, unicode_str)
37
/home/jerome/opt/SageMath/local/lib/python2.7/site-packages/sagenb/misc/misc.py in <module>()
379
380 try:
--> 381 from sage.misc.cython import cython
382 except ImportError:
383 #stub
/home/jerome/opt/SageMath/local/lib/python2.7/site-packages/sage/misc/cython.py in <module>()
28
29 # CBLAS can be one of multiple implementations
---> 30 cblas_pc = pkgconfig.parse('cblas')
31 cblas_libs = list(cblas_pc['libraries'])
32 cblas_library_dirs = list(cblas_pc['library_dirs'])
/home/jerome/opt/SageMath/local/lib/python2.7/site-packages/pkgconfig-1.1.0-py2.7.egg/pkgconfig/pkgconfig.py in parse(packages)
185
186 for package in packages.split():
--> 187 for k, v in parse_package(package).items():
188 result[k].update(v)
189
/home/jerome/opt/SageMath/local/lib/python2.7/site-packages/pkgconfig-1.1.0-py2.7.egg/pkgconfig/pkgconfig.py in parse_package(package)
158
159 # Execute the query to pkg-config and clean the result.
--> 160 out = _query(package, '--cflags --libs')
161 out = out.replace('\\"', '')
162
/home/jerome/opt/SageMath/local/lib/python2.7/site-packages/pkgconfig-1.1.0-py2.7.egg/pkgconfig/pkgconfig.py in _wrapper(*args, **kwargs)
56 return func(*args, **kwargs)
57 except OSError:
---> 58 raise EnvironmentError("pkg-config is not installed")
59
60 return _wrapper
EnvironmentError: pkg-config is not installed
If you guys can help me, I'll be very thankful!