We Keep Coding

Python3 firebase storage SDK cannot upload to Emulator

I have the following code snippet:
import firebase_admin
from firebase_admin import credentials
from firebase_admin import storage
from google.cloud import storage
class firebase_storage():
def __init__(self, path_to_sak, root_bucket):
try:
self.cred = credentials.Certificate(path_to_sak)
firebase_admin.initialize_app(self.cred)
except Exception as e:
print("Firebase App may have already been initialized")
self.bucket = firebase_admin.storage.bucket(root_bucket)
def upload(self, key, file_path):
blob = storage.Blob(key, self.bucket)
blob.upload_from_filename(file_path)
def download(self, key, file_path):
blob = storage.Blob(key, self.bucket)
blob.download_to_filename(file_path)
def upload_string(self, key, string, mime_type):
blob = storage.Blob(key, self.bucket)
blob.upload_from_string(string, content_type=mime_type)
I'm using Firebase Emulators for Storage, I have verified that downloads work using the method call firebase_storage.download().
However, when I try to call upload() the following exception is thrown:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2348, in upload_from_file
created_json = self._do_upload(
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2170, in _do_upload
response = self._do_multipart_upload(
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 1732, in _do_multipart_upload
response = upload.transmit(
File "/usr/local/lib/python3.8/dist-packages/google/resumable_media/requests/upload.py", line 149, in transmit
self._process_response(response)
File "/usr/local/lib/python3.8/dist-packages/google/resumable_media/_upload.py", line 116, in _process_response
_helpers.require_status_code(response, (http_client.OK,), self._get_status_code)
File "/usr/local/lib/python3.8/dist-packages/google/resumable_media/_helpers.py", line 99, in require_status_code
raise common.InvalidResponse(
google.resumable_media.common.InvalidResponse: ('Request failed with status code', 400, 'Expected one of', <HTTPStatus.OK: 200>)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "boot.py", line 55, in <module>
run()
File "boot.py", line 35, in run
fb_storage.upload(key, file)
File "/root/python_db_client/src/firebase_storage.py", line 20, in upload
blob.upload_from_filename(file_path)
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2475, in upload_from_filename
self.upload_from_file(
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2364, in upload_from_file
_raise_from_invalid_response(exc)
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 3933, in _raise_from_invalid_response
raise exceptions.from_http_status(response.status_code, message, response=response)
google.api_core.exceptions.BadRequest: 400 POST http://myserver.com:9194/upload/storage/v1/b/xxxxxx.appspot.com/o?uploadType=multipart: Bad Request: ('Request failed with status code', 400, 'Expected one of', <HTTPStatus.OK: 200>)
My storage.rules look like this:
rules_version = '2';
service firebase.storage {
match /b/{bucket}/o {
match /{allPaths=**} {
allow write, read: if true;
}
}
}
And so, it would appear that public read/write access is allowed.
Everything is working, I have other emulators (Firestore, Auth) that is working fine, but Storage uploads refuse to work :(
Any help would be greatly appreciated thank you!

Maybe there is a problem initializing your app. I see your are taking granted that the app is initialized if there is an error while initializing. Try checking connection first! It may help...

Python Admin SDK does not currently support the Storage emulator according to the documentation
https://firebase.google.com/docs/emulator-suite/install_and_configure#admin_sdk_availability

How to upload files to GCS from a python script in GCP?

I'm trying to upload a file into GCS, but I'm running into a permission issue which I'm not sure how to resolve. Reading a file from a bucket in GCS doesn't seem to be an issue. However, I'm getting issues for upload.
client = storage.Client()
bucket = client.get_bucket('fda-drug-label-data')
blob = bucket.get_blob(f'fda-label-doc-links.csv')
bt = blob.download_as_string()
s = str(bt, 'utf-8')
s = StringIO(s)
df = pd.read_csv(s)
df_doc_links = list(df['Link'])
a = pd.DataFrame([len(df_doc_links)])
a.to_csv('test.csv', index=False)
client = storage.Client()
bucket = client.get_bucket('fda-drug-label-data')
blob = bucket.blob('test.csv')
blob.upload_from_filename('test.csv')
This is the message I'm getting:
Traceback (most recent call last): File "/home/.../.local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1567, in upload_from_file
if_metageneration_not_match, File "/home/.../.local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1420, in _do_upload
if_metageneration_not_match, File "/home/.../.local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1098, in _do_multipart_upload
response = upload.transmit(transport, data, object_metadata, content_type) File "/home/.../.local/lib/python3.7/site-packages/google/resumable_media/requests/upload.py", line 108, in transmit
self._process_response(response) File "/home/.../.local/lib/python3.7/site-packages/google/resumable_media/_upload.py", line 109, in _process_response
_helpers.require_status_code(response, (http_client.OK,), self._get_status_code) File "/home/.../.local/lib/python3.7/site-packages/google/resumable_media/_helpers.py", line 96, in require_status_code
*status_codes google.resumable_media.common.InvalidResponse: ('Request failed with status code', 403, 'Expected one of', <HTTPSta tus.OK: 200>) During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "scrape.py", line 134, in <module>
blob.upload_from_filename('test.csv') File "/home/.../.local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1655, in upload_from_filename
if_metageneration_not_match=if_metageneration_not_match, File "/home/.../.local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 1571, in upload_from_file
_raise_from_invalid_response(exc) File "/home/.../.local/lib/python3.7/site-packages/google/cloud/storage/blob.py", line 2620, in _raise_from_invalid_response
raise exceptions.from_http_status(response.status_code, message, response=response) google.api_core.exceptions.Forbidden: 403 POST https://storage.googleapis.com/upload/storage/v1/b/fda-drug-label-da ta/o?uploadType=multipart: ('Request failed with status code', 403, 'Expected one of', <HTTPStatus.OK: 200>)

You don't have permission to upload to the data in your service account.Go to IAM and Admin section and under service accounts assign permission role to your account.After that generate the KEY again.

ML-engine fails from composer -Unknown name "python-version"

Im trying to launch an ml-engine jobs submit training using a cloud composer, i'm using this guide for instructions recommendation-system-tensorflow-deploy.
Im using a plugin which google created (see the implementation here)
Im trying to make it work on python version 3.5, this by changing line 206 from:
training_request = {
'jobId': job_id,
'trainingInput': {
'scaleTier': self._scale_tier,
'packageUris': self._package_uris,
'pythonModule': self._training_python_module,
'region': self._region,
'args': self._training_args,
'masterType': self._master_type
}
To:
training_request = {
'jobId': job_id,
'trainingInput': {
'scaleTier': self._scale_tier,
'packageUris': self._package_uris,
'pythonModule': self._training_python_module,
'region': self._region,
'args': self._training_args,
'masterType': self._master_type,
'python-version': '3.5' #self._python_version
}
I also tried to add to it the run time version (runtime-version='1.12') but i keep on getting the following error:
[2019-01-20 11:58:36,331] {models.py:1594} ERROR - <HttpError 400 when requesting https://ml.googleapis.com/v1/projects/hallowed-forge-577/jobs?alt=json returned "Invalid JSON payload received. Unknown name "python-version" at 'job.training_input': Cannot find field.">
Traceback (most recent call last)
File "/usr/local/lib/airflow/airflow/models.py", line 1492, in _run_raw_tas
result = task_copy.execute(context=context
File "/home/airflow/gcs/plugins/ml_engine_plugin.py", line 241, in execut
self._project_id, training_request, check_existing_job
File "/home/airflow/gcs/plugins/ml_engine_plugin.py", line 79, in create_jo
request.execute(
File "/usr/local/lib/python3.6/site-packages/oauth2client/util.py", line 135, in positional_wrappe
return wrapped(*args, **kwargs
File "/usr/local/lib/python3.6/site-packages/googleapiclient/http.py", line 838, in execut
raise HttpError(resp, content, uri=self.uri
googleapiclient.errors.HttpError: <HttpError 400 when requesting https://ml.googleapis.com/v1/projects/hallowed-forge-577/jobs?alt=json returned "Invalid JSON payload received. Unknown name "python-version" at 'job.training_input': Cannot find field."
[2019-01-20 11:58:36,334] {models.py:1623} INFO - Marking task as FAILED.
[2019-01-20 11:58:36,513] {models.py:1627} ERROR - Failed to send email to: ['airflow#example.com']
[2019-01-20 11:58:36,516] {models.py:1628} ERROR - HTTP Error 401: Unauthorized
Traceback (most recent call last)
File "/usr/local/lib/airflow/airflow/models.py", line 1625, in handle_failur
self.email_alert(error, is_retry=False
File "/usr/local/lib/airflow/airflow/models.py", line 1778, in email_aler
send_email(task.email, title, body
File "/usr/local/lib/airflow/airflow/utils/email.py", line 44, in send_emai
return backend(to, subject, html_content, files=files, dryrun=dryrun, cc=cc, bcc=bcc, mime_subtype=mime_subtype
File "/usr/local/lib/airflow/airflow/contrib/utils/sendgrid.py", line 116, in send_emai
_post_sendgrid_mail(mail.get()
File "/usr/local/lib/airflow/airflow/contrib/utils/sendgrid.py", line 122, in _post_sendgrid_mai
response = sg.client.mail.send.post(request_body=mail_data
File "/usr/local/lib/python3.6/site-packages/python_http_client/client.py", line 252, in http_reques
return Response(self._make_request(opener, request, timeout=timeout)
File "/usr/local/lib/python3.6/site-packages/python_http_client/client.py", line 176, in _make_reques
raise ex
python_http_client.exceptions.UnauthorizedError: HTTP Error 401: Unauthorize
Notice that the python version actually changes (to 3.6 from the original 2.7) so changing the python version, does something, but then gets stuck
Any help on what i'm missing here will be awesome!

It seems like the example uses an old version of airflow MLEngineTrainingOperator.
The last version implements the runtime-version/python-version training params.
Use the current version:
mlengine_operator.py

Python Script Returning an error when trying to Obtain TFS work Items

I have written the following Pythin script to try and gather TFS (Team Foundation Server) work item data. Below is my script:-
from tfs import TFSAPI
user="andrew.xxxx"
password="xxxxxxxx"
tfsAddress = "http://man-tfsmig-1:8080/"
print(tfsAddress)
client = TFSAPI(tfsAddress, project="DefaultCollection/xxxxxxxx", user=user, password=password)
work_item = client.get_workitem(28274)
When I run the code I get the following error, but I do not know what the problem is:-
Traceback (most recent call last):
File ".\TFS_Release_Notes_app.py", line 13, in <module>
work_item = client.get_workitem(28274)
File "C:\Python\lib\site-packages\tfs\connection.py", line 70, in get_workitem
return self.get_workitems(id_, fields)[0]
File "C:\Python\lib\site-packages\tfs\connection.py", line 80, in get_workitems
work_items_batch_info = self.__get_workitems(work_items_batch, fields=fields, expand=expand)
File "C:\Python\lib\site-packages\tfs\connection.py", line 65, in __get_workitems
object_class=Workitem)
File "C:\Python\lib\site-packages\tfs\connection.py", line 46, in get_tfs_object
raw = self.rest_client.send_get(uri=uri, payload=payload, project=project)
File "C:\Python\lib\site-packages\tfs\connection.py", line 323, in send_get
return self.__send_request('GET', uri, None, payload=payload, project=project, json=json)
File "C:\Python\lib\site-packages\tfs\connection.py", line 360, in __send_request
response.raise_for_status()
File "C:\Python\lib\site-packages\requests\models.py", line 939, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http://xxxx-xxxxx-1:8080/DefaultCollection/_apis/wit/workitems?ids=28274&$expand=all&api-version=1.0

It's telling you the error: 404, not found.
That means the URI you're providing for the API is incorrect.
It's generating this URI: http://xxxx-xxxxx-1:8080/DefaultCollection/_apis/wit/workitems. Validate if that is correct.
Usually, when TFS is running on port 8080 over HTTP, there's a /tfs/ virtual directory.

I needed to add the following line to authenticate:-
# Use NTLM authorization
from requests_ntlm import HttpNtlmAuth
client = TFSAPI("https://tfs.tfs.ru/tfs/", user=user, password=password, auth_type=HttpNtlmAuth)

Spotipy: User_create_playlist - Method Not Allowed for url

Using the spotipy library, I'm trying to create a playlist. However, the user_create_playlist method is not allowed for my url. Here is part of my code to show an example of how I'm authenticating my app and what I'm using to run the method:
username = 'my-username'
token = util.prompt_for_user_token(username = username,
scope = 'playlist-modify-public'
client_id='my-spotify-client-id',
client_secret='my-spotify-client-secret-id',
redirect_uri='https://developer.spotify.com/')
spotifyObject = spotipy.Spotify(auth=token)
playlist_name = "Test Playlist"
playlist_description = "This is a test playlist."
playlists = spotifyObject.user_playlist_create(username, playlist_name,
playlist_description)
pprint.pprint(playlists)
Do you know why I am getting the following error message?
Traceback (most recent call last):
File "C:\Users....\spotipy\client.py", line 121, in _internal_call
r.raise_for_status()
File "C:\Users....requests\models.py", line 935, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 405 Client Error: Method Not Allowed for url: https://api.spotify.com/v1/users/'username'/playlists
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users....\SpotifyTest.py", line 108, in
main()
File "C:\Users....\SpotifyTest.py", line 53, in main
playlists = spotifyObject.user_playlist_create(username, playlist_name, playlist_description)
File "C:\Users....\spotipy\client.py", line 415, in user_playlist_create
return self._post("users/%s/playlists" % (user,), payload=data)
File "C:\Users.....\spotipy\client.py", line 180, in _post
return self._internal_call('POST', url, payload, kwargs)
File "C:\Users....\spotipy\client.py", line 129, in _internal_call
-1, '%s:\n %s' % (r.url, 'error'), headers=r.headers)
spotipy.client.SpotifyException: http status: 405, code:-1 - https://api.spotify.com/v1/users/'username'/playlists:
error