We Keep Coding

Python3 firebase storage SDK cannot upload to Emulator

I have the following code snippet:
import firebase_admin
from firebase_admin import credentials
from firebase_admin import storage
from google.cloud import storage
class firebase_storage():
def __init__(self, path_to_sak, root_bucket):
try:
self.cred = credentials.Certificate(path_to_sak)
firebase_admin.initialize_app(self.cred)
except Exception as e:
print("Firebase App may have already been initialized")
self.bucket = firebase_admin.storage.bucket(root_bucket)
def upload(self, key, file_path):
blob = storage.Blob(key, self.bucket)
blob.upload_from_filename(file_path)
def download(self, key, file_path):
blob = storage.Blob(key, self.bucket)
blob.download_to_filename(file_path)
def upload_string(self, key, string, mime_type):
blob = storage.Blob(key, self.bucket)
blob.upload_from_string(string, content_type=mime_type)
I'm using Firebase Emulators for Storage, I have verified that downloads work using the method call firebase_storage.download().
However, when I try to call upload() the following exception is thrown:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2348, in upload_from_file
created_json = self._do_upload(
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2170, in _do_upload
response = self._do_multipart_upload(
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 1732, in _do_multipart_upload
response = upload.transmit(
File "/usr/local/lib/python3.8/dist-packages/google/resumable_media/requests/upload.py", line 149, in transmit
self._process_response(response)
File "/usr/local/lib/python3.8/dist-packages/google/resumable_media/_upload.py", line 116, in _process_response
_helpers.require_status_code(response, (http_client.OK,), self._get_status_code)
File "/usr/local/lib/python3.8/dist-packages/google/resumable_media/_helpers.py", line 99, in require_status_code
raise common.InvalidResponse(
google.resumable_media.common.InvalidResponse: ('Request failed with status code', 400, 'Expected one of', <HTTPStatus.OK: 200>)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "boot.py", line 55, in <module>
run()
File "boot.py", line 35, in run
fb_storage.upload(key, file)
File "/root/python_db_client/src/firebase_storage.py", line 20, in upload
blob.upload_from_filename(file_path)
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2475, in upload_from_filename
self.upload_from_file(
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 2364, in upload_from_file
_raise_from_invalid_response(exc)
File "/usr/local/lib/python3.8/dist-packages/google/cloud/storage/blob.py", line 3933, in _raise_from_invalid_response
raise exceptions.from_http_status(response.status_code, message, response=response)
google.api_core.exceptions.BadRequest: 400 POST http://myserver.com:9194/upload/storage/v1/b/xxxxxx.appspot.com/o?uploadType=multipart: Bad Request: ('Request failed with status code', 400, 'Expected one of', <HTTPStatus.OK: 200>)
My storage.rules look like this:
rules_version = '2';
service firebase.storage {
match /b/{bucket}/o {
match /{allPaths=**} {
allow write, read: if true;
}
}
}
And so, it would appear that public read/write access is allowed.
Everything is working, I have other emulators (Firestore, Auth) that is working fine, but Storage uploads refuse to work :(
Any help would be greatly appreciated thank you!

Maybe there is a problem initializing your app. I see your are taking granted that the app is initialized if there is an error while initializing. Try checking connection first! It may help...

Python Admin SDK does not currently support the Storage emulator according to the documentation
https://firebase.google.com/docs/emulator-suite/install_and_configure#admin_sdk_availability

How to add constraints in client.services.create while creating a new service in existing docker swarm Python Docker SDK?

I want to create a service in existing swarm network using python docker sdk. I have a swarm network named test_net.
Installation of library : pip3 install docker
Below is the code used for creating the service
import docker
from docker.types import RestartPolicy, Placement
def python_sdk():
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
service_created = client.services.create(
image='python:3.7-alpine',
command='python /home/ubuntu/python.py',
constraints=Placement(constraints=['worker']),
mounts='/home/ubuntu/deployment/python.py:/home/ubuntu/python.py:rw',
networks=['test_net'],
restart_policy=RestartPolicy(condition='none'),
name='python_sdk'
)
print("Created service : ", service_created)
Below is the error which I got by executing above code :
Traceback (most recent call last):
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/docker/api/client.py", line 268, in _raise_for_status
response.raise_for_status()
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/requests/models.py", line 943, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: http+docker://localhost/v1.41/services/create
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "client.py", line 20, in <module>
python_sdk()
File "client.py", line 16, in python_sdk
name='python_sdk'
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/docker/models/services.py", line 227, in create
service_id = self.client.api.create_service(**create_kwargs)
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/docker/utils/decorators.py", line 34, in wrapper
return f(self, *args, **kwargs)
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/docker/api/service.py", line 190, in create_service
self._post_json(url, data=data, headers=headers), True
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/docker/api/client.py", line 274, in _result
self._raise_for_status(response)
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/docker/api/client.py", line 270, in _raise_for_status
raise create_api_error_from_http_exception(e)
File "/home/ubuntu/deployment/dags/venv/lib/python3.6/site-packages/docker/errors.py", line 31, in create_api_error_from_http_exception
raise cls(e, response=response, explanation=explanation)
docker.errors.APIError: 400 Client Error for http+docker://localhost/v1.41/services/create: Bad Request ("json: cannot unmarshal object into Go struct field Placement.TaskTemplate.Placement.Constraints of type []string")
I am referring to this documentation.
How can I use Placement object to use constraints?
I also tried constraints = ["Placement(constraints=['worker']"]

I got the answer for the above issue. In the documentation, it is mentioned that the list of str needs to be passed in constraints.
The mentioned parameter is documentation : constraints (list of str) – Placement constraints.
So the final code will be,
import docker
from docker.types import RestartPolicy, Placement
def python_sdk():
client = docker.DockerClient(base_url='unix://var/run/docker.sock')
service_created = client.services.create(
image='python:3.7-alpine',
command='python /home/ubuntu/python.py',
constraints=['node.role == worker']),
mounts='/home/ubuntu/deployment/python.py:/home/ubuntu/python.py:rw',
networks=['test_net'],
restart_policy=RestartPolicy(condition='none'),
name='python_sdk'
)
print("Created service : ", service_created)

Bottle GET request is broken with certain strings when running with paste server

I have a problem similar to this one:
Python bottle: UTF8 path string invalid when using app.mount()
When I try to GET /languages/Inglês I receive the error below (notice the accent "ê"). When passing [az] strings it works fine.
Critical error while processing request: /languages/Inglês
I tried the fix mentioned on the link above without success.
Working example:
from bottle import route, run, debug
#route('/languages/<name>')
def hello(name):
return name
if __name__ == '__main__':
debug(False)
#run(reloader=False, port = 8080) # works
run(server='paste', port = 8080) # fails
Running with server='paste' causes the crash, but using the Bottle server it runs OK. The problem seems to happen at the bottle._handle() method, where a UnicodeError is thrown (bottle.py line 844):
def _handle(self, environ):
path = environ['bottle.raw_path'] = environ['PATH_INFO']
if py3k:
try:
environ['PATH_INFO'] = path.encode('latin1').decode('utf8')
except UnicodeError:
return HTTPError(400, 'Invalid path string. Expected UTF-8')
I'm using Python 3.6.2, Bottle v0.12.13 and Paste 2.0.3 on a Windows 10 machine. What's going on? Is that a problem with Bottle or Paste?
Note: I've already solved my problem by refactoring all the code to use integer IDs instead of names. But I still would like to learn more about this.
Stack trace:
Critical error while processing request: /hello/inglês
Error:
RuntimeError('Request context not initialized.',)
Traceback:
Traceback (most recent call last):
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 1661, in fget
try: return ls.var
AttributeError: '_thread._local' object has no attribute 'var'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 954, in wsgi
out = self._cast(self._handle(environ))
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 907, in _cast
out = self.error_handler.get(out.status_code, self.default_error_handler)(out)
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 842, in default_error_handler
return tob(template(ERROR_PAGE_TEMPLATE, e=res))
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 3619, in template
return TEMPLATES[tplid].render(kwargs)
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 3409, in render
self.execute(stdout, env)
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 3396, in execute
eval(self.co, env)
File "<string>", line 17, in <module>
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 1249, in url
return self.urlparts.geturl()
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 165, in __get__
key, storage = self.key, getattr(obj, self.attr)
File "C:\Users\fernando.filho\AppData\Local\Programs\Python\Python36\lib\site-packages\bottle.py", line 1663, in fget
raise RuntimeError("Request context not initialized.")
RuntimeError: Request context not initialized.

Answering my own question, quoting #GrahamDumpleton:
The Paste server is not tolerant of being sent non ASCII characters.

this is the bottle's problem!
In order to use paste server u can change this:
"""bottle.py"""
environ['PATH_INFO'] = path.encode('latin1').decode('utf8', 'ignore')
to:
environ['PATH_INFO'] = path.encode('utf8').decode('utf8', 'ignore') #utf-8 or else
it's work well.

python and boto : 403 accessDenied

import boto
from boto.s3.connection import S3Connection
from boto.s3.connection import OrdinaryCallingFormat
conn = S3Connection(access_key, secret_key, calling_format=OrdinaryCallingFormat())
bucket = conn.get_bucket(file_name)
print(bucket.name)
And the console display:
raise err
boto.exception.S3ResponseError: S3ResponseError: 403 Forbidden
I have seen many post about the same problem but I can't figure out how to solve it...
note that I am not the owner of the bucket, but I succeed to connect and download the file with a gui tool. I need to process it by script for automation.
EDIT:
Succeed to connect, but still struggling...
I begin to hesitate to process it automatically rather than manually ...
conn = S3Connection(access_key, secret_key, calling_format=OrdinaryCallingFormat())
bucket = conn.get_bucket(bucket_name, validate=False)
print('bucket:', bucket)
print('bucket.name:', bucket.name)
key = bucket.get_key(file_name)
print("key: {name}\t{size}\t{modified}".format(name = key.name,size = key.size,modified = key.last_modified))
print('bucket.list():',bucket.list(prefix='GA-Exports/Events_3112/DEV'))
for key in bucket.list(prefix='DEV/',delimiter='/'):
print('bucket list -> key:',key)
console :
bucket: <Bucket: GA-Exports/Events_3112/>
bucket.name: GA-Exports/Events_3112/
key: DEV/EVENTS_3113_120002892.csv.gz 3826 Sat, 16 May 2015 10:05:44 GMT
bucket.list(): <boto.s3.bucketlistresultset.BucketListResultSet object at 0x0000000004E9F7F0>
Traceback (most recent call last):
File "D:\Python\lib\xml\sax\expatreader.py", line 207, in feed
self._parser.Parse(data, isFinal)
xml.parsers.expat.ExpatError: no element found: line 1, column 0
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Francois\OneDrive\IDE\Workspace\eclipse\Python_test\etltest.py", line 31, in <module>
for key in bucket.list(prefix='DEV/',delimiter='/'):
File "D:\Python\lib\site-packages\boto\s3\bucketlistresultset.py", line 34, in bucket_lister
encoding_type=encoding_type)
File "D:\Python\lib\site-packages\boto\s3\bucket.py", line 472, in get_all_keys
'', headers, **params)
File "D:\Python\lib\site-packages\boto\s3\bucket.py", line 406, in _get_all
xml.sax.parseString(body, h)
File "D:\Python\lib\xml\sax\__init__.py", line 46, in parseString
parser.parse(inpsrc)
File "D:\Python\lib\xml\sax\expatreader.py", line 107, in parse
xmlreader.IncrementalParser.parse(self, source)
File "D:\Python\lib\xml\sax\xmlreader.py", line 125, in parse
self.close()
File "D:\Python\lib\xml\sax\expatreader.py", line 217, in close
self.feed("", isFinal = 1)
File "D:\Python\lib\xml\sax\expatreader.py", line 211, in feed
self._err_handler.fatalError(exc)
File "D:\Python\lib\xml\sax\handler.py", line 38, in fatalError
raise exception
xml.sax._exceptions.SAXParseException: <unknown>:1:0: no element found

By default, boto will attempt to validate the bucket when you call get_bucket by performing a HEAD request on the bucket. You may not have permission to do this even though you may have permission to retrieve objects from the bucket. Try this to disable the validation step:
bucket = conn.get_bucket(bucket_name, validate=False)
Also, make sure you are passing in the name of the bucket. Your example code is passing in file_name which doesn't sound right.

Python suds Exchange Web Service getting empty service attribute

I finally got my EWS client to not give me 401 errors but I don't know if that actually means anything. Now when I instantiate the suds Client object, it's got an empty service attribute.
from suds.transport.https import *
from suds.client import Client
from os import environ
import sys
def first(car=None, *cdr):
return car
def cleaned(lines):
return map(str.strip, lines)
def getauth(f=open("%s/.ews/auth"%(environ.get("HOME")), "rt")):
return first(cleaned(f.readlines()), f.close())
def serviceURI():
return "https://%s/ews/Services.wsdl"%(environ.get("WEBMAIL"))
def auth():
def nclnt(tx):
return Client(serviceURI(), transport=tx)
def ntauth(username, password):
'''Authenticate with NTLM and return the Client object.'''
return nclnt(WindowsHttpAuthenticated(username=username,
password=password))
def webauth(username, password):
'''Use standard web authentication.'''
return nclnt(HttpAuthenticated(username=username,
password=password))
def authWith(method):
return method(*getauth())
return authWith(ntauth if "ntlm" in sys.argv else webauth)
def main():
def _go(client):
print client
print client.last_received
print dir(client.service)
return 0
return _go(auth())
if __name__=="__main__":
main()
And when I run this:
[ishpeck#slcyoshimitsu random_scripts]$ python ews.py ntlm
Suds ( https://fedorahosted.org/suds/ ) version: 0.4 GA build: R699-20100913
<bound method Client.last_received of <suds.client.Client object at 0x17ea6d0>>
Traceback (most recent call last):
File "ews.py", line 42, in <module>
main()
File "ews.py", line 39, in main
return _go(auth())
File "ews.py", line 37, in _go
print dir(client.service)
File "/usr/lib/python2.7/site-packages/suds/client.py", line 296, in __getattr__
port = self.__find(0)
File "/usr/lib/python2.7/site-packages/suds/client.py", line 331, in __find
raise Exception, 'No services defined'
Exception: No services defined
[ishpeck#slcyoshimitsu random_scripts]$ python ews.py
Suds ( https://fedorahosted.org/suds/ ) version: 0.4 GA build: R699-20100913
<bound method Client.last_received of <suds.client.Client object at 0x136c6d0>>
Traceback (most recent call last):
File "ews.py", line 42, in <module>
main()
File "ews.py", line 39, in main
return _go(auth())
File "ews.py", line 37, in _go
print dir(client.service)
File "/usr/lib/python2.7/site-packages/suds/client.py", line 296, in __getattr__
port = self.__find(0)
File "/usr/lib/python2.7/site-packages/suds/client.py", line 331, in __find
raise Exception, 'No services defined'
Exception: No services defined
I'm noticing a lot of people complaining about having problems with this very thing but haven't found anybody who claims to have gotten it working.

your "print client" line didn't return anything, so I suspect it had a problem with the wsdl.
Turn on some debugging to see what's going on..
import logging
logging.basicConfig(level=logging.DEBUG, filename="suds.log")
logging.getLogger('suds.client').setLevel(logging.DEBUG)
logging.getLogger('suds.transport').setLevel(logging.DEBUG)
logging.getLogger('suds.xsd.schema').setLevel(logging.DEBUG)
logging.getLogger('suds.wsdl').setLevel(logging.DEBUG)