I am trying to do a timer trigger whereby every 5 minutes, the function will upload a file into my blob storage.
When I run my code locally, it works, but it fails when it is deployed on Azure. Any help will be appreciated.
Main Method
device_client = IoTHubDeviceClient.create_from_connection_string(CONNECTION_STRING)
PATH_TO_FILE = wget.download("link-of-something", out=os.getcwd())
device_client.connect()
blob_name = os.path.basename(PATH_TO_FILE)
storage_info = device_client.get_storage_info_for_blob(blob_name)
store_blob(storage_info, PATH_TO_FILE)
device_client.shutdown()
Helper method
def store_blob(blob_info, file_name):
try:
sas_url = "https://{}/{}/{}{}".format(
blob_info["hostName"],
blob_info["containerName"],
blob_info["blobName"],
blob_info["sasToken"]
)
print("\nUploading file: {} to Azure Storage as blob: {} in container {}\n".format(file_name, blob_info["blobName"], blob_info["containerName"]))
# Upload the specified file
with BlobClient.from_blob_url(sas_url) as blob_client:
with open(file_name, "rb") as f:
result = blob_client.upload_blob(f, overwrite=True)
return (True, result)
except FileNotFoundError as ex:
# catch file not found and add an HTTP status code to return in notification to IoT Hub
ex.status_code = 404
return (False, ex)
except AzureError as ex:
# catch Azure errors that might result from the upload operation
return (False, ex)
This is the error log (Edited)
Result: Failure Exception: OSError: [Errno 30] Read-only file system: './nasa_graphics_manual_nhb_1430-2_jan_1976.pdfjo243l48.tmp' Stack: File "/azure-functions-host/workers/python/3.9/LINUX/X64/azure_functions_worker/dispatcher.py", line 407, in _handle__invocation_request call_result = await self._loop.run_in_executor( File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 58, in run result = self.fn(*self.args, **self.kwargs) File "/azure-functions-host/workers/python/3.9/LINUX/X64/azure_functions_worker/dispatcher.py", line 649, in _run_sync_func return ExtensionManager.get_sync_invocation_wrapper(context, File "/azure-functions-host/workers/python/3.9/LINUX/X64/azure_functions_worker/extension.py", line 215, in _raw_invocation_wrapper result = function(**args) File "/home/site/wwwroot/azure-function-timer/__init__.py", line 134, in main PATH_TO_FILE = wget.download("https://www.nasa.gov/sites/default/files/atoms/files/nasa_graphics_manual_nhb_1430-2_jan_1976.pdf", out=os.getcwd()) # wget to get the filename and path File "/home/site/wwwroot/.python_packages/lib/site-packages/wget.py", line 506, in download (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=prefix, dir=".") File "/usr/local/lib/python3.9/tempfile.py", line 336, in mkstemp return _mkstemp_inner(dir, prefix, suffix, flags, output_type) File "/usr/local/lib/python3.9/tempfile.py", line 255, in _mkstemp_inner fd = _os.open(file, flags, 0o600)
What you can do is containerize the function using docker and inside the container the place the file so that you can read the file later.
Because if you don't containerize the function, only the function's code will be deployed and not the file.
Refer this documentation for Indepth explanation.
Related
While downloading azure blobs to local file system, I'm getting the following exception:
Client-Request-ID=99bdb0e4-2d1c-11e8-8fe6-00155dbf7128 Retry policy did not allow for a retry: Server-Timestamp=Wed, 21 Mar 2018 15:29:09 GMT, Server-Request-ID=1e7ab8f5-101e-0076-5329-c16a24000000, HTTP status code=404, Exception=The specified blob does not exist.
ErrorCode: BlobNotFound<?xml version="1.0" encoding="utf-8"?><Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.
RequestId:1e7ab8f5-101e-0076-5329-c16a24000000Time:2018-03-21T15:29:09.6565984Z</Message></Error>.
Traceback (most recent call last):
File "C:\Program Files\Commvault\ContentStore\Automation\CloudApps\CloudAppsUtils\cahelper.py", line 483, in download_contents_azure
session.get_blob_to_path(container_name,fl,fl)
File "C:\Program Files\Python36\lib\site-packages\azure\storage\blob\baseblobservice.py", line 1817, in get_blob_to_path
timeout)
File "C:\Program Files\Python36\lib\site-packages\azure\storage\blob\baseblobservice.py", line 2003, in get_blob_to_stream
raise ex
File "C:\Program Files\Python36\lib\site-packages\azure\storage\blob\baseblobservice.py", line 1971, in get_blob_to_stream
_context=operation_context)
File "C:\Program Files\Python36\lib\site-packages\azure\storage\blob\baseblobservice.py", line 1695, in _get_blob
operation_context=_context)
File "C:\Program Files\Python36\lib\site-packages\azure\storage\common\storageclient.py", line 354, in _perform_request
raise ex
File "C:\Program Files\Python36\lib\site-packages\azure\storage\common\storageclient.py", line 289, in _perform_request
raise ex
File "C:\Program Files\Python36\lib\site-packages\azure\storage\common\storageclient.py", line 275, in _perform_request
HTTPError(response.status, response.message, response.headers, response.body))
File "C:\Program Files\Python36\lib\site-packages\azure\storage\common\_error.py", line 111, in _http_error_handler
raise AzureHttpError(message, http_error.status)
azure.common.AzureMissingResourceHttpError: The specified blob does not exist.ErrorCode: BlobNotFound
<?xml version="1.0" encoding="utf-8"?><Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.
RequestId:1e7ab8f5-101e-0076-5329-c16a24000000
Time:2018-03-21T15:29:09.6565984Z</Message></Error>
I'm using the following code to download blobs:
def download_contents_azure(self, account_name, account_key, content):
session=self.create_session_azure(account_name,account_key)
os.mkdir('in place')
os.chdir('in place')
for item in content:
# s = os.path.basename(item)
path_to_file = ("/".join(item.strip("/").split('/')[1:]))
container_name = Path(item).parts[1]
gen = session.list_blobs(container_name)
li = []
for i in gen:
li.append(i.name)
if path_to_file in li:
fl = os.path.basename(path_to_file)
print(fl)
c = self.splitall(item)
for i in range(1,len(c)-1):
if path.exists(c[i]) is False:
os.mkdir(c[i])
os.chdir(c[i])
session.get_blob_to_path(container_name,fl,fl)
for i in range(1,len(c)-1):
os.chdir("..")
else:
c = self.splitall(item)
for i in range(1,len(c)):
os.mkdir(c[i])
os.chdir(c[i])
generator = session.list_blobs(container_name,path_to_file+'/',delimiter='/')
for blob in generator:
bl = os.path.basename(blob.name)
session.get_blob_to_path(container_name,bl,bl)
I've a path in azure (/container/folder/subfolder).
I'm trying to download the structure and all the files under subfolder. the first file under the subfolder gets downloaded and then I've the above exception. Due to this, I'm unable to loop through and print the next items.
Thoughts?
Double check your line
session.get_blob_to_path(container_name,fl,fl)
You have fl for blob_name.
documentation here indicates second argument is blob_name while third is file path on file system where to download.
I think your blob name is wrong. Hence it does not exist.
NOTE: consider installing fiddler so you can look at the network traces. It allows you to see the raw request.
In a Django web app of mine, users post photos on the website. Once photos accumulate high enough upvotes, I post them to a Facebook fan page using this function from the Python Facebook SDK. Initial tests of the SDK work correctly. One simply passes a file object representing the image, and a caption to graph.put_image() method, and it works.
But I can only get it to work when the image is being uploaded from the client. I'm struggling if an image object is already saved on the server; then the SDK throws an AttributeError: read error. I might be making a rookie mistake, so please advise. I get the error when I do:
from PIL import Image
api.put_photo(image=Image.open(photo.image_file),message='my caption')
Where the photo object is defined as follows in my models.py:
class Photo(models.Model):
image_file = models.ImageField(upload_to=upload_photo, storage=OverwriteStorage())
upload_time = models.DateTimeField(auto_now_add=True)
OverwriteStorate() is an overwritten Django Storage class (imported from django.core.files.storage). It contains functionality to upload to Azure blob storage, and has been working correctly. upload_photo_to_location is simply:
def upload_photo(instance, filename):
try:
blocks = filename.split('.')
ext = blocks[-1]
filename = "%s.%s" % (uuid.uuid4(), ext)
instance.title = blocks[0]
return os.path.join('photos/', filename)
except Exception as e:
print '%s (%s)' % (e.message, type(e))
return 0
The full error traceback:
File "/home/myuser/Desktop/myproj/myapp/tasks.py", line 179, in rank_all_photos
photo_poster(photo.image_file, photo.caption)
File "/home/myuser/Desktop/myproj/myapp/facebook_api.py", line 14, in photo_poster
status = api.put_photo(image=Image.open(image_obj),message=caption)
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/facebook/__init__.py", line 193, in put_photo
method="POST")
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/facebook/__init__.py", line 247, in request
files=files)
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/requests/api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/requests/sessions.py", line 451, in request
prep = self.prepare_request(req)
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/requests/sessions.py", line 382, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/requests/models.py", line 307, in prepare
self.prepare_body(data, files, json)
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/requests/models.py", line 453, in prepare_body
(body, content_type) = self._encode_files(files, data)
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/requests/models.py", line 150, in _encode_files
fdata = fp.read()
File "/home/myuser/.virtualenvs/myenv/local/lib/python2.7/site-packages/PIL/Image.py", line 626, in __getattr__
raise AttributeError(name)
AttributeError: read
I've written a Python script that is part of my attempt to automate daily ftp transfers from my server. I've tested the script with a number of files and file types (html, mp3, png, jpg, etc.) and everything seems to work out fine so far.
However, when I try to download a simple text file, 'file.txt' (9 kb), the download fails, although I account for text files and switch from binary to text mode for the transfer. The following exception is thrown by ftplib:
ftplib.error_perm: 550 file.txt: No such file or directory
Here's my script:
from ftplib import FTP_TLS, error_perm
import os
def open_connection(server, user, pwd, work_dir=None):
global ftps
try:
ftps = FTP_TLS(host=server)
ftps.login(user=user, passwd=pwd)
ftps.prot_p() # switch to secure data connection
if work_dir != None:
ftps.cwd(work_dir)
else:
pass
except:
pass
def download_file(remote_path, local_path):
remote_file = os.path.basename(remote_path)
local_file_path = os.path.join(local_path, remote_file)
# differentiate between text and binary files
file_type, encoding = guess_type_and_encoding(remote_file)
# possibly needs a permission exception catch
if file_type.split("/")[0] == "text" and encoding == None:
# use text mode for transfer
local_file = open(local_file_path, 'w')
def callback(line): local_file.write(line + "\n")
ftps.retrlines("RETR " + remote_file, callback)
local_file.close()
else:
# use binary mode for transfer
local_file = open(local_file_path, 'wb')
ftps.retrbinary("RETR " + remote_file, local_file.write)
local_file.close()
return
def guess_type_and_encoding(filename):
from mimetypes import guess_type, add_type
add_type('text/x-python-win', '.pyw') # not in tables
mimetype, encoding = guess_type(filename, False) # allow extras
mimetype = mimetype or "?/?" # type unknown
return mimetype, encoding
open_connection(server, user, pwd, work_dir)
download_file("/files/dir/file.txt", "/Users/username/Desktop")
ftps.close()
I don't get why the error is raised!? The arguments 'remote_path' and 'local_path' are correctly provided. Both paths exist! 'file.txt' exists on the server under /files/dir and /Users/username/Desktop points to my desktop on OS X.
Here's the detailed ftplib error:
Traceback (most recent call last):
File "ftp2.py", line 138, in <module>
download_file("/files/dir/file.txt", "/Users/username/Desktop")
File "ftp2.py", line 93, in download_file
ftps.retrlines("RETR " + remote_file, callback)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.py", line 735, in retrlines
conn = self.transfercmd(cmd)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.py", line 376, in transfercmd
return self.ntransfercmd(cmd, rest)[0]
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.py", line 710, in ntransfercmd
conn, size = FTP.ntransfercmd(self, cmd, rest)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.py", line 339, in ntransfercmd
resp = self.sendcmd(cmd)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.py", line 249, in sendcmd
return self.getresp()
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/ftplib.py", line 224, in getresp
raise error_perm, resp
ftplib.error_perm: 550 file.txt: No such file or directory
Any help is greatly appreciated.
Thanks. :)
Try to
replace remote_file
in ftps.retrlines("RETR " + remote_file, callback)
with remote_path.
I'm trying to connect to an FTP server using TLS and upload a text file. The below code connects to the site just fine, but it's not uploading the file. Instead I'm getting the following error:
Traceback (most recent call last):
File "X:/HR & IT/Ryan/Python Scripts/ftps_connection_test.py", line 16, in <module>
ftps.storlines("STOR " + filename, open(filename,"r"))
File "C:\Python33\lib\ftplib.py", line 816, in storlines
with self.transfercmd(cmd) as conn:
File "C:\Python33\lib\ftplib.py", line 391, in transfercmd
return self.ntransfercmd(cmd, rest)[0]
File "C:\Python33\lib\ftplib.py", line 756, in ntransfercmd
conn, size = FTP.ntransfercmd(self, cmd, rest)
File "C:\Python33\lib\ftplib.py", line 357, in ntransfercmd
resp = self.sendcmd(cmd)
File "C:\Python33\lib\ftplib.py", line 264, in sendcmd
return self.getresp()
File "C:\Python33\lib\ftplib.py", line 238, in getresp
raise error_perm(resp)
ftplib.error_perm: 550 The parameter is incorrect.
There's probably something really basic I'm missing, my code is below and any help is much appreciated.
import os
from ftplib import FTP_TLS as f
# Open secure connection
ftps = f("ftp.foo.com")
ftps.login(username,password)
ftps.prot_p()
# Create the test txt file to upload
filename = r"c:\path\to\file"
testFile = open(filename,"w")
testFile.write("Test file with test text")
testFile.close()
# Transfer testFile
ftps.storlines("STOR " + filename, open(filename,"r"))
# Quit connection
ftps.quit()
I have got the same error when trying to write upload a file to FTP server. In my case, the destination file name is not the correct format. It was something like
data_20180411T12:00:12.3435Z.txt
I renamed something like
data_20180411T120012_3435Z.txt. Then it worked.
filename = r"c:\path\to\file"
is the absolute path to a local file. This same value is being passed in the STOR command, i.e.
ftps.storlines("STOR " + filename, open(filename,"r"))
attempts to perform a STOR c:\path\to\file operation, however, it is unlikely that the path exists on the remote server, and the ftplib.error_perm exception would suggest that you don't have permission to write there (even if it does exist).
You could try this instead:
ftps.storlines("STOR " + os.path.basename(filename), open(filename,"r"))
which would issue a STOR file operation and upload the file to the default directory on the remote server. If you need to upload to a different path on the remote server, just add that to STOR.
I am trying to download the log files that are present in S3 bucket using boto. The reason behind not using s3cmd and some other tools is that I don't want my code to be dependent on some kind of software/tool so that others can also use my code directly and don't have to worry about downloading some other dependencies.
I am getting the following stack trace. I saw various related posts but none of them solved my problem.
Traceback (most recent call last):
File "/Library/Python/2.7/site-packages/fabric/main.py", line 743, in main
*args, **kwargs
File "/Library/Python/2.7/site-packages/fabric/tasks.py", line 405, in execute
results['<local-only>'] = task.run(*args, **new_kwargs)
File "/Library/Python/2.7/site-packages/fabric/tasks.py", line 171, in run
return self.wrapped(*args, **kwargs)
File "/pgbadger/pgbadger_html.py", line 86, in dlogs
s3 = S3()
File "/pgbadger/pgbadger_html.py", line 46, in __init__
self.bucket = self._get_bucket(self.log_bucket)
File "/pgbadger/pgbadger_html.py", line 65, in _get_bucket
return self.s3_conn.get_bucket(bucket)
File "/Library/Python/2.7/site-packages/boto/s3/connection.py", line 471, in get_bucket
return self.head_bucket(bucket_name, headers=headers)
File "/Library/Python/2.7/site-packages/boto/s3/connection.py", line 518, in head_bucket
response.status, response.reason, body)
boto.exception.S3ResponseError: S3ResponseError: 400 Bad Request
I have seen the code and I don't know why I am getting this error. My code is as follows:
from fabric.api import task
from fabric.api import env
S3_LOG_BUCKET = BUCKET-NAME
class S3(object):
s3_conn = None
log_bucket = S3_LOG_BUCKET
region = region
bucket = None
env.host_string = REGION-NAME
def __init__(self):
self._s3_connect()
self.bucket = self._get_bucket(self.log_bucket)
def _s3_connect(self):
if not self.s3_conn:
self.s3_conn = boto.s3.connect_to_region(
self.region,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
)
if not self.s3_conn:
raise ValueError('Invalid Region Name: {}'.format(region))
def download_s3_logs(self):
for l in self.bucket.list():
key_string = str(l.key)
l.get_contents_to_filename("/tempLogFiles/" + key_string)
print l.key
def _get_bucket(self, bucket):
return self.s3_conn.get_bucket(bucket)
#task
def dlogs():
s3 = S3()
s3.download_s3_logs()
Problem Solved:
In the S3-Log-Bucket, I was mentioning the entire path specific to my bucket.. Like I have my bucket and there are multiple folders in it. So I was mentioning the entire path to it but BOTO doesn't expect that to happen. Hence I just had to mention the bucket name only instead of mentioning the entire path.
Previously I was doing-->
log_bucket = Bucket/Inner Folder 1/Inner Folder 2/.../ which was wrong
Correct way of doing it-->
log_bucket = Bucket