I am new to Python scripting. I need to copy few folders from my local machine (windows) to Linux server. As of now, I am copying the folders by opening WinSCP console. I need to automate this process. I have written a below code in Python using Paramiko module library.
import paramiko
import os
transport = paramiko.Transport(('10.10.10.10', 22))
transport.connect(username='weblogic', password='weblogic')
sftp = paramiko.SFTPClient.from_transport(transport)
filepath = '/apps/logs'
localpath = 'C:\\Users\\Public\\test'
sftp.put(localpath,filepath)
Above is not working properly and giving below error. Can you please help me to copy the folder present in the windows path C:\Users\Public\test to Linux server path /apps/logs?
Traceback (most recent call last):
File "C:\Users\Desktop\python\execute_script.py", line 28, in <module>
sftp.put(localpath,filepath)
File "C:\Python27\lib\paramiko\sftp_client.py", line 548, in put
fl = file(localpath, 'rb')
IOError: [Errno 13] Permission denied: 'C:\\Users\\Public\\test'
Please check the below code from the link https://gist.github.com/johnfink8/2190472. I have used put_all method in the snippet.
import paramiko
import socket
import os
from stat import S_ISDIR
class SSHSession(object):
# Usage:
# Detects DSA or RSA from key_file, either as a string filename or a
# file object. Password auth is possible, but I will judge you for
# using it. So:
# ssh=SSHSession('targetserver.com','root',key_file=open('mykey.pem','r'))
# ssh=SSHSession('targetserver.com','root',key_file='/home/me/mykey.pem')
# ssh=SSHSession('targetserver.com','root','mypassword')
# ssh.put('filename','/remote/file/destination/path')
# ssh.put_all('/path/to/local/source/dir','/path/to/remote/destination')
# ssh.get_all('/path/to/remote/source/dir','/path/to/local/destination')
# ssh.command('echo "Command to execute"')
def __init__(self,hostname,username='root',key_file=None,password=None):
#
# Accepts a file-like object (anything with a readlines() function)
# in either dss_key or rsa_key with a private key. Since I don't
# ever intend to leave a server open to a password auth.
#
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((hostname,22))
self.t = paramiko.Transport(self.sock)
self.t.start_client()
keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
key = self.t.get_remote_server_key()
# supposed to check for key in keys, but I don't much care right now to find the right notation
if key_file is not None:
if isinstance(key,str):
key_file=open(key,'r')
key_head=key_file.readline()
key_file.seek(0)
if 'DSA' in key_head:
keytype=paramiko.DSSKey
elif 'RSA' in key_head:
keytype=paramiko.RSAKey
else:
raise Exception("Can't identify key type")
pkey=keytype.from_private_key(key_file)
self.t.auth_publickey(username, pkey)
else:
if password is not None:
self.t.auth_password(username,password,fallback=False)
else: raise Exception('Must supply either key_file or password')
self.sftp=paramiko.SFTPClient.from_transport(self.t)
def command(self,cmd):
# Breaks the command by lines, sends and receives
# each line and its output separately
#
# Returns the server response text as a string
chan = self.t.open_session()
chan.get_pty()
chan.invoke_shell()
chan.settimeout(20.0)
ret=''
try:
ret+=chan.recv(1024)
except:
chan.send('\n')
ret+=chan.recv(1024)
for line in cmd.split('\n'):
chan.send(line.strip() + '\n')
ret+=chan.recv(1024)
return ret
def put(self,localfile,remotefile):
# Copy localfile to remotefile, overwriting or creating as needed.
self.sftp.put(localfile,remotefile)
def put_all(self,localpath,remotepath):
# recursively upload a full directory
os.chdir(os.path.split(localpath)[0])
parent=os.path.split(localpath)[1]
for walker in os.walk(parent):
try:
self.sftp.mkdir(os.path.join(remotepath,walker[0]))
except:
pass
for file in walker[2]:
self.put(os.path.join(walker[0],file),os.path.join(remotepath,walker[0],file))
def get(self,remotefile,localfile):
# Copy remotefile to localfile, overwriting or creating as needed.
self.sftp.get(remotefile,localfile)
def sftp_walk(self,remotepath):
# Kindof a stripped down version of os.walk, implemented for
# sftp. Tried running it flat without the yields, but it really
# chokes on big directories.
path=remotepath
files=[]
folders=[]
for f in self.sftp.listdir_attr(remotepath):
if S_ISDIR(f.st_mode):
folders.append(f.filename)
else:
files.append(f.filename)
print (path,folders,files)
yield path,folders,files
for folder in folders:
new_path=os.path.join(remotepath,folder)
for x in self.sftp_walk(new_path):
yield x
def get_all(self,remotepath,localpath):
# recursively download a full directory
# Harder than it sounded at first, since paramiko won't walk
#
# For the record, something like this would gennerally be faster:
# ssh user#host 'tar -cz /source/folder' | tar -xz
self.sftp.chdir(os.path.split(remotepath)[0])
parent=os.path.split(remotepath)[1]
try:
os.mkdir(localpath)
except:
pass
for walker in self.sftp_walk(parent):
try:
os.mkdir(os.path.join(localpath,walker[0]))
except:
pass
for file in walker[2]:
self.get(os.path.join(walker[0],file),os.path.join(localpath,walker[0],file))
def write_command(self,text,remotefile):
# Writes text to remotefile, and makes remotefile executable.
# This is perhaps a bit niche, but I was thinking I needed it.
# For the record, I was incorrect.
self.sftp.open(remotefile,'w').write(text)
self.sftp.chmod(remotefile,755)
In addition to the answer #user1041177, but here a way to do it when you are on windows to linux host (not really sure which kind of host actually).
I don't know why, but if I keep backslash onto remote path, I get a FileNotFoundException. The only way to work was to replace all '\' by '/'
Maybe someone could tell me the proper way to avoid this situation at all ?
Here a part of the exact same code above to give you breadcrumbs if you encounter the same issue :
def sftp_walk(socket, remotepath):
remotepath = remotepath.replace('\\', '/')
path = remotepath
files = []
folders = []
for f in socket.listdir_attr(remotepath.replace('\\', '/')):
if S_ISDIR(f.st_mode):
folders.append(f.filename)
else:
files.append(f.filename)
print(path, folders, files)
yield path, folders, files
for folder in folders:
new_path = os.path.join(remotepath.replace('\\', '/'), folder)
for x in sftp_walk(socket, new_path):
yield x
def get_all(socket, remotepath, localpath):
remotepath = remotepath.replace('\\', '/')
socket.chdir(os.path.split(remotepath)[0])
parent = os.path.split(remotepath)[1]
try:
os.mkdir(localpath)
except:
pass
for walker in sftp_walk(socket, parent):
try:
os.mkdir(os.path.join(localpath, walker[0]).replace('\\', '/'))
except:
pass
for file in walker[2]:
socket.get(os.path.join(walker[0], file).replace('\\', '/'), os.path.join(localpath, walker[0], file).replace('\\', '/'))
BTW, I am not using those function inside an object, that's why their is 'socket' instead of 'self' because I call those function by passing the SFTP socket to them.
Finally have to say thank you to #user1041177, working like a charm.
I was trying to copy from a windows box to a linux box and got the same error as #Apex above. I was using the put_all method and I had to do some "replace" on parts of the code.
def put_all(self,localpath,remotepath):
remotepath = remotepath.replace('\\', '/')
# recursively upload a full directory
os.chdir(os.path.split(localpath)[0])
parent=os.path.split(localpath)[1]
for walker in os.walk(parent):
try:
self.sftp.mkdir(os.path.join(remotepath,walker[0]).replace('\\', '/'))
except:
pass
for file in walker[2]:
self.put(os.path.join(walker[0],file).replace('\\', '/'),os.path.join(remotepath,walker[0],file).replace('\\', '/'))
I found a few shortcomings with the above methods - first, the putter/getter doesn't function in the way you'd expect - if you want to put /foo/bar into /some/folder, you can't as it won't let you put files from a source folder to a different destination folder - the only thing you can do is put /foo/bar into /some/bar. In addition, you have to specify the source as /foo/bar and the destination as /some to end up with /some/bar - I find this confusing as it's not how most operating/ftp systems handle putting/getting/copying/etc. So, I improved on the answers listed above:
If you're going from Windows to Linux:
def put_dir(source, dest):
source = os.path.expandvars(source).rstrip('\\').rstrip('/')
dest = os.path.expandvars(dest).rstrip('\\').rstrip('/')
for root, dirs, files in os.walk(source):
for dir in dirs:
try:
sftp.mkdir(posixpath.join(dest, ''.join(root.rsplit(source))[1:].replace('\\', '/'), dir))
except:
pass
for file in files:
sftp.put(os.path.join(root, file), posixpath.join(dest, ''.join(root.rsplit(source))[1:].replace('\\', '/'), file))
source = '%USERPROFILE%\\Downloads\\'
dest = '/foo/bar'
put_dir(source, dest)
If you're just doing Windows then swap out posixpath.join with os.path.join and remove .replace('\\', '/'):
def put_dir(source, dest):
source = os.path.expandvars(source).rstrip('\\').rstrip('/')
dest = os.path.expandvars(dest).rstrip('\\').rstrip('/')
for root, dirs, files in os.walk(source):
for dir in dirs:
try:
sftp.mkdir(os.path.join(dest, ''.join(root.rsplit(source))[1:], dir))
except:
pass
for file in files:
sftp.put(os.path.join(root, file), os.path.join(dest, ''.join(root.rsplit(source))[1:], file))
source = '%USERPROFILE%\\Downloads\\'
dest = 'foo\\bar'
put_dir(source, dest)
The reason for the try statement is that sftp.mkdir errors out if the folder already exists.
Paramiko does not support recursive operations.
You can use pysftp. It's a wrapper around Paramiko that has more Python-ish look and feel and supports recursive operations. See
pysftp.Connection.put_r()
pysftp.Connection.get_r()
Or you can just base your code on pysftp source code. Or see my answer to Python pysftp get_r from Linux works fine on Linux but not on Windows.
Related
I want to download a directory with unknown contents recursively via SSH and have been trying Paramiko. I have seen several examples how to upload directories but none that covers recursive download.
I can list all items in a directory but haven't been able to find a way of knowing if the item is a file (to download) or a directory (to call recursively).
transport = paramiko.Transport((MY_IP, 22))
transport.connect(username=MY_NAME, password=MY_PASS)
sftp = paramiko.SFTPClient.from_transport(transport)
file_list = sftp.listdir(path='/home/MY_HOME_DIR')
for item in file_list:
# Here is an item name... but is it a file or directory?
print(item)
sftp.close()
transport.close()
So how do I know if an item is a file or if it is a directory?
from stat import S_ISDIR
def isdir(path):
try:
return S_ISDIR(sftp.stat(path).st_mode)
except IOError:
#Path does not exist, so by definition not a directory
return False
...assuming sftp is an open Paramiko SFTP connection.
An old question, but a solution I came up with that works quite well, it's a little bit sloppy (typecasting and slashes and all) - but it does work.
Note this uses fabric.api.local to make the directories in the destination.
def sftp_get_recursive(path, dest, sftp=sftp):
item_list = sftp.listdir(path)
dest = str(dest)
if not os.path.isdir(dest):
local("mkdir %s" % dest)
for item in item_list:
item = str(item)
if is_directory(path + "/" + item, sftp):
sftp_get_recursive(path + "/" + item, dest + "/" + item, sftp)
else:
sftp.get(path + "/" + item, dest + "/" + item)
Paramiko does not support recursive operations.
But it's easy to implement:
import os
from stat import S_ISDIR, S_ISREG
def get_r_portable(sftp, remotedir, localdir):
for entry in sftp.listdir_attr(remotedir):
remotepath = remotedir + "/" + entry.filename
localpath = os.path.join(localdir, entry.filename)
mode = entry.st_mode
if S_ISDIR(mode):
try:
os.mkdir(localpath)
except OSError:
pass
get_r_portable(sftp, remotepath, localpath)
elif S_ISREG(mode):
sftp.get(remotepath, localpath)
You also can use pysftp. It's a wrapper around Paramiko that has more Python-ish look and feel and supports recursive operations. See
pysftp.Connection.put_r()
pysftp.Connection.get_r()
Or see my answer to Python pysftp get_r from Linux works fine on Linux but not on Windows.
But pysftp seems to be an abandoned project, so you better stick with Paramiko.
A small update to Dan LaManna's answer that works in 2021.
import paramiko
import os
from stat import S_ISDIR, S_ISREG
def sftp_get_recursive(path, dest, sftp):
item_list = sftp.listdir_attr(path)
dest = str(dest)
if not os.path.isdir(dest):
os.makedirs(dest, exist_ok=True)
for item in item_list:
mode = item.st_mode
if S_ISDIR(mode):
sftp_get_recursive(path + "/" + item.filename, dest + "/" + item.filename, sftp)
else:
sftp.get(path + "/" + item.filename, dest + "/" + item.filename)
transport = paramiko.Transport((host, port))
transport.connect(username=username, password=password)
sftp = paramiko.SFTPClient.from_transport(transport)
sftp_get_recursive(remote_path, local_path, sftp)
sftp.close()
You can use the stat() method of your sftp object:
http://www.lag.net/paramiko/docs/paramiko.SFTPClient-class.html
stat() method among other attributes returns permissions. d (for example drwxrwxrwx) shows that it is directory.
As example:
dir = oct(sftp.stat(path).st_mode)
print dir[0:2]
output interpritation:
01 fifo
02 character special
04 directory
06 block special
10 regular file
12 symbolic link
14 socket
Here is an answer for 2022.
pysftp is abandoned. paramiko does not implement recursivity for SFTP. So I made a library named sftputil (to make a parallel with ftputil) based on Paramiko.
sftputil implements functionalities such as walk, glob and sync. To copy an entire directory, the easiest way is to use the synchronization feature :
from sftputil import SFTP
sftp = SFTP("hostname", "username", password="password")
# To copy recursively
sftp.sync_pull("remote/path", "local/dir")
# To copy only the files
sftp.sync_pull("remote/path", "local/dir", recursive=False)
If u using Linux or Unix-like. U can use 'file' utility with popen.
Or simple u can use os.path.isdir() =)
I'm successfully able to download CSV files from a folder on Exavault, using the files provided by Exavault, but the download gets saved in a temporary folder on my Mac.
How do I define the directory to save it to? This is an excerpt of the script.
resources_api = ResourcesApi()
resources_api.api_client.configuration.host = ACCOUNT_URL
try:
list_result = resources_api.list_resources(
API_KEY, ACCESS_TOKEN, "/files", offset=0, type='file', name='*.csv')
if list_result.returned_results == 0:
print("Found no files to download")
sys.exit(0)
else:
print("Found {} CSV files to download".format(list_result.returned_results))
except Exception as e:
raise e
print('Exception when calling Api:', str(e))
sys.exit(1)
downloads = []
listed_files = list_result.data
for listed_file in listed_files:
downloads.append("id:{}".format(listed_file.id))
print(listed_file.attributes.path)
try:
downloaded_file = resources_api.download(API_KEY, ACCESS_TOKEN, downloads, download_archive_name="sample-csvs")
print("File(s) downloaded to", os.path(downloaded_file))
Separately there's a resources_api.py file, which might hold the answer, but if I edit that, I'm not sure how I would invoke the changes.
Any help would be appreciated.
When looking at the code of the API, you can see that it is written such that it always creates a temporary folder for you:
# Write the file to the temporary folder configured for our client
fd, path = tempfile.mkstemp(dir=self.api_client.configuration.temp_folder_path)
You could check if you can try to change api_client.configuration.temp_folder_path and see if that works for you.
Or even better, just copy the file to a location of your choice. You can do it with shutil
import shutil
import os
targetFolder= "/your/folder/"
filename = os.path.basename(downloaded_file)
destination = os.path.join(targetFolder,filename )
shutil.copy(downloaded_file, destination)
I am using pysftp connection using walktree() method to list all files as below
with pysftp.Connection(domain, username=USER,
password=PWD, port=port,
cnopts=cnopts) as sftp:
# call back method to list the files for the directory found in walktree
def dir_found_list_files(path):
if path:
files = sftp.listdir_attr(path)
for file in files:
if 'r' in file.longname[:4]:
filelist.append(path+'/'+file.filename)
# call back method when file found in walktree
def file_found(path):
pass
# call back method when unknown found in walktree
def unknown_file(path):
pass
# sftp walk tree to list files
sftp.walktree("/", file_found, dir_found_list_files, unknown_file, recurse=True)
This part works fine. But when any of the folder does not have the permission to read, the walktree method raises the permission exception.
How can I ignore the the access denied folder and continue with walktree for accessible folders?
You cannot. The Connection.walktree does not allow such customization.
But the method is quite trivial, check its source code. Just copy the code over and customize it anyway you need. Something like this (untested):
def robust_walktree(sftp, remotepath, fcallback, dcallback, ucallback, recurse=True)
try:
entries = sftp.listdir(remotepath)
except IOError as e:
if e.errno != errno.EACCES:
raise
else
entries = []
for entry in entries:
pathname = posixpath.join(remotepath, entry)
mode = sftp.stat(pathname).st_mode
if S_ISDIR(mode):
# It's a directory, call the dcallback function
dcallback(pathname)
if recurse:
# now, recurse into it
robust_walktree(sftp, pathname, fcallback, dcallback, ucallback)
elif S_ISREG(mode):
# It's a file, call the fcallback function
fcallback(pathname)
else:
# Unknown file type
ucallback(pathname)
I'm currently putting together a script in Python which will do the following:-
Create a directory in my Dropbox folder called 'Spartacus'
Create a subdirectory in this location with the naming convention of the date and time of creation
Within this directory, create a file called iprecord.txt and information will then be written to this file.
Here is my code thusfar using Python v2.7 on Windows 7:-
import os
import time
import platform
import urllib
current_dir = os.getcwd()
targetname = "Spartacus"
target_dir = os.path.join(current_dir, targetname)
timenow = time.strftime("\%d-%b-%Y %H-%M-%S")
def directoryVerification():
os.chdir(current_dir)
try:
os.mkdir('Spartacus')
except OSError:
pass
try:
os.system('attrib +h Spartacus')
except OSError:
pass
def gatherEvidence():
os.chdir(target_dir)
try:
evidential_dir = os.mkdir(target_dir + timenow)
os.chdir(evidential_dir)
except OSError:
pass
f = iprecord.txt
with f as open:
ip_addr = urllib.urlopen('http://www.biranchi.com/ip.php').read()
f.write("IP Address:\t %s\t %s" % ip_addr, time.strftime("\%d-%b-%Y %H-%M-%S"))
x = directoryVerification()
y = gatherEvidence()
I keep on getting an error in line 26 whereby it cannot resolve the full path to the dynamically named directory (date and time) one. I've printed out the value of 'evidential_dir' and it shows as being Null.
Any pointers as to where I am going wrong? Thanks
PS: Any other advice on my code to improve it would be appreciated
PPS: Any advice on how to locate the default directory for 'Dropbox'? Is there a way of scanning a file system for a directory called 'Dropbox' and capturing the path?
os.mkdir() does not return a pathname as you might be thinking. It seems like you do inconsistent methods of the same thing at different spots of your code.
Try this:
evidential_dir = os.path.join(target_dir, timenow)
os.mkdir(evidential_dir)
And fix your other line:
f = "iprecord.txt"
os.mkdir doesn't return anything.
evidential_dir = target_dir + timenow
try:
os.mkdir(evidential_dir)
except OSError:
pass
os.chdir(evidential_dir)
Have you ever tried this feedback calling an external zip.py script to work? My CGITB does not show any error messages. It simply did not invoke external .py script to work. It simply skipped over to gush. I should be grateful if you can assist me in making this zip.py callable in feedback.py.
Regards. David
#**********************************************************************
# Description:
# Zips the contents of a folder.
# Parameters:
# 0 - Input folder.
# 1 - Output zip file. It is assumed that the user added the .zip
# extension.
#**********************************************************************
# Import modules and create the geoprocessor
#
import sys, zipfile, arcgisscripting, os, traceback
gp = arcgisscripting.create()
# Function for zipping files. If keep is true, the folder, along with
# all its contents, will be written to the zip file. If false, only
# the contents of the input folder will be written to the zip file -
# the input folder name will not appear in the zip file.
#
def zipws(path, zip, keep):
path = os.path.normpath(path)
# os.walk visits every subdirectory, returning a 3-tuple
# of directory name, subdirectories in it, and filenames
# in it.
#
for (dirpath, dirnames, filenames) in os.walk(path):
# Iterate over every filename
#
for file in filenames:
# Ignore .lock files
#
if not file.endswith('.lock'):
gp.AddMessage("Adding %s..." % os.path.join(path, dirpath, file))
try:
if keep:
zip.write(os.path.join(dirpath, file),
os.path.join(os.path.basename(path),
os.path.join(dirpath, file)[len(path)+len(os.sep):]))
else:
zip.write(os.path.join(dirpath, file),
os.path.join(dirpath[len(path):], file))
except Exception, e:
gp.AddWarning(" Error adding %s: %s" % (file, e))
return None
if __name__ == '__main__':
try:
# Get the tool parameter values
#
infolder = gp.GetParameterAsText(0)
outfile = gp.GetParameterAsText(1)
# Create the zip file for writing compressed data. In some rare
# instances, the ZIP_DEFLATED constant may be unavailable and
# the ZIP_STORED constant is used instead. When ZIP_STORED is
# used, the zip file does not contain compressed data, resulting
# in large zip files.
#
try:
zip = zipfile.ZipFile(outfile, 'w', zipfile.ZIP_DEFLATED)
zipws(infolder, zip, True)
zip.close()
except RuntimeError:
# Delete zip file if exists
#
if os.path.exists(outfile):
os.unlink(outfile)
zip = zipfile.ZipFile(outfile, 'w', zipfile.ZIP_STORED)
zipws(infolder, zip, True)
zip.close()
gp.AddWarning(" Unable to compress zip file contents.")
gp.AddMessage("Zip file created successfully")
except:
# Return any python specific errors as well as any errors from the geoprocessor
#
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = "PYTHON ERRORS:\nTraceback Info:\n" + tbinfo +
"\nError Info:\n " + str(sys.exc_type) +
": " + str(sys.exc_value) + "\n"
gp.AddError(pymsg)
msgs = "GP ERRORS:\n" + gp.GetMessages(2) + "\n"
gp.AddError(msgs)
zip() is a built-in function in Python. Therefore it is a bad practice to use zip as a variable name. zip_ can be used instead of.
execfile() function reads and executes a Python script.
It is probably that you actually need just import zip_ in feedback.py instead of execfile().
Yay ArcGIS.
Just to clarify how are you trying to call this script using popen, can you post some code?
If your invoking this script via another script in the ArcGIS environment, then the thing is, when you use Popen the script wont be invoked within the ArcGIS environment, instead it will be invoked within windows. So you will loose all real control over it.
Also just another ArcGIS comment you never initalize a license for the geoprocessor.
My suggestion refactor your code, into a module function that simply attempts to zip the files, if it fails print the message out to ArcGIS.
If you want post how you are calling it, and how this is being run.