I am trying to find duplicates between two folders and copy only unique image files to the 'dest' folder. I can copy all the non-dupes using the code below, however it doesn't maintain the source directory structure. I think OS.walk returns 3 tuples, but they aren't linked so not sure how to re-construct the sub dir?
Example:
import shutil, os
from difPy import dif
source = input('Input source folder:')
dest = input('Input backup \ destination folder:')
ext = ('.jpg','.jpeg','.gif','.JPG','.JPEG','.GIF')
search = dif(source, dest)
result = search.result
result
dupes = []
srcfiles = []
filecount = []
failed = []
removed = []
for i in result.values():
dupes.append(i['location'])
for dirpath, subdirs, files in os.walk(source):
for x in files:
if x.endswith(ext):
srcfiles.append(os.path.join(dirpath, x))
for f in srcfiles:
if f not in dupes:
shutil.copy(f, dest)
print('File copied successfully - '+f)
filecount.append(f)
else:
print('File not copied successfully !!!! - '+f)
failed.append(f)
I have also tried using the shutil.copytree function with an ignore list, however it requires a new folder and can't get the ignore list function to work
shutil.copytree example:
for i in result.values():
df = []
df.append(i['filename'])
def ignorelist(source, df):
return [f for f in df if os.path.isfile(os.path.join(source, f))]
shutil.copytree(source, destnew, ignore=ignorelist)
This function ignorelist should do the trick:
import shutil, os
from difPy import dif
source = input('Input source folder:')
dest = input('Input backup \ destination folder:')
ext = ('.jpg','.jpeg','.gif')
search = dif(source, dest)
dupes = [value['location'] for value in search.result.values()]
def ignorelist(source, files):
return [file for file in files
if (os.path.isfile(os.path.join(source, file))
and (os.path.join(source, file) in dupes
or not file.lower().endswith(ext)))]
shutil.copytree(source, dest, ignore=ignorelist)
And the other "more manual" way would be
import shutil, os
from difPy import dif
source = input('Input source folder:').rstrip('/\\')
dest = input('Input backup \ destination folder:').rstrip('/\\')
ext = ('.jpg','.jpeg','.gif')
search = dif(source, dest)
dupes = [value['location'] for value in search.result.values()]
srcfiles = []
copied = []
failed = []
skipped = []
for dirpath, subdirs, files in os.walk(source):
for file in files:
if file.lower().endswith(ext):
srcfile = os.path.join(dirpath,file)
srcfiles.append(srcfile)
if srcfile in dupes:
print('File not copied (duplicate) - '+srcfile)
skipped.append(srcfile)
else:
try:
destfile = os.path.join(dest,srcfile[len(source)+1:])
os.makedirs(os.path.dirname(destfile), exist_ok=True)
shutil.copy(srcfile,destfile)
print('File copied successfully - '+srcfile)
copied.append(srcfile)
except Exception as err:
print('File not copied (error %s) - %s' % (str(err),srcfile))
failed.append(f)
I have changed some variable names to make them more descriptive. And what you call failed is really just a list of files that are not copied because they are duplicates rather than files whose copying was attempted but failed.
import shutil, os
from difPy import dif
source = input('Input source folder: ')
dest = input('Input backup \ destination folder: ')
# Remove trailing path separators if they exist:
if source.endswith(('/', '\\')):
source = source[:-1]
if dest.endswith(('/', '\\')):
dest = dest[:-1]
# Use the correct path separator to
# ensure correct matching with dif results:
if os.sep == '/':
source = source.replace('\\', os.sep)
elif os.sep == '\\':
source = source.replace('/', os.sep)
source_directory_length = len(source) + 1
ext = ('.jpg','.jpeg','.gif','.JPG','.JPEG','.GIF')
search = dif(source, dest)
result = search.result
# Set comprehension:
dupes = {duplicate['location'] for duplicate in result.values()}
copied = []
not_copied = []
for dirpath, subdirs, files in os.walk(source):
for file in files:
if file.endswith(ext):
source_path = os.path.join(dirpath, file)
if source_path not in dupes:
# get subdirectory of source directory that this file is in:
file_length = len(file) + 1
# Get subdirectory relative to the source directory:
subdirectory = source_path[source_directory_length:-file_length]
if subdirectory:
dest_directory = os.path.join(dest, subdirectory)
# ensure directory exists:
os.makedirs(dest_directory, exist_ok=True)
else:
dest_directory = dest
dest_path = os.path.join(dest_directory, file)
shutil.copy(source_path, dest_path)
print('File copied successfully -', source_path)
copied.append(source_path)
else:
print('File not copied -', source_path)
not_copied.append(source_path)
I'm using this Python code in Windows:
shutil.move(documents_dir + "\\" + file_name, documents_dir + "\\backup\\"
+ subdir_name + "\\" + file_name)
When this code is called more times, it overwrites the destination file. I would like to move the file
and if the destination already exists, to rename it
e.g. file_name = foo.pdf
and in backup folder will be foo.pdf, foo(1).pdf, foo(2).pdf etc. or similarly e.g. with dashes
foo-1.pdf, foo-2.pdf etc.
You could just check with os.path.exists() as you're going.
import os
import shutil
file_name = 'test.csv'
documents_dir = r'C:\BR\Test'
subdir_name = 'test'
# using os.path.join() makes your code easier to port to another OS
source = os.path.join(documents_dir, file_name)
dest = os.path.join(documents_dir, 'backup', subdir_name, file_name)
num = 0
# loop until we find a file that doesn't exist
while os.path.exists(dest):
num += 1
# use rfind to find your file extension if there is one
period = file_name.rfind('.')
# this ensures that it will work with files without extensions
if period == -1:
period = len(file_name)
# create our new destination
# we could extract the number and increment it
# but this allows us to fill in the gaps if there are any
# it has the added benefit of avoiding errors
# in file names like this "test(sometext).pdf"
new_file = f'{file_name[:period]}({num}){file_name[period:]}'
dest = os.path.join(documents_dir, 'backup', subdir_name, new_file)
shutil.move(source, dest)
Or since this is probably used in a loop you could just drop it into a function.
import os
import shutil
def get_next_file(file_name, dest_dir):
dest = os.path.join(dest_dir, file_name)
num = 0
while os.path.exists(dest):
num += 1
period = file_name.rfind('.')
if period == -1:
period = len(file_name)
new_file = f'{file_name[:period]}({num}){file_name[period:]}'
dest = os.path.join(dest_dir, new_file)
return dest
file_name = 'test.csv'
documents_dir = r'C:\BR\Test'
subdir_name = 'test'
source = os.path.join(documents_dir, file_name)
dest = get_next_file(file_name, os.path.join(documents_dir, 'backup', subdir_name))
shutil.move(source, dest)
I have a script that takes a file from a form, renames it and uploads it to a folder and inserts record into a database. I would like to add the functionality where before the file is saved, it checks the upload folder to determine if the filename exists. If it does exist, renames the file in a loop and then saves the file.
What I have currently:
file = request.files['xx']
extension = os.path.splitext(file.filename)[1]
xx = str(uuid.uuid4()) + extension
## if xx exists .. xx = str(uuid.uuid4()) + extension.. loop endlessly.
file.save(os.path.join(app.config['UPLOAD_FOLDER'], xx)
Haven't tested this yet but you can use os.path.isfile() to check if a file already exists (for directories, use os.path.exists).
import os
def save():
file = request.files['xx']
extension = os.path.splitext(file.filename)[1]
xx = generate_filename(extension)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], xx))
def generate_filename(extension):
xx = str(uuid.uuid4()) + extension
if os.path.isfile(os.path.join(app.config['UPLOAD_FOLDER'], xx)):
return generate_filename(extension)
return xx
quick and dirty, haven't tested this. using the check and rename function recursively to add "_1", "_2" etc to the end of the file name until it can be saved.
def check_and_rename(file, add=0):
original_file = file
if add != 0:
split = file.split(".")
part_1 = split[0] + "_" + str(add)
file = ".".join([part1, split[1]])
if not os.path.isfile(file):
# save here
else:
check_and_rename(original_file, add+=1)
This will check if a file exist and generate a new name that does not exist by increasing a number:
from os import path
def check_file(filePath):
if path.exists(filePath):
numb = 1
while True:
newPath = "{0}_{2}{1}".format(*path.splitext(filePath) + (numb,))
if path.exists(newPath):
numb += 1
else:
return newPath
return filePath
Improving on N.Walters answer, but so you have a function that just parses the file_path and gives you a valid one back and using the internal Path class:
import os
from pathlib import Path
def check_and_rename(file_path: Path, add: int = 0) -> Path:
original_file_path = file_path
if add != 0:
file_path = file_path.with_stem(file_path.stem + "_" + str(add))
if not os.path.isfile(file_path):
return file_path
else:
return check_and_rename(original_file_path, add + 1)
Have you tried to use the glob Module, it provides an interface similar to ls, you can use it as it follows:
import os
import glob
file_list = glob.glob('my_file')
if len(file_list) > 0:
os.rename('my_file', 'new_name')
if not os.path.isfile(xx):
file.save(os.path.join(app.config['UPLOAD_FOLDER'], xx)
else:
print("File does not exist")
Run the following code from a directory that contains a directory named bar (containing one or more files) and a directory named baz (also containing one or more files). Make sure there is not a directory named foo.
import shutil
shutil.copytree('bar', 'foo')
shutil.copytree('baz', 'foo')
It will fail with:
$ python copytree_test.py
Traceback (most recent call last):
File "copytree_test.py", line 5, in <module>
shutil.copytree('baz', 'foo')
File "/System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/shutil.py", line 110, in copytree
File "/System/Library/Frameworks/Python.framework/Versions/2.5/lib/python2.5/os.py", line 172, in makedirs
OSError: [Errno 17] File exists: 'foo'
I want this to work the same way as if I had typed:
$ mkdir foo
$ cp bar/* foo/
$ cp baz/* foo/
Do I need to use shutil.copy() to copy each file in baz into foo? (After I've already copied the contents of 'bar' into 'foo' with shutil.copytree()?) Or is there an easier/better way?
Here's a solution that's part of the standard library:
from distutils.dir_util import copy_tree
copy_tree("/a/b/c", "/x/y/z")
See this similar question.
Copy directory contents into a directory with python
Reference - https://docs.python.org/3/distutils/apiref.html#distutils.dir_util.copy_tree
This limitation of the standard shutil.copytree seems arbitrary and annoying. Workaround:
import os, shutil
def copytree(src, dst, symlinks=False, ignore=None):
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
if os.path.isdir(s):
shutil.copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
Note that it's not entirely consistent with the standard copytree:
it doesn't honor symlinks and ignore parameters for the root directory of the src tree;
it doesn't raise shutil.Error for errors at the root level of src;
in case of errors during copying of a subtree, it will raise shutil.Error for that subtree instead of trying to copy other subtrees and raising single combined shutil.Error.
Python 3.8 introduced the dirs_exist_ok argument to shutil.copytree:
Recursively copy an entire directory tree rooted at src to a directory named dst and return the destination directory. dirs_exist_ok dictates whether to raise an exception in case dst or any missing parent directory already exists.
Therefore, with Python 3.8+ this should work:
import shutil
shutil.copytree('bar', 'foo') # Will fail if `foo` exists
shutil.copytree('baz', 'foo', dirs_exist_ok=True) # Fine
In slight improvement on atzz's answer to the function where the above function always tries to copy the files from source to destination.
def copytree(src, dst, symlinks=False, ignore=None):
if not os.path.exists(dst):
os.makedirs(dst)
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
if os.path.isdir(s):
copytree(s, d, symlinks, ignore)
else:
if not os.path.exists(d) or os.stat(s).st_mtime - os.stat(d).st_mtime > 1:
shutil.copy2(s, d)
In my above implementation
Creating the output directory if not already exists
Doing the copy directory by recursively calling my own method.
When we come to actually copying the file I check if the file is modified then only
we should copy.
I am using above function along with scons build. It helped me a lot as every time when I compile I may not need to copy entire set of files.. but only the files which are modified.
A merge one inspired by atzz and Mital Vora:
#!/usr/bin/python
import os
import shutil
import stat
def copytree(src, dst, symlinks = False, ignore = None):
if not os.path.exists(dst):
os.makedirs(dst)
shutil.copystat(src, dst)
lst = os.listdir(src)
if ignore:
excl = ignore(src, lst)
lst = [x for x in lst if x not in excl]
for item in lst:
s = os.path.join(src, item)
d = os.path.join(dst, item)
if symlinks and os.path.islink(s):
if os.path.lexists(d):
os.remove(d)
os.symlink(os.readlink(s), d)
try:
st = os.lstat(s)
mode = stat.S_IMODE(st.st_mode)
os.lchmod(d, mode)
except:
pass # lchmod not available
elif os.path.isdir(s):
copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
Same behavior as shutil.copytree, with symlinks and ignore parameters
Create directory destination structure if non existant
Will not fail if dst already exists
docs explicitly state that destination directory should not exist:
The destination directory, named by dst, must not already exist; it will be created as well as missing parent directories.
I think your best bet is to os.walk the second and all consequent directories, copy2 directory and files and do additional copystat for directories. After all that's precisely what copytree does as explained in the docs. Or you could copy and copystat each directory/file and os.listdir instead of os.walk.
This is inspired from the original best answer provided by atzz, I just added replace file / folder logic. So it doesn't actually merge, but deletes the existing file/ folder and copies the new one:
import shutil
import os
def copytree(src, dst, symlinks=False, ignore=None):
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
if os.path.exists(d):
try:
shutil.rmtree(d)
except Exception as e:
print e
os.unlink(d)
if os.path.isdir(s):
shutil.copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
#shutil.rmtree(src)
Uncomment the rmtree to make it a move function.
Here is my pass at the problem. I modified the source code for copytree to keep the original functionality, but now no error occurs when the directory already exists. I also changed it so it doesn't overwrite existing files but rather keeps both copies, one with a modified name, since this was important for my application.
import shutil
import os
def _copytree(src, dst, symlinks=False, ignore=None):
"""
This is an improved version of shutil.copytree which allows writing to
existing folders and does not overwrite existing files but instead appends
a ~1 to the file name and adds it to the destination path.
"""
names = os.listdir(src)
if ignore is not None:
ignored_names = ignore(src, names)
else:
ignored_names = set()
if not os.path.exists(dst):
os.makedirs(dst)
shutil.copystat(src, dst)
errors = []
for name in names:
if name in ignored_names:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
i = 1
while os.path.exists(dstname) and not os.path.isdir(dstname):
parts = name.split('.')
file_name = ''
file_extension = parts[-1]
# make a new file name inserting ~1 between name and extension
for j in range(len(parts)-1):
file_name += parts[j]
if j < len(parts)-2:
file_name += '.'
suffix = file_name + '~' + str(i) + '.' + file_extension
dstname = os.path.join(dst, suffix)
i+=1
try:
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
os.symlink(linkto, dstname)
elif os.path.isdir(srcname):
_copytree(srcname, dstname, symlinks, ignore)
else:
shutil.copy2(srcname, dstname)
except (IOError, os.error) as why:
errors.append((srcname, dstname, str(why)))
# catch the Error from the recursive copytree so that we can
# continue with other files
except BaseException as err:
errors.extend(err.args[0])
try:
shutil.copystat(src, dst)
except WindowsError:
# can't copy file access times on Windows
pass
except OSError as why:
errors.extend((src, dst, str(why)))
if errors:
raise BaseException(errors)
Here is a version that expects a pathlib.Path as input.
# Recusively copies the content of the directory src to the directory dst.
# If dst doesn't exist, it is created, together with all missing parent directories.
# If a file from src already exists in dst, the file in dst is overwritten.
# Files already existing in dst which don't exist in src are preserved.
# Symlinks inside src are copied as symlinks, they are not resolved before copying.
#
def copy_dir(src, dst):
dst.mkdir(parents=True, exist_ok=True)
for item in os.listdir(src):
s = src / item
d = dst / item
if s.is_dir():
copy_dir(s, d)
else:
shutil.copy2(str(s), str(d))
Note that this function requires Python 3.6, which is the first version of Python where os.listdir() supports path-like objects as input. If you need to support earlier versions of Python, you can replace listdir(src) by listdir(str(src)).
Here is my version of the same task::
import os, glob, shutil
def make_dir(path):
if not os.path.isdir(path):
os.mkdir(path)
def copy_dir(source_item, destination_item):
if os.path.isdir(source_item):
make_dir(destination_item)
sub_items = glob.glob(source_item + '/*')
for sub_item in sub_items:
copy_dir(sub_item, destination_item + '/' + sub_item.split('/')[-1])
else:
shutil.copy(source_item, destination_item)
Here is a version inspired by this thread that more closely mimics distutils.file_util.copy_file.
updateonly is a bool if True, will only copy files with modified dates newer than existing files in dst unless listed in forceupdate which will copy regardless.
ignore and forceupdate expect lists of filenames or folder/filenames relative to src and accept Unix-style wildcards similar to glob or fnmatch.
The function returns a list of files copied (or would be copied if dryrun if True).
import os
import shutil
import fnmatch
import stat
import itertools
def copyToDir(src, dst, updateonly=True, symlinks=True, ignore=None, forceupdate=None, dryrun=False):
def copySymLink(srclink, destlink):
if os.path.lexists(destlink):
os.remove(destlink)
os.symlink(os.readlink(srclink), destlink)
try:
st = os.lstat(srclink)
mode = stat.S_IMODE(st.st_mode)
os.lchmod(destlink, mode)
except OSError:
pass # lchmod not available
fc = []
if not os.path.exists(dst) and not dryrun:
os.makedirs(dst)
shutil.copystat(src, dst)
if ignore is not None:
ignorepatterns = [os.path.join(src, *x.split('/')) for x in ignore]
else:
ignorepatterns = []
if forceupdate is not None:
forceupdatepatterns = [os.path.join(src, *x.split('/')) for x in forceupdate]
else:
forceupdatepatterns = []
srclen = len(src)
for root, dirs, files in os.walk(src):
fullsrcfiles = [os.path.join(root, x) for x in files]
t = root[srclen+1:]
dstroot = os.path.join(dst, t)
fulldstfiles = [os.path.join(dstroot, x) for x in files]
excludefiles = list(itertools.chain.from_iterable([fnmatch.filter(fullsrcfiles, pattern) for pattern in ignorepatterns]))
forceupdatefiles = list(itertools.chain.from_iterable([fnmatch.filter(fullsrcfiles, pattern) for pattern in forceupdatepatterns]))
for directory in dirs:
fullsrcdir = os.path.join(src, directory)
fulldstdir = os.path.join(dstroot, directory)
if os.path.islink(fullsrcdir):
if symlinks and dryrun is False:
copySymLink(fullsrcdir, fulldstdir)
else:
if not os.path.exists(directory) and dryrun is False:
os.makedirs(os.path.join(dst, dir))
shutil.copystat(src, dst)
for s,d in zip(fullsrcfiles, fulldstfiles):
if s not in excludefiles:
if updateonly:
go = False
if os.path.isfile(d):
srcdate = os.stat(s).st_mtime
dstdate = os.stat(d).st_mtime
if srcdate > dstdate:
go = True
else:
go = True
if s in forceupdatefiles:
go = True
if go is True:
fc.append(d)
if not dryrun:
if os.path.islink(s) and symlinks is True:
copySymLink(s, d)
else:
shutil.copy2(s, d)
else:
fc.append(d)
if not dryrun:
if os.path.islink(s) and symlinks is True:
copySymLink(s, d)
else:
shutil.copy2(s, d)
return fc
The previous solution has some issue that src may overwrite dst without any notification or exception.
I add a predict_error method to predict errors before copy.copytree mainly base on Cyrille Pontvieux's version.
Using predict_error to predict all errors at first is best, unless you like to see exception raised one by another when execute copytree until fix all error.
def predict_error(src, dst):
if os.path.exists(dst):
src_isdir = os.path.isdir(src)
dst_isdir = os.path.isdir(dst)
if src_isdir and dst_isdir:
pass
elif src_isdir and not dst_isdir:
yield {dst:'src is dir but dst is file.'}
elif not src_isdir and dst_isdir:
yield {dst:'src is file but dst is dir.'}
else:
yield {dst:'already exists a file with same name in dst'}
if os.path.isdir(src):
for item in os.listdir(src):
s = os.path.join(src, item)
d = os.path.join(dst, item)
for e in predict_error(s, d):
yield e
def copytree(src, dst, symlinks=False, ignore=None, overwrite=False):
'''
would overwrite if src and dst are both file
but would not use folder overwrite file, or viceverse
'''
if not overwrite:
errors = list(predict_error(src, dst))
if errors:
raise Exception('copy would overwrite some file, error detail:%s' % errors)
if not os.path.exists(dst):
os.makedirs(dst)
shutil.copystat(src, dst)
lst = os.listdir(src)
if ignore:
excl = ignore(src, lst)
lst = [x for x in lst if x not in excl]
for item in lst:
s = os.path.join(src, item)
d = os.path.join(dst, item)
if symlinks and os.path.islink(s):
if os.path.lexists(d):
os.remove(d)
os.symlink(os.readlink(s), d)
try:
st = os.lstat(s)
mode = stat.S_IMODE(st.st_mode)
os.lchmod(d, mode)
except:
pass # lchmod not available
elif os.path.isdir(s):
copytree(s, d, symlinks, ignore)
else:
if not overwrite:
if os.path.exists(d):
continue
shutil.copy2(s, d)
Try This:
import os,shutil
def copydir(src, dst):
h = os.getcwd()
src = r"{}".format(src)
if not os.path.isdir(dst):
print("\n[!] No Such directory: ["+dst+"] !!!")
exit(1)
if not os.path.isdir(src):
print("\n[!] No Such directory: ["+src+"] !!!")
exit(1)
if "\\" in src:
c = "\\"
tsrc = src.split("\\")[-1:][0]
else:
c = "/"
tsrc = src.split("/")[-1:][0]
os.chdir(dst)
if os.path.isdir(tsrc):
print("\n[!] The Directory Is already exists !!!")
exit(1)
try:
os.mkdir(tsrc)
except WindowsError:
print("\n[!] Error: In[ {} ]\nPlease Check Your Dirctory Path !!!".format(src))
exit(1)
os.chdir(h)
files = []
for i in os.listdir(src):
files.append(src+c+i)
if len(files) > 0:
for i in files:
if not os.path.isdir(i):
shutil.copy2(i, dst+c+tsrc)
print("\n[*] Done ! :)")
copydir("c:\folder1", "c:\folder2")
I couldn't edit the "Boris Dalstein" answer above so here is the improved version of this code:
EDIT on the improvements made:
The input args could be str path or pathlib.Path object. Type hint will help.
If the source is a directory, it will create that directory as well
types are defined for local variables so no warning by the IDE
# Recusively copies the content of the directory src to the directory dst.
# If dst doesn't exist, it is created, together with all missing parent directories.
# If a file from src already exists in dst, the file in dst is overwritten.
# Files already existing in dst which don't exist in src are preserved.
# Symlinks inside src are copied as symlinks, they are not resolved before copying.
#
def copy_dir(source: Union[str, pathlib.Path], destination: Union[str, pathlib.Path]):
destination_path: pathlib.Path
if isinstance(source, str):
source_path = pathlib.Path(source)
elif isinstance(source, pathlib.Path):
source_path = source
if isinstance(destination, str):
destination_path = pathlib.Path(destination)
elif isinstance(destination, pathlib.Path):
destination_path = destination
destination_path.mkdir(parents=True, exist_ok=True)
if source_path.is_dir():
destination_path = destination_path.joinpath(source_path.name)
destination_path.mkdir(parents=True, exist_ok=True)
for item in os.listdir(source_path):
s: pathlib.Path = source_path / item
d: pathlib.Path = destination_path / item
if s.is_dir():
copy_dir(s, d)
else:
shutil.copy2(str(s), str(d))
i would assume fastest and simplest way would be have python call the system commands...
example..
import os
cmd = '<command line call>'
os.system(cmd)
Tar and gzip up the directory.... unzip and untar the directory in the desired place.
yah?