I have multiple files in 7 different folder directories. All of these files have the same name, and I want to combine those files with the same name as one file, in another directory
import os
from itertools import chain
paths = (r'C:/Users/Test_folder/Input/', r'C:/Users/Test_folder/Input_2/')
for path, dirs, files in chain.from_iterable(os.walk(path) for path in paths):
for fname in paths:
for line in fname:
f = open(os.path.join(r'C:/Users/Test_folder/Test_output/', os.path.basename(fname)), 'a')
f.write ('{:}\n'.format(line))
f.close()
Error:
f = open(os.path.join(r'C:/Users/Test_folder/Test_output/', os.path.basename(fname)), 'a')
IOError: [Errno 13] Permission denied: 'C:/Users/Test_folder/Test_output/'
>>>
For issue of permisson denied
with open(os.path.join('type filename here' , os.path.basename(line)), 'w')
Or
for filename in os.listdir(src):
path = os.path.join(src, filename)
with open(path, "r") as inputFile:
content = inputFile.read()
The logic of your code is wrong:
for fname in paths should be for fname in files
for line in fname will not read the file fname line by line as fname is a string, not a file object
The permission error is due to that your code try to open a directory for appending.
Try:
import os
from itertools import chain
paths = (r'C:/Users/Test_folder/Input/', r'C:/Users/Test_folder/Input_2/')
for path, dirs, files in chain.from_iterable(os.walk(path) for path in paths):
for fname in files:
with open(os.path.join(path, fname)) as fin, open(os.path.join('C:/Users/Test_folder/Test_output/', fname), 'a') as fout:
fout.write(fin.read())
If you're using windows, re-run your ide as administrator.
Related
I'm trying to loop through files in multiple subdirectories in order to :
1- Add some text inside the files (ending with .ext)
2- Change the extension of each file from .ext to .ext2
The script works fine when I have only one subdir in the main directory, but when I try to run the script on multiple subdirs it says:
line 8, in
with open(name, "r") as f:
FileNotFoundError: [Errno 2] No such file or directory: "here the name of the subdir"
import os
directory = 'C:\\Users\\folder\\subfolders'
for dir, subdirs, files in os.walk(directory):
for name in files:
if name.endswith((".ext")):
with open(name, "r") as f:
XMLContent = f.readlines()
XMLContent.insert(6, '<XMLFormat>\n')
XMLContent.insert(40, '\n</XMLFormat>')
with open(name, "w") as f:
XMLContent = "".join(XMLContent)
f.write(XMLContent)
os.rename(os.path.join(dir, name), os.path.join(dir, name[:name.index('.ext')] +".ext1"))
Above is a screenshot of the sub dirs I have in the folder (1.Modified).
I've also created a new folder called all and put in it three folders and for each folder, I've created 2 files of .ext type.
So, I was able to write inside each file of them and change its name as well.
import os
for root, dirs, files in os.walk("/Users/ghaith/Desktop/test/all"):
for file in files:
if file.endswith('.ext'):
path = root + '/' + file
with open(path, "r") as f:
content = f.readlines()
content.insert(1, '<XMLFormat>\n')
content.insert(3, '\n</XMLFormat>')
with open(path, "w") as f:
content = "".join(content)
f.write(content)
os.rename(path, path+'2')
Output:
< XMLFormat >
< /XMLFormat >
you need to pass the directory to open the file
with open(os.path.join(directory, name), "r") as f:
But, I think the best way is use the os.listdir() to loop in the directory
for item in os.listdir(directory):
if item.endswith(".ext"):
with open(os.path.join(directory, item), "r") as r:
I want to combine multiple file from different folder data in one file but only same file name is all folder
Script:
import os
filenames = [os.path.join('C:/Users/Vishnu/Desktop/Test_folder/Input/','*.txt'), os.path.join('C:/Users/Vishnu/Desktop/Test_folder/Output/','*.txt')]
f = open(r'C:/Users/Vishnu/Desktop/Test_output/', 'wb')
for fname in filenames:
with open(fname) as infile:
for line in infile:
f.write(line)
Getting Error:
f = open(r"C:/Users/Vishnu/Desktop/Test_output/", "wb")
IOError: [Errno 13] Permission denied: 'C:/Users/Vishnu/Desktop/Test_output/'
>>>
After Modification
import os
import glob
import os.mkdir
filenames = [glob.globos(mkdir(os.path.join(os.path.expanduser('~'),'C:', 'Users' , 'Vishnu' ,'Desktop','Test_folder','Input','*.txt'))), glob.glob(os.mkdir(os.path.join(os.path.expanduser('~'),'C:', 'Users' , 'Vishnu' , 'Desktop','Test_folder','Output','*.txt')))]
filenames[0].extend(filenames[1])
filenames=filenames[0]
if( not os.path.isdir(os.path.join(os.path.expanduser('~'),'C:', 'Users' , 'Vishnu' , 'Desktop' ,'Test_folder', 'Test_output'))):
os.mkdir(os.path.join(os.path.expanduser('~'),'C:', 'Users' , 'Vishnu' , 'Desktop' ,'Test_folder', 'Test_output'))
for fname in filenames:
with open(fname) as file:
for line in file.readlines():
f = open(os.mkdir(os.path.join(os.path.expanduser('~'), 'Desktop', 'Test_output')),'{:}.txt'.format(os.path.split(fname)[-1] ), 'a+')
f.write(line)
f.close()
Firstly, you are trying to open the folder itself. Secondly, we have to close the file everytime we read it to avoid Permission issues
I tried this code. It should work now
import os
import glob #So that * in directory listing can be interpretted as all filenames
filenames = [glob.glob(os.path.join(os.path.expanduser('~'),'Desktop','Test_folder','Input','*.txt')), glob.glob(os.path.join(os.path.expanduser('~'),'Desktop','Test_folder','Output','*.txt'))]
filenames[0].extend(filenames[1])
filenames=filenames[0]
if( not os.path.isdir(os.path.join(os.path.expanduser('~'), 'Desktop', 'Test_output'))):
os.mkdir(os.path.join(os.path.expanduser('~'), 'Desktop', 'Test_output'))
for fname in filenames:
with open(fname) as file:
for line in file.readlines():
f = open(os.path.join(os.path.expanduser('~'), 'Desktop', 'Test_output','{:}.txt'.format(os.path.split(fname)[-1] )), 'a+')
f.write(line)
f.close() #This should take care of the permissions issue
As I understand it, you have multiple files in multiple directories. Some of these files have the same name, and you want to combine those files with the same name as one file, in another directory.
The first task, is to figure out what are all the unique file names in the directories; and collect the paths to those unique files.
import os, glob
from collections import defaultdict
dirs = [r'/foo/bar/directory_1', r'/foo/bar/directory_2']
file_pattrn = r'*.txt'
unique_files = defaultdict(list)
for d in dirs:
for i in glob.iglob(os.path.join(d, file_pattrn)):
unique_files[os.path.basename(i)].append(i)
Now we have a dictionary, where the key is the filename, and the value is a list of all the paths to that file.
Once we have that, then we just loop through them, to create our new files:
destination = r'/foo/bar/new_directory'
for unique_filename, copies in unique_files.items():
with open(os.path.join(destination, unique_filename), 'w') as f:
for copy in copies:
with open(copy, 'r') as cp:
for line in cp:
f.write(line)
I'm trying to write data from a fast5 file to a txt file. I'm able to do so by going into the directory where the files are and using this code:
for filename in os.listdir(os.getcwd()):
if filename.endswith('.fast5'):
with h5py.File(filename, 'r') as hdf:
with open(new_txt, 'a') as myfile:
myfile.write('%s \t' % (filename))
However, I am now trying to access the files through the main directory, by looping through specific subfolders where the files are located and accessing the files that way, by using this code:
for root, dirs, files in os.walk(path):
for d in dirs:
if d.startswith('pass') or d.startswith('fail')
for rootfolder, blankdirs, fast5files in os.walk(d):
for filename in fast5files:
if filename.endswith('.fast5'):
with h5py.File(filename, 'r') as hdf:
with open(new_txt, 'a') as myfile:
myfile.write('%s \t' % (filename))
This code gives the error:
IOError: Unable to open file (Unable to open file: name = 'minion2_chip61_re_n90_yt2_2644_1_ch108_file0_strand.fast5', errno = 2, error message = 'no such file or directory', flags = 0, o_flags = 0)
which confuses me since it is able to get the filename, but somehow not able to read from it, which it could under the original code. The error occurs at this line:
with h5py.File(filename, 'r') as hdf:
Why is h5py not able to open/read the file in this way?
you need to add the directory os.walk is currently traversing to the filename:
....
if filename.endswith('.fast5'):
hdf5_path = os.path.join(root, filename)
with h5py.File(hdf5_path, 'r') as hdf:
...
I am new to python. I am trying to read multiple files one by one from subfolders, do some processing and output. Below is my code:
import os
rootdir = 'dir'
for subdir, dirs, files in os.walk(rootdir):
for fname in files:
print os.path.join(subdir, fname)
f = open(fname, 'r')
lines = f.readlines()
f.close()
f = open(fname, 'w')
for line in lines:
f['X1'] = f['X1'].astype(str)
But I am having the following error:
IOError: [Errno2] No such file or directory : 'test.txt'
Your problem is that you are doing this:
os.path.join(subdir, fname)
But you are not assigning it to a variable, so when you do this:
f=open(fname,'r')
You are still only using the filename.
You should do this:
file_path = os.path.join(subdir, fname)
So now you actually have the file path. Then this:
f=open(file_path,'r')
With the below code I receive IOError: [Errno 13] Permission denied, and I know this is due to the output directory being a sub-folder of the input directory:
import datetime
import os
inputdir = "C:\\temp2\\CSV\\"
outputdir = "C:\\temp2\\CSV\\output\\"
keyword = "KEYWORD"
for path, dirs, files in os.walk(os.path.abspath(inputdir)):
for f in os.listdir(inputdir):
file_path = os.path.join(inputdir, f)
out_file = os.path.join(outputdir, f)
with open(file_path, "r") as fh, open(out_file, "w") as fo:
for line in fh:
if keyword not in line:
fo.write(line)
However, when I change the output folder to: outputdir = "C:\\temp2\\output\\" the code runs successfully. I want to be able to write the modified files to a sub-folder of the input directory. How would I do this without getting the 'Permission denied' error? Would the tempfile module be useful in this scenario?
os.listdir will return directory as well as file names. output is within inputdir so the with is trying to open a directory for reading/writing.
What exactly are you trying to do? path, dirs, files aren't even being used in the recursive os.walk.
Edit: I think you're looking for something like this:
import os
INPUTDIR= "c:\\temp2\\CSV"
OUTPUTDIR = "c:\\temp2\\CSV\\output"
keyword = "KEYWORD"
def make_path(p):
'''Makes sure directory components of p exist.'''
try:
os.makedirs(p)
except OSError:
pass
def dest_path(p):
'''Determines relative path of p to INPUTDIR,
and generates a matching path on OUTPUTDIR.
'''
path = os.path.relpath(p,INPUTDIR)
return os.path.join(OUTPUTDIR,path)
make_path(OUTPUTDIR)
for path, dirs, files in os.walk(INPUTDIR):
for d in dirs:
dir_path = os.path.join(path,d)
# Handle case of OUTPUTDIR inside INPUTDIR
if dir_path == OUTPUTDIR:
dirs.remove(d)
continue
make_path(dest_path(dir_path))
for f in files:
file_path = os.path.join(path, f)
out_path = dest_path(file_path)
with open(file_path, "r") as fh, open(out_path, "w") as fo:
for line in fh:
if keyword not in line:
fo.write(line)
If you are successful in writing to a output directory outside of the input traversing directory, then write it there first using the same code as above and then move it to a sub-directory within the input directory. You could use os.move for that.