We Keep Coding

I want to rename all the files in a folder with python

from glob import glob
from os import rename
import time
arrr = []
def getnames():
with open("name.txt", "r+") as nameFile:
for name in nameFile:
nameFile.readline()
newlinestrip = name.strip("\n")
arrr.append(newlinestrip)
renames()
def renames():
for fname in glob('*.png'):
print(fname)
for name in arrr:
time.sleep(1)
rename(fname, name)
print("bruh")
getnames()
It renames the 1st file and the it crashes with the error
Traceback (most recent call last):
File ".\rename.py", line 24, in <module>
getnames()
File ".\rename.py", line 13, in getnames
renames()
File ".\rename.py", line 20, in renames
rename(fname, name)
FileNotFoundError: [WinError 2] Den angivne fil blev ikke fundet: 'sans (100).png' -> 'acacia_door_top.png'
And i don't know how to fix this, i have a txt file with the new names that looks something like this
name.png
name1.png
and so on.

On way is to use zip() function if the len of both the files and arr is equal, However you could just the following if the intention is to rename all the files with names like name1.png, name2.png .. name608.png:
import os
for count, filename in enumerate(os.listdir("someDir")):
dst = "name" + str(count) + ".png"
src = 'someDir' + filename
dst = 'someDir' + dst
# rename() function will rename all the files
os.rename(src, dst)

Browse and merger Excel files in a folder and its subfolders in Python

I have a folder which contains multiple subfolders, I want to browser all excel files end with xlsx and merger them into one single xlsx file with following code:
import os
import glob
for root, dirs, files in os.walk("D:/Test"):
for file in files:
if file.endswith(".xlsx"):
#print(os.path.join(root, file))
s = os.path.join(root, file)
print(s)
all_data = pd.DataFrame()
for f in s:
df = pd.read_excel(f)
all_data = all_data.append(df,ignore_index=True)
# now save the data frame
writer = pd.ExcelWriter('result.xlsx')
all_data.to_excel(writer,'sheet1')
writer.save()
While an error happens when it run:
Traceback (most recent call last):
File "<ipython-input-169-41c6d76207e7>", line 12, in <module>
df = pd.read_excel(f)
File "C:\Users\User\Anaconda3\lib\site-packages\pandas\util\_decorators.py", line 118, in wrapper
return func(*args, **kwargs)
File "C:\Users\User\Anaconda3\lib\site-packages\pandas\io\excel.py", line 230, in read_excel
io = ExcelFile(io, engine=engine)
File "C:\Users\User\Anaconda3\lib\site-packages\pandas\io\excel.py", line 294, in __init__
self.book = xlrd.open_workbook(self._io)
File "C:\Users\User\Anaconda3\lib\site-packages\xlrd\__init__.py", line 116, in open_workbook
with open(filename, "rb") as f:
FileNotFoundError: [Errno 2] No such file or directory: 'D'
Does someone know how to deal with this problem? Thanks.

Your problem is with df = pd.read_excel(f). What are the contents of f? It looks like Python thinks that it's 'D'.
This is because your for f in s: is just iterating over the string that you created with s = os.path.join(root, file). I think you want to be saving this in some container like so
paths = []
for root, dirs, files in os.walk("D:/Test"):
for file in files:
if file.endswith(".xlsx"):
#print(os.path.join(root, file))
s = os.path.join(root, file)
print(s)
paths.append(s)
all_data = pd.DataFrame()
for f in paths:
df = pd.read_excel(f)
all_data = all_data.append(df,ignore_index=True)
You can also reduce that initial for loop into a list comprehension with
paths = [os.path.join(root, file) for root, _, files in os.walk('D:/Test') for file in files if file.endswith('.xlsx')]

import os
import pandas as pd
listof_files = os.listdir()
current_file_name = os.path.basename(__file__)
#flag to make sure append is happening properly
count = 0
mainFrame = 0
for file in listof_files:
#To ignore the python script file for pd.read_excel
if((file != current_file_name) and (file.endswith(".xlsx"))):
tempdf = pd.read_excel(str(file))
if(count == 0):
mainFrame = tempdf.copy()
else:
mainFrame = pd.concat([mainFrame,tempdf])
count += 1
mainFrame.to_excel('final.xlsx',index=False)
You can do like this also, put the script in the folder where you have all the xlsx files, then run the script, it will fetch all the xlsx file and concat with each other, finally, a single excel file is formed.

IOError: [Errno 2] No such file or directory?

I'm trying to move 220 files in Wheat to train_reuters file package,and the another files in wheat move to train_reuters test_reuters file package,but when I run the code,it give me the error,I actually have the file in the right place!how can I solve the problem?
#!/usr/bin/python
#coding:utf-8
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import os
import os.path
import shutil
import random
path = '/home/user1/zhouchun/lda/reuters-21578/Wheat'
targetpath1 = '/home/user1/zhouchun/lda/reuters-21578/train_reuters'
targetpath2 = '/home/user1/zhouchun/lda/reuters-21578/test_reuters'
list=random.sample(range(1, 306),220)
for i in list:
file_dir = os.path.join(path, str(i))
# print file_dir
shutil.move(file_dir, targetpath1)
files = os.listdir(path)
for file in files:
# print file
dir = os.path.join(path, file)
if dir != file_dir:
shutil.move(dir, targetpath2)

I checked your code, it is right.
Then the problem maybe:
1. you could only run your code one time, two or more times will cause this error.
2. Before you run your code, make sure all the 306 files are in Wheat directory.
I suggest using copy, but not move,then clear the train and test file before each run.

Please check the ome/user1/zhouchun/lda/reuters-21578/Wheat files number is 305.
I created a function write the random file, the code you can reference.
import random
import os
path = r'E:\temp\temp'
list= random.sample(range(1, 306), 220)
for i in list:
file_dir = os.path.join(path, str(i))
with open(file_dir, 'w') as f:
f.write('file_dir: %s' % file_dir)
f.close()
please notices the 220 in line list= random.sample(range(1, 306), 220).
After do this paste you code and change the path,
#!/usr/bin/python
#coding:utf-8
import sys
import os.path
import shutil
import random
import time
path = r'E:\temp\temp'
targetpath1 = r'E:\temp\old'
targetpath2 = r'E:\temp\new'
# move the file
list = random.sample(range(1, 306), 220)
for i in list:
file_dir = os.path.join(path, str(i))
# print file_dir
# targetpath1_dir = os.path.join(targetpath1, str(i))
shutil.move(file_dir, targetpath1)
files = os.listdir(path)
for file in files:
# print(file)
# print file
dir = os.path.join(path, file)
if dir != file_dir:
shutil.move(dir, targetpath2)
Than run the code, the error information will income.
Traceback (most recent call last):
File "D:\Python_3.5\lib\shutil.py", line 544, in move
os.rename(src, real_dst)
FileNotFoundError: [WinError 2] System can't found the file.: 'E:\\temp\\temp\\182' -> 'E:\\temp\\old\\182'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "F:/Python_Code/FaceDetect/123123.py", line 31, in <module>
shutil.move(file_dir, targetpath1)
File "D:\Python_3.5\lib\shutil.py", line 558, in move
copy_function(src, real_dst)
File "D:\Python_3.5\lib\shutil.py", line 257, in copy2
copyfile(src, dst, follow_symlinks=follow_symlinks)
File "D:\Python_3.5\lib\shutil.py", line 120, in copyfile
with open(src, 'rb') as fsrc:
FileNotFoundError: [Errno 2] No such file or directory: 'E:\\temp\\temp\\182'
After change the number from 220 to 305 in line list= random.sample(range(1, 306), 220), the error will disappear.
The complete code.
#!/usr/bin/python
#coding:utf-8
import sys
import os.path
import shutil
import random
import time
path = r'E:\temp\temp'
targetpath1 = r'E:\temp\old'
targetpath2 = r'E:\temp\new'
# create the random file.
list= random.sample(range(1, 306), 220)
for i in list:
file_dir = os.path.join(path, str(i))
with open(file_dir, 'w') as f:
f.write('file_dir: %s' % file_dir)
f.close()
time.sleep(1)
# move the file
list = random.sample(range(1, 306), 220)
for i in list:
file_dir = os.path.join(path, str(i))
# print file_dir
# targetpath1_dir = os.path.join(targetpath1, str(i))
shutil.move(file_dir, targetpath1)
files = os.listdir(path)
for file in files:
# print(file)
# print file
dir = os.path.join(path, file)
if dir != file_dir:
shutil.move(dir, targetpath2)
Please reference.

Python/Excel - IOError: [Errno 2] No such file or directory:

Attempting to extract .xlsx docs from a file and compile the data into a single worksheet.
Receiving a IOError despite that the files exist
Program is as follows
#-------------- loop that pulls in files from folder--------------
import os
#create directory from which to pull the files
rootdir = r'C:\Users\username\Desktop\Mults'
for subdir, dir, files in os.walk(rootdir):
for file in files:
print os.path.join(subdir,file)
#----------------------merge work books-----------------------
import xlrd
import xlsxwriter
wb = xlsxwriter.Workbook('merged.xls')
ws = wb.add_worksheet()
for file in files:
r = xlrd.open_workbook(file)
head, tail = os.path.split(file)
count = 0
for sheet in r:
if sheet.number_of_rows()>0:
count += 1
for sheet in r:
if sheet.number_of_rosw()>0:
if count == 1:
sheet_name = tail
else:
sheet_name = "%s_%s" (tail, sheet.name)
new_sheet = wb.create_sheet(sheet_name)
new_sheet.write_reader(sheet)
new_sheet.close()
wb.close()
Return error as follows
doc1.xlsx
doc2.xlsx
doc3.xlsx
doc4.xlsx
Traceback (most recent call last):
File "C:\Users\username\Desktop\Work\Python\excel practice\xlsx - loops files - 09204.py", line 23, in <module>
r = xlrd.open_workbook(file)
File "C:\Python27\lib\site-packages\xlrd\__init__.py", line 394, in open_workbook
f = open(filename, "rb")
IOError: [Errno 2] No such file or directory: 'doc1.xlsx'
Any suggestions or changes?
Also, any advice if I'm heading in the right direction?
I'm new to the python world, so any advice will be much appreciated!
Thank you!!

You are opening the plain filename without the path; you are ignoring the directory component.
Don't just print the os.path.join() result, actually use it:
filename = os.path.join(subdir, file)
r = xlrd.open_workbook(filename)

For the first problem...
Instead of:
r = xlrd.open_workbook(file)
Use:
r = xlrd.open_workbook(os.path.join(subdir,file))
For the TypeError:
Instead of:
for sheet in r:
if sheet.number_of_rows()>0:
count += 1
Use:
for nsheet in r.sheet_names() #you need a list of sheet names to loop throug
sheet = r.sheet_by_name(nsheet) #then you create a sheet object with each name in the list
if sheet.nrows>0: #use the property nrows of the sheet object to count the number of rows
count += 1
Do the same for the second for loop.

Gzip problem,traceback and IOError: [Errno 2] No such file or directory

I'm new to python and bioinformatics field. I'm using python-2.6. Now I'm trying to select all fastq.gz files, then gzip.open(just a few lines because it's too huge and time-wasting), then count 'J' , then pick out those files with 'J' count NOT equal to 0.
The following is my code:
#!/usr/bin/python
import os,sys,re,gzip
path = "/home/XXX/nearline"
for file in os.listdir(path):
if re.match('.*\.recal.fastq.gz', file):
text = gzip.open(file,'r').readlines()[:10]
word_list = text.split()
number = word_list.count('J') + 1
if number !== 0:
print file
But I got some errors:
Traceback (most recent call last):
File "fastqfilter.py", line 9, in <module>
text = gzip.open(file,'r').readlines()[:10]
File "/share/lib/python2.6/gzip.py", line 33, in open
return GzipFile(filename, mode, compresslevel)
File "/share/lib/python2.6/gzip.py", line 79, in __init__
fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
IOError: [Errno 2] No such file or directory: 'ERR001268_1.recal.fastq.gz'
What's this traceback: File......
Is there anything wrong with gzip here?
And why can't it find ERR001268_1.recal.fastq.gz? It's the first fastq file in the list, and DOES exist there.
Hope give me some clues, and any point out any other errors in the script.
THanks a lot.
Edit: thx everyone. I followed Dan's suggestion. And I tried on ONE fastq file first. My script goes like:
#!/usr/bin/python
import os,sys
import gzip
import itertools
file = gzip.open('/home/xug/nearline/ERR001274_1.recal.fastq.gz','r')
list(itertools.islice(file.xreadlines(),10))
word_list = list.split()
number = word_list.count('J') + 1
if number != 0:
print 'ERR001274_1.recal.fastq.gz'
Then errors are:
Traceback (most recent call last):
File "try2.py", line 8, in <module>
list(itertools.islice(text.xreadlines(),10))
AttributeError: GzipFiles instance has no attribute 'xreadlines'
Edit again: Thx Dan, I've solved the problem yesterday. Seems GzipFiles don't support xreadlines. So I tried the similar way as you suggested later. And it works. See below:
#!/usr/bin/python
import os,sys,re
import gzip
from itertools import islice
path = "/home/XXXX/nearline"
for file in os.listdir(path):
if re.match('.*\.recal.fastq.gz', file):
fullpath = os.path.join(path, file)
myfile = gzip.open(fullpath,'r')
head = list(islice(myfile,1000))
word_str = ";".join(str(x) for x in head)
number = word_str.count('J')
if number != 0:
print file

on this line:
text = gzip.open(file,'r').read()
file is a filename not a full path so
fullpath = os.path.join(path, file)
text = gzip.open(fullpath,'r').read()
about F.readlines()[:10] will read the whole file in to a list of lines and then take the first 10
import itertools
list(itertools.islice(F.xreadlines(),10))
this will not read the whole file into memory and will only read the first 10 lines into a list
but as gzip.open returns an object that doesn't have .xreadlines() and but as files are iterable on their lines just:
list(itertools.islice(F,10))
would work as this test shows:
>>> import gzip,itertools
>>> list(itertools.islice(gzip.open("/home/dan/Desktop/rp718.ps.gz"),10))
['%!PS-Adobe-2.0\n', '%%Creator: dvips 5.528 Copyright 1986, 1994 Radical Eye Software\n', '%%Title: WLP-94-int.dvi\n', '%%CreationDate: Mon Jan 16 16:24:41 1995\n', '%%Pages: 6\n', '%%PageOrder: Ascend\n', '%%BoundingBox: 0 0 596 842\n', '%%EndComments\n', '%DVIPSCommandLine: dvips -f WLP-94-int.dvi\n', '%DVIPSParameters: dpi=300, comments removed\n']

Change your code to:
#!/usr/bin/python
import os,sys,re,gzip
path = "/home/XXX/nearline"
for file in os.listdir(path):
if re.match('.*\.recal.fastq.gz', file):
text = gzip.open(os.path.join(path,file),'r').readlines()[:10]
word_list = text.split()
number = word_list.count('J') + 1
if number !== 0:
print file

It's trying to open ERR001268_1.recal.fastq.gz from the working directory, not from /home/XXX/nearline.