We Keep Coding

How to ignore corrupted files?

How to loop through a directory in Python and open wave files that are good whilst ignoring bad (corrupted) ones?
I want to open various wave files from a directory. However, some of these files may be corrupted, some may not be to specification. In particular there will be files in that directory which when trying to open them will raise the error:
wave.Error: file does not start with RIFF id
I want to ignore those files. I want to catch the error and continue with the loop. How can this be done?
My code:
for file_path in files:
sig=0
file = str(file_path)
sig, wave_params = DataGenerator.open_wave(file)
if sig == 0:
print(
'WARNING: Could not open wave file during data creation: ' + file)
continue
if wave_params[0] != 1:
print("WARNING: Wrong NUMBER OF CHANNELS in " + file)
txt.write(
"WARNING: Wrong NUMBER OF CHANNELS in " + file + "\n")
continue
if wave_params[1] != 2:
print("WARNING: Wrong SAMPLE WIDTH in " + file)
txt.write("WARNING: Wrong SAMPLE WIDTH in " + file + "\n")
continue
if wave_params[2] != RATE:
print("WARNING: Wrong FRAME RATE in " + file)
txt.write("WARNING: Wrong FRAME RATE in " + file + "\n")
continue
if wave_params[3] != SAMPLES:
print("WARNING: Wrong NUMBER OF SAMPLES in " + file)
txt.write(
"WARNING: Wrong NUMBER OF SAMPLES in " + file + "\n")
continue
if wave_params[4] != 'NONE':
print("WARNING: Wrong comptype: " + file)
txt.write("WARNING: Wrong comptype: " + file + "\n")
continue
if wave_params[5] != 'not compressed':
print("WARNING: File appears to be compressed " + file)
txt.write(
"WARNING: File appears to be compressed " + file + "\n")
continue
if bit_depth != (wave_params[2] * (2**4) * wave_params[1]):
print("WARNING: Wring bit depth in " + file)
txt.write("WARNING: Wring bit depth in " + file + "\n")
continue
if isinstance(sig, int):
print("WARNING: No signal in " + file)
txt.write("WARNING: No signal in " + file + "\n")
continue
My code for opening the wave file:
def open_wave(sound_file):
"""
Open wave file
Links:
https://stackoverflow.com/questions/16778878/python-write-a-wav-file-into-numpy-float-array
https://stackoverflow.com/questions/2060628/reading-wav-files-in-python
"""
if Path(sound_file).is_file():
sig = 0
with wave.open(sound_file, 'rb') as f:
n_channels = f.getnchannels()
samp_width = f.getsampwidth()
frame_rate = f.getframerate()
num_frames = f.getnframes()
wav_params = f.getparams()
snd = f.readframes(num_frames)
audio_as_np_int16 = np.frombuffer(snd, dtype=np.int16)
sig = audio_as_np_int16.astype(np.float32)
return sig, wav_params
else:
print('ERROR: File ' + sound_file + ' does not exist. BAD.')
print("Problem with openng wave file")
exit(1)
The missing lines which scale the output of the wave file correctly is done on purpose.
I am interested in how to catch the error mentioned above. A tipp of how to open wave files defensively would be nice, too. That is how can I simply ignore wave files that throw errors?

just wrap your function in a try:except block
for file_path in files:
sig=0
file = str(file_path)
try: # attempt to use `open_wave`
sig, wave_params = DataGenerator.open_wave(file)
except wave.Error as ex:
print(f"caught Exception reading '{file}': {repr(ex)}")
continue # next file_path
# opportunity to catch other or more generic Exceptions
... # rest of loop

You could make use of a try-catch block. where you 'try' accessing the file and you catch a potential exception. here you could just make a 'pass'

Error while trying to generate xml file with specific name

I tried to generate a file named, for example, 23-10-2022|21-03-11.xml or if the user enters his own name userGeneratedName23-10-2022-21-03-11.xml. I don't know why when I tried to specify a particular folder where the generated file should be saved the program throws me an Invalid argument error. I suspect that I am using join incorrectly, I don't really know how to correct it
if not os.path.exists("Generated XMLs"):
os.makedirs("Generated XMLs")
#open file
today = date.today()
now = datetime.now()
#if filename is not specified, create file with today's date and time of creation
if filename == "":
filename = today.strftime("%d-%m-%Y") +"|"+ now.strftime("%H-%M-%S")
#if filename is specified,ad at the and of filename today's date and time of creation
else:
filename = filename + today.strftime("%d-%m-%Y")+"|"+ now.strftime("%H-%M-%S")
# open file wchich is in Generated XMLs folder and name it with variable filename
file = open(os.path.join("Generated XMLs", filename + ".xml"), "w")
#write to file
#write header
file.write("\n<root>\n<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
file.write("\t<POZYCJE>\n")
#write data
# ask user if he wont netto or brutto meters and ad flag
flag = input("Enter 'n' if you want to use netto meters or 'b' if you want to use brutto meters: ")
for i in range(len(convertedData)):
file.write("\t\t<POZYCJA>\n")
file.write("\t\t\t<LP>" + (i+1) + "</LP>\n")
file.write("\t\t\t<TOWAR>\n")
file.write("\t\t\t\t<KOD>" + convertedData[i][0] + " " + convertedData[i][1] + "</KOD>\n")
file.write("\t\t\t\t<NAZWA>" + convertedData[i][0] + " " + convertedData[i][1] + "</NAZWA>\n")
file.write("\t\t\t\t<OPIS/>")
file.write("\t\t\t\t<EAN/>")
file.write("\t\t\t\t<SWW/>")
file.write("\t\t\t\t<NUMER_KATALOGOWY/>")
file.write("\t\t\t\t<MPP>" + "0" + "</MPP>\n")
file.write("\t\t\t</TOWAR>\n")
file.write("\t\t\t<STAWKA_VAT>\n")
file.write("\t\t\t\t<STAWKA>" + "23.00" + "</STAWKA>\n")
file.write("\t\t\t\t<FLAGA>" + "2" + "</FLAGA>\n")
file.write("\t\t\t\t<ZRODLOWA>" + "0.00" + "</ZRODLOWA>\n")
file.write("\t\t\t</STAWKA_VAT>\n")
file.write("\t\t\t<CENY>\n")
file.write("\t\t\t\t<CENAZCZTEREMAMIEJSCAMI>0</CENAZCZTEREMAMIEJSCAMI>\n")
file.write("\t\t\t\t<POCZATKOWA_WAL_CENNIKA>00.0000</POCZATKOWA_WAL_CENNIKA>\n")
file.write("\t\t\t\t<POCZATKOWA_WAL_DOKUMENTU>00.0000</POCZATKOWA_WAL_DOKUMENTU>\n")
file.write("\t\t\t\t<PO_RABACIE_WAL_CENNIKA>00.0000</PO_RABACIE_WAL_CENNIKA>\n")
file.write("\t\t\t\t<PO_RABACIE_PLN>00.0000</PO_RABACIE_PLN>\n")
file.write("\t\t\t\t<PO_RABACIE_WAL_DOKUMENTU>00.0000</PO_RABACIE_WAL_DOKUMENTU>\n")
file.write("\t\t\t</CENY>\n")
file.write("\t\t\t<WALUTA>\n")
file.write("\t\t\t\t<SYMBOL>PLN</SYMBOL>\n")
file.write("\t\t\t\t<KURS_L>1.00</KURS_L>\n")
file.write("\t\t\t\t<KURS_M>1</KURS_M>\n")
file.write("\t\t\t</WALUTA>\n")
file.write("\t\t\t<RABAT>0.00</RABAT>\n")
file.write("\t\t\t<WARTOSC_NETTO>0.00</WARTOSC_NETTO>\n")
file.write("\t\t\t<WARTOSC_BRUTTO>0.00</WARTOSC_BRUTTO>\n")
file.write("\t\t\t<WARTOSC_NETTO_WAL>00.00</WARTOSC_NETTO_WAL>\n")
file.write("\t\t\t<WARTOSC_BRUTTO_WAL>833.94</WARTOSC_BRUTTO_WAL>\n")
if flag == "n":
file.write("\t\t\t<ILOSC>" + convertedData[i][3] + "00" + "</ilosc>\n")
elif flag == "b":
file.write("\t\t\t<ILOSC>" + convertedData[i][2] + "00" + "</ilosc>\n")
else:
print("Error: Wrong flag. Enter 'n' or 'b'.")
file.write("\t\t\t<JB>" + convertedData[i][4] + "</JB>\n")
file.write("\t\t\t<JM_CALKOWITE>0.00</JM_CALKOWITE>\n")
file.write("\t\t\t<JM_ZLOZONA>\n")
file.write("\t\t\t\t\n")
file.write("\t\t\t\t\n")
file.write("\t\t\t\t\n")
file.write("\t\t\t</JM_ZLOZONA>\n")
file.write("\t\t\t<JMZ>" + convertedData[i][4] + "</JMZ>\n")
file.write("\t\t\t<JM_PRZELICZNIK_L>1.00</JM_PRZELICZNIK_L>\n")
file.write("\t\t\t<JM_PRZELICZNIK_M>1</JM_PRZELICZNIK_M>\n")
file.write("\t\t</POZYCJA>\n")
#write footer
file.write("\t</POZYCJE>\n")
file.write("</root>")
#close file
file.close()
The exact error I get:
Traceback (most recent call last):
File "C:\Users\reczul\PycharmProjects\pythonProject5\main.py", line 23, in <module>
main()
File "C:\Users\reczul\PycharmProjects\pythonProject5\main.py", line 13, in main
XMLCreator.CreateXML(DataConverter.ConvertData(data))
File "C:\Users\reczul\PycharmProjects\pythonProject5\venv\Functions\XMLCreator.py", line 20, in CreateXML
file = open(os.path.join("Generated XMLs", filename + ".xml"), "w")
OSError: [Errno 22] Invalid argument: 'Generated XMLs\\24-10-2022|09-23-45.xml'
Process finished with exit code 1

I can not use "|" also I had to convert filename to str.
if not os.path.exists("Generated XMLs"):
os.makedirs("Generated XMLs")
#open file
today = date.today()
now = datetime.now()
#if filename is not specified, create file with today's date and time of creation
if filename == "":
filename = today.strftime("%d-%m-%Y") +"--"+ now.strftime("%H-%M-%S")
#if filename is specified,ad at the and of filename today's date and time of creation
else:
filename = filename + today.strftime("%d-%m-%Y")+"--"+ now.strftime("%H-%M-%S")
filename = str(filename)
# open file wchich is in Generated XMLs folder and name it with variable filename
#file = open(os.path.join(os.path.dirname(os.path.abspath(__file__)),"Generated XMLs",filename + ".xml"), "w")
#cerate XML file which is in Generated XMLs folder
file = open(os.path.join("Generated XMLs", filename + ".xml"), "w")

python - read file info, permissions from raw ext4 image

I am trying to unpack android 11 image / get info from the raw .img for selinux info, symlinks etc.
I am using this wonderful tool: https://github.com/cubinator/ext4/blob/master/ext4.py35.py
and my code looks like this:
#!/usr/bin/env python3
import argparse
import sys
import os
import ext4
parser = argparse.ArgumentParser(description='Read <modes, symlinks, contexts and capabilities> from an ext4 image')
parser.add_argument('ext4_image', help='Path to ext4 image to process')
args = parser.parse_args()
exists = os.path.isfile(args.ext4_image)
if not exists:
print("Error: input file " f"[{args.ext4_image}]" " was not found")
sys.exit(1)
file = open(args.ext4_image, "rb")
volume = ext4.Volume(file)
def scan_dir (root_inode, root_path = ""):
for entry_name, entry_inode_idx, entry_type in root_inode.open_dir():
if entry_name == "." or entry_name == "..":
continue
entry_inode = root_inode.volume.get_inode(entry_inode_idx)
entry_inode_path = root_path + "/" + entry_name
if entry_inode.is_dir:
scan_dir(entry_inode, entry_inode_path)
if entry_inode_path[-1] == '/':
continue
xattrs_perms = list(entry_inode.xattrs())
found_cap = False
found_con = False
if "security.capability" in f"{xattrs_perms}": found_cap = True
if "security.selinux" in f"{xattrs_perms}": found_con = True
contexts = ""
capability = ", \"capabilities\", 0x0"
if found_cap:
if found_con:
capability = f"{xattrs_perms[1:2]}"
else:
capability = f"{xattrs_perms[0:1]}"
capability = capability.split(" ")[1][:-3][+2:].encode('utf-8').decode('unicode-escape').encode('ISO-8859-1')
capability = hex(int.from_bytes(capability[4:8] + capability[14:18], "little"))
capability = ", \"capabilities\", " f"{capability}"
capability = f"{capability}"
if found_con:
contexts = f"{xattrs_perms[0:1]}"
contexts = f"{contexts.split( )[1].split('x00')[0][:-1][+2:]}"
contexts = f"{contexts}"
filefolder = ''.join(entry_inode_path.split('/', 1))
print("set_metadata(\""f"{filefolder}" "\", \"uid\", " f"{str(entry_inode.inode.i_uid)}" ", \"gid\", " f"{str(entry_inode.inode.i_gid)}" ", \"mode\", " f"{entry_inode.inode.i_mode & 0x1FF:0>4o}" f"{capability}" ", \"selabel\", \"" f"{contexts}" "\");")
scan_dir(volume.root)
file.close()
then I just have to do ./read.py vendor.img and it works.
Untill recently I tried this weird vendor.img from android 11 and got this weird issue.
Traceback (most recent call last):
File "./tools/metadata.py", line 53, in <module>
scan_dir(volume.root)
File "./tools/metadata.py", line 26, in scan_dir
scan_dir(entry_inode, entry_inode_path)
File "./tools/metadata.py", line 26, in scan_dir
scan_dir(entry_inode, entry_inode_path)
File "./tools/metadata.py", line 29, in scan_dir
xattrs_perms = list(entry_inode.xattrs())
File "/home/semaphore/unpacker/tools/ext4.py", line 976, in xattrs
for xattr_name, xattr_value in self._parse_xattrs(inline_data[offset:], 0, prefix_override = prefix_override):
File "/home/semaphore/unpacker/tools/ext4.py", line 724, in _parse_xattrs
xattr_inode = self.volume.get_inode(xattr.e_value_inum, InodeType.FILE)
NameError: name 'xattr' is not defined
I have tried removing the if and keeping code after else only here: https://github.com/cubinator/ext4/blob/master/ext4.py35.py#L722
Sadly no luck. It looks like the tool is not finished? But there are no other alternatives.
Any help is welcome :)
Thank you.
EDIT: someone suggested replace xattr with xattr_entry
So i did and i got this error: takes 2 positional arguments but 3 were given
I tried fixing that and got:
File "/home/semaphore/unpacker/tools/ext4.py", line 724, in _parse_xattrs
xattr_inode = self.volume.get_inode(xattr_entry.e_value_inum)
File "/home/semaphore/unpacker/tools/ext4.py", line 595, in get_inode
inode_table_offset = self.group_descriptors[group_idx].bg_inode_table * self.block_size
IndexError: list index out of range
And I could not fix this error :(
Maybe theres an alternative to getting selinux info, capabilities, uid, gid, permissions from raw ext4 image?

I read that you had tried to fix the issue yourself but you never posted a snippet of the code you're currently using.
I am not sure but it seems to me you modified the signature of get_inode instead of modifying which parameters get passed to it.
E.g. did you try:
xattr_inode = self.volume.get_inode(xattr_entry.e_value_inum)

I figured out how to do it in an alternative way.
First mount the image (needs root access):
os.system("sudo mount -t ext4 -o loop vendor.img vendor")
Then use: os.lstat and os.getxattr on each file. It gives all the information:
stat_info = os.lstat(file)
try:
cap = hex(int.from_bytes(os.getxattr(file, "security.capability")[4:8] + os.getxattr(file, "security.capability")[14:18], "little"))
except:
cap = "0x0"
try:
selabel = os.getxattr(file, b"security.selinux", follow_symlinks=False).decode().strip('\n\0')
except:
selabel = "u:object_r:unlabeled:s0"
metadata.append("set_metadata(\"/" + file + "\", \"uid\", " + str(stat_info.st_uid) + ", \"gid\", " + str(stat_info.st_gid) + ", \"mode\", " + oct(stat_info.st_mode)[-4:] + ", \"capabilities\", " + cap + ", \"selabel\", \"" + selabel + "\");")
Like so. This is the only solution I could find

Python - Path/Folder/File creation

I am running the following block of code to create the path to a new file:
# Opens/create the file that will be created
device_name = target_device["host"].split('.')
path = "/home/user/test_scripts/configs/" + device_name[-1] + "/"
print(path)
# Check if path exists
if not os.path.exists(path):
os.makedirs(path)
# file = open(time_now + "_" + target_device["host"] + "_config.txt", "w")
file = open(path + time_now + "_" + device_name[0] + "_config.txt", "w")
# Time Stamp File
file.write('\n Create on ' + now.strftime("%Y-%m-%d") +
' at ' + now.strftime("%H:%M:%S") + ' GMT\n')
# Writes output to file
file.write(output)
# Close file
file.close()
The code run as intended with the exception that it creates and saves the files on the directory: /home/user/test_scripts/configs/ instead on the indented one that should be: /home/user/test_scripts/configs/device_name[-1]/.
Please advise.
Regards,
./daq

Try using os.path.join(base_path, new_path) [Reference] instead of string concatenation. For example:
path = os.path.join("/home/user/test_scripts/configs/", device_name[-1])
os.makedirs(path, exist_ok=True)
new_name = time_now + "_" + device_name[0] + "_config.txt"
with open(os.path.join(path, new_name), "w+") as file:
file.write("something")
Although I don't get why you're creating a directory with device_name[-1] and as a file name using device_name[0].

i have this code. It is supposed to classify the segment files with the ratings in sys.argv[5].But it keeps having error

I have the following code tha uses FFmpeg . It has 5 argv and it takes in filename,video,segment size, start time, end time, ratings. thats supposed to let me classify segments many times with my ratings "PG G M18..." but there's this error,
"File "C:\c.py",line 92, in <module> os.rename<filename + str(x), filename + str(x) + classification)
WindowsError: [Error2] The system cannot find the file specified.
Ive tried to edit many times but this error still persists. Anyone have any idea what could this error mean and anyway to solve it?
import sys
import subprocess
import os
#change hh:mm:ss to seconds:
def getSeconds(sec):
l = sec.split(':')
return int(l[0])* 3600 + int(l[1])* 60 + float(l[2])
def get_total_time(filename):
proc = subprocess.Popen(["ffmpeg", "-i", filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = proc.communicate()[1]
target = [line for line in lines.split('\n') if 'Duration:' in line][0]
time = target.split('Duration: ')[-1].split(',', 1)[0]
return time
#check command line arguments
if len(sys.argv) < 5:
print "Error: not enough arguments"
sys.exit()
#save filename to file_name
file_name = sys.argv[1]
if not file_name.endswith('mpg'):
print 'Error! File extension not supported'
sys.exit()
# save a size of chunk in chunk
segsize = int(sys.argv[2])
chunk = (segsize * 1024)
# get time of starting censorship in seconds
start_censorship = getSeconds(sys.argv[3])
# get time of ending censorship in seconds
end_censorship = getSeconds(sys.argv[4])
classification = sys.argv[5]
if classification not in ['P','PG','G','NC16','M18','R21']:
print "Error: invalid classification"
sys.exit()
#initialize variable for extension
file_ext = ''
# if extension exists then save it into file_ext
if '.' in file_name:
# split file_name on two parts from right
file_ext = sys.argv[1].split('.')[-1]
# file_name without extension
filename = '.'.join(file_name.split('.')[:-1])
# total_time of file in seconds
total_time = getSeconds(get_total_time(file_name))
print total_time
#open file
in_file = open(file_name,'rb')
#read first chunks
s = in_file.read(chunk)
in_file.seek(0, 2)
file_size = in_file.tell()
chunks = (file_size / chunk) + 1
chunk_time = total_time/ file_size * chunk
#close input file
in_file.close()
#loop for each chunk
for x in range(0, chunks):
# starting time of current chunk
t1 = chunk_time*x
# ending time of current chunk
t2 = chunk_time*(x+1)
if t2 < start_censorship or t1 > end_censorship:
pass
else:
if os.path.exists(filename + str(x) + 'x'):
os.rename(filename + str(x) + 'x', filename + str(x))
os.rename(filename + str(x), filename + str(x) + classification)
#read next bytes

You're not checking whether filename + str(x) exists before you try to rename it. Either check first, or put it in a try block and catch OSError (which WindowsError subclasses). Either that or you're missing an indent on the second rename.

In that last for loop it looks like you are possibly renaming multiple files -- where do the original files come from?
Or, asked another way, for every chunk that is not censored you are renaming a file -- is that really what you want?