We Keep Coding

How to convert XML Files into Text files (YOLOV3 Format) for object detection

How we convert XML annotation folder into text or YOLOv3 Format for detection??? I used this code for conversion but it only take one xml image and convert into .txt ..but i want to convert my full folder at once time. You have Any easy solution to convert xml files into text files. i Have 15000+ images.
from xml.dom import minidom
import os
import glob
lut={}
lut["14111"] =0
lut["14131"] =1
lut["14141"] =2
def convert_coordinates(size, box):
dw = 1.0/size[0]
dh = 1.0/size[1]
x = (box[0]+box[1])/2.0
y = (box[2]+box[3])/2.0
w = box[1]-box[0]
h = box[3]-box[2]
x = x*dw
w = w*dw
y = y*dh
h = h*dh
return (x,y,w,h)
def convert_xml2yolo( lut ):
for fname in glob.glob("/content/gdrive/MyDrive/Dataset /Annotation/14111_00000002.xml"):
xmldoc = minidom.parse(fname)
fname_out = (fname[:-4]+'.txt')
with open(fname_out, "w") as f:
itemlist = xmldoc.getElementsByTagName('object')
size = xmldoc.getElementsByTagName('size')[0]
width = int((size.getElementsByTagName('width')[0]).firstChild.data)
height = int((size.getElementsByTagName('height')[0]).firstChild.data)
for item in itemlist:
# get class label
classid = (item.getElementsByTagName('name')[0]).firstChild.data
if classid in lut:
label_str = str(lut[classid])
else:
label_str = "-1"
print ("warning: label '%s' not in look-up table" % classid)
# get bbox coordinates
xmin = ((item.getElementsByTagName('bndbox')[0]).getElementsByTagName('xmin')[0]).firstChild.data
ymin = ((item.getElementsByTagName('bndbox')[0]).getElementsByTagName('ymin')[0]).firstChild.data
xmax = ((item.getElementsByTagName('bndbox')[0]).getElementsByTagName('xmax')[0]).firstChild.data
ymax = ((item.getElementsByTagName('bndbox')[0]).getElementsByTagName('ymax')[0]).firstChild.data
b = (float(xmin), float(xmax), float(ymin), float(ymax))
bb = convert_coordinates((width,height), b)
#print(bb)
f.write(label_str + " " + " ".join([("%.6f" % a) for a in bb]) + '\n')
print ("wrote %s" % fname_out)
def main():
convert_xml2yolo( lut )
if __name__ == '__main__':
main()

Follow this github repository.

You just need to edit this line:
for fname in glob.glob("/content/gdrive/MyDrive/Dataset/Annotation/*.xml"):
This means you will read all the .xml files in the Annotation folder and convert them to .txt files.

National Instruments USB X-SERIES with Python-nidaqmx

I am trying to drive 4 independent channels using the python nidaqmx module and the NI X-series 6341 (PN: 781438-01). I have 2 analogue outputs and two digital outputs and I would like all these streams independent of each other. For some reason when I execute the code only my 2 analogue outs and digital out on line 0 fire. I do not see the digital stream on line 1. Does anyone know what may be going on here? I've tried it with another box and get the same behaviour so I don't think its hardware related. Here is the code:
import nidaqmx
from numpy import array
from nidaqmx import stream_writers
import numpy as np
from tkinter import filedialog
from numpy import genfromtxt
import pandas as pd
from nidaqmx.constants import LineGrouping
Devs = []
system = nidaqmx.system.System.local()
print(system.driver_version)
for device in system.devices:
dev = str(device).split('=')[1].split(')')[0]
Devs.append(dev)
print(device)
print(dev)
def detectDelimiter(csvFile):
with open(csvFile, 'r') as myCsvfile:
header=myCsvfile.readline()
if header.find(";")!=-1:
return ";"
if header.find(",")!=-1:
return ","
if header.find("\t")!=-1:
return "\t"
My_Data = []
My_Data_unscaled = []
def load_data():
file_path = filedialog.askopenfilename()
delim = detectDelimiter(file_path)
my_data = genfromtxt(file_path, delimiter=delim)
if len (My_Data) > 0 :
print('Deleting Data in the buffer...')
My_Data.clear()
My_Data.append(my_data)
My_Data_unscaled.append(my_data)
else:
#original_data = my_data
#My_Data = []
My_Data.append(my_data)
My_Data_unscaled.append(my_data)
load_data()
look = My_Data[0]
e_dataframe = pd.DataFrame(look)
v_step = 20/2**16
e_dataframe[0] = e_dataframe[0]*v_step
e_dataframe[1] = e_dataframe[1]*v_step
samples_new = [e_dataframe[1].T,e_dataframe[0].T]
samples_new = array(samples_new)
dig_samples_new = [e_dataframe[2].T,e_dataframe[2].T]
dig_samples_new = array(dig_samples_new)
dig_samples_new[0,0] = 1
dig_samples_new[0,0] = 1
dig_samples_new[1,0] = 1
def fire_galvos(dev,rate,dig_las):
#define channels
channel1 = dev +'/' + 'ao0' # laser trigger
channel2 = dev + '/' + 'ao1' # this is the auxillary trigger signal
channel3 = dev + '/line0'
channel4 = dev + '/line1'
#define clock
sample_clock = '/'+dev+'/ao/SampleClock'
with nidaqmx.Task() as analog_output, nidaqmx.Task() as digital_output:
dig_las = np.uint32(dig_las)
#add channels
analog_output.ao_channels.add_ao_voltage_chan(channel1,'mychannel1',-10,10)
analog_output.ao_channels.add_ao_voltage_chan(channel2,'mychannel2',-10,10)
digital_output.do_channels.add_do_chan(channel3, 'mychannel3')
digital_output.do_channels.add_do_chan(channel4, 'mychannel4')
#digital_output.do_channels.add_do_chan(channel4, 'mychannel4',line_grouping=LineGrouping.CHAN_PER_LINE)
#digital_output.ao_load_impedance = 50
#define clock timings
analog_output.timing.cfg_samp_clk_timing(rate=rate, sample_mode=nidaqmx.constants.AcquisitionType.FINITE, samps_per_chan = len(samples_new[0]))
digital_output.timing.cfg_samp_clk_timing(rate=rate, source = sample_clock, sample_mode=nidaqmx.constants.AcquisitionType.FINITE, samps_per_chan=len(dig_samples_new[0])) #source=sample_clock,
#writing commands
writer_ana = stream_writers.AnalogMultiChannelWriter(analog_output.out_stream, auto_start=False)
writer_dig = stream_writers.DigitalMultiChannelWriter(digital_output.out_stream,auto_start=False)
#writer_dig = stream_writers.DigitalSingleChannelWriter(digital_output.out_stream,auto_start=False)
writer_ana.write_many_sample(samples_new)
writer_dig.write_many_sample_port_uint32(dig_las)
digital_output.start()
analog_output.start()
digital_output.wait_until_done(timeout=60)
analog_output.wait_until_done(timeout=60)
fire_galvos(dev,3000,dig_samples_new)```

Getting the memory layout out of an (avr)elf file by useing python + pyElftools

I am creating my own bootloader for an ATXmega128A4U. To use the bootloader I want to transform the ELF-file of the firmware into a memory map used in the the ATXmega.
For that I use python and the modul "pyelftools". The documentation of it is poor and so I run into a problem: I do not know what information I can use to get the address, offset etc. from the data at the sections.
My goal is to create a bytearray, copy the data/code into it and transfer it to the bootlaoder. Below is my code:
import sys
# If pyelftools is not installed, the example can also run from the root or
# examples/ dir of the source distribution.
sys.path[0:0] = ['.', '..']
from elftools.common.py3compat import bytes2str
from elftools.elf.elffile import ELFFile
# 128k flash for the ATXmega128a4u
flashsize = 128 * 1024
def process_file(filename):
with open(filename, 'rb') as f:
# get the data
elffile = ELFFile(f)
dataSec = elffile.get_section_by_name(b'.data')
textSec = elffile.get_section_by_name(b'.text')
# prepare the memory
flashMemory = bytearray(flashsize)
# the data section
startAddr = dataSec.header.sh_offset
am = dataSec.header.sh_size
i = 0
while i < am:
val = dataSec.stream.read(1)
flashMemory[startAddr] = val[0]
startAddr += 1
i += 1
# the text section
startAddr = textSec.header.sh_offset
am = textSec.header.sh_size
i = 0
while i < am:
print(str(startAddr) + ' : ' + str(i))
val = textSec.stream.read(1)
flashMemory[startAddr] = val[0]
startAddr += 1
i += 1
print('finished')
if __name__ == '__main__':
process_file('firmware.elf')
Hope someone can tell me how to solve this problem.

I manged to solve the problem.
don't read the data manualy from the stream by "textSec.stream.read" use "textSec.data()" instead. Internaly (see "sections.py") a seek operation in the file is done. Afterwards the data is read. The result will be the valid data chunk.
The following code reads the code(text) section of a atxmega firmware and copies it into a bytearray which has the layout of the flash of an atxmega128a4u device.
#vlas_tepesch: the hex conversation is not needed and the the 64k pitfall is avoided.
sys.path[0:0] = ['.', '..']
from elftools.common.py3compat import bytes2str
from elftools.elf.elffile import ELFFile
# 128k flash for the ATXmega128a4u
flashsize = 128 * 1024
def __printSectionInfo (s):
print ('[{nr}] {name} {type} {addr} {offs} {size}'.format(
nr = s.header['sh_name'],
name = s.name,
type = s.header['sh_type'],
addr = s.header['sh_addr'],
offs = s.header['sh_offset'],
size = s.header['sh_size']
)
)
def process_file(filename):
print('In file: ' + filename)
with open(filename, 'rb') as f:
# get the data
elffile = ELFFile(f)
print ('sections:')
for s in elffile.iter_sections():
__printSectionInfo(s)
print ('get the code from the .text section')
textSec = elffile.get_section_by_name(b'.text')
# prepare the memory
flashMemory = bytearray(flashsize)
# the text section
startAddr = textSec.header['sh_addr']
val = textSec.data()
flashMemory[startAddr:startAddr+len(val)] = val
# print memory
print('finished')
if __name__ == '__main__':
process_file('firmware.elf')
Tanks for the comments!

downsampling images as a function in Python

I am trying to resample some tiff files from 2000*2000 to 500*500.
I have created a function and I tried for one file and it worked nicely. Now I want to apply it for all the available file I have.
I want to write the output of the function and I have written the code based on my knowledge and I receive error on the writing out_file. I have copied the both function and main code for your consideration. The main code just read the tif files according to their naming and applies the function. I would be thankful if sb could guide me where my mistake is.
#*********function********************
def ResampleImage(infile):
fp = open(infile, "rb")
p = ImageFile.Parser()
while 1:
s = fp.read()
if not s:
break
p.feed(s)
img = p.close()
basewidth = 500
wpercent = (basewidth / float(img.size[0]))
hsize = int((float(img.size[1]) * float(wpercent)))
outfile=img.resize((basewidth, hsize), PIL.Image.ANTIALIAS)
return outfile
#********* main code********
import os,sys
import ImageResizeF
import PIL
from PIL import Image
from PIL import Image,ImageFile
tpath = 'e:/.../resampling_test/all_tiles/'
tifext = '.tif'
east_start = 32511616
north_start = 5400756
ilist = range (0,14)
jlist = range (0,11)
north = north_start
ee = ',4_'
en = ',2'
for i in ilist:
east = east_start
north = north_start + i * 400
snorth = str (north)
for j in jlist:
east = east_start + j * 400
seast = str (east)
infile = tpath + seast + ee + snorth + en + tifext
output = tpath + seast + ee + snorth + en + '_res'+tifext
out_file = ImageResizeF.ResampleImage(infile)
out_file.write (output)
out_file.close ()

Your error is probably related to what you are returning from ImageResizeF.ResampleImage, is it a file handle? Otherwise you are doing it wrong because you cannot close() something which is not a file handle. You should do the whole file processing inside the function or return an image object, for example:
def process_image(image):
"Processes the image"
image.resize((x, y), Image.ANTIALIAS) # or whatever you are doing to the image
return image
image = Image.open('infile.tiff')
proc_image = process_image(image)
proc_image.save('outfile.tiff')

python read from fd directly into bytearray

Is there a means to read from a file descriptor (not an IO-like object) directly into a bytearray?
Right now I use a temporary FileIO object to mediate, something like:
def fd_readinto(fd, ba):
fio = io.FileIO(fd, closefd = False)
return fio.readinto(ba)

There is no function that does this, and your method is already the fastest approach.
I was going to suggest bytearray(mmap), array.fromfile, and even a homebrew os.read() using bytearray and memoryview, but FileIO.readinto is screaming fast. (It makes sense that it would be because it performs only one system call.)
import os
import mmap, io, array
import timeit
fn = 'path-to-largeish-file'
def fd_readinto_mmap(fd, ba):
m = mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
ba.extend(m)
m.close()
def fd_readinto_fio(fd, ba):
sz = os.fstat(fd).st_size
ba2 = bytearray(sz)
with io.FileIO(fd, closefd = False) as fio:
fio.readinto(ba2)
ba.extend(ba2)
def fd_readinto_array(fd, ba):
ar = array.array('c')
sz = os.fstat(fd).st_size
fp = os.fdopen(fd, 'rb')
ar.fromfile(fp, sz)
ba.extend(ar)
def fd_readinto_mv(fd, ba):
stat = os.fstat(fd)
blksize = getattr(stat, 'st_blksize', 4096)
bufsize = stat.st_size
buf = bytearray(bufsize)
m = memoryview(buf)
while True:
b = os.read(fd, blksize)
s = len(b)
if not s: break
m[:s], m = b, m[s:]
writtenbytes = buffer(buf, 0, bufsize-len(m))
ba.extend(writtenbytes)
setup = """
from __main__ import fn, fd_readinto_mmap, fd_readinto_fio, fd_readinto_array, fd_readinto_mv
import os
openfd = lambda : os.open(fn, os.O_RDONLY)
closefd = lambda fd: os.close(fd)
"""
reps = 2
tests = {
'fio' : "fd=openfd(); fd_readinto_fio(fd, bytearray()); closefd(fd)",
'mmap': "fd=openfd(); fd_readinto_mmap(fd, bytearray()); closefd(fd)",
'array': "fd=openfd(); fd_readinto_array(fd, bytearray());",
'mv' : "fd=openfd(); fd_readinto_mv(fd, bytearray()); closefd(fd)",
}
width = max(map(len, tests))
for n,t in tests.iteritems():
time = timeit.timeit(t, setup, number=reps)
print ("{:%s} {}" % width).format(n, time)
On my system (OS X 10.14.6, Python 2.7.10), FileIO is the fastest option:
mmap 7.19839119911
array 5.72453403473
mv 0.49933886528
fio 0.299485206604