We Keep Coding

How to read multiple DICOM files from a folder?

I have the following code in which I am loading a single DICOM file and checking if there are sagittal and coronal view present or not.
I want to modify this to read all DICOM files from the folder.
print there is no sagittal and coronal view if sag_aspect,cor_aspect value is zero
How do I do this?
import pydicom
import numpy as np
import matplotlib.pyplot as plt
import sys
import glob
# load the DICOM files
files = []
print('glob: {}'.format(sys.argv[1]))
for fname in glob.glob('dicom/3.dcm', recursive=False):
print("loading: {}".format(fname))
files.append(pydicom.dcmread(fname))
print("file count: {}".format(len(files)))
# skip files with no SliceLocation (eg scout views)
slices = []
skipcount = 0
for f in files:
if hasattr(f, 'SliceLocation'):
slices.append(f)
else:
skipcount = skipcount + 1
print("skipped, no SliceLocation: {}".format(skipcount))
# ensure they are in the correct order
slices = sorted(slices, key=lambda s: s.SliceLocation)
# pixel aspects, assuming all slices are the same
ps = slices[0].PixelSpacing
ss = slices[0].SliceThickness
ax_aspect = ps[1]/ps[0]
sag_aspect = ps[1]/ss
cor_aspect = ss/ps[0]
# create 3D array
img_shape = list(slices[0].pixel_array.shape)
img_shape.append(len(slices))
img3d = np.zeros(img_shape)
# fill 3D array with the images from the files
for i, s in enumerate(slices):
img2d = s.pixel_array
img3d[:, :, i] = img2d
# plot 3 orthogonal slices
print(img3d.shape)
print(img_shape)
a1 = plt.subplot(2, 2, 1)
plt.imshow(img3d[:, :, img_shape[2]//2])
a1.set_title("transverse view")
a1.set_aspect(ax_aspect)
a2 = plt.subplot(2, 2, 2)
#print(img3d[:, img_shape[1]//2, :].shape)
plt.imshow(img3d[:, img_shape[1]//2, :])
a2.set_title("sagital view")
a2.set_aspect(sag_aspect)
a3 = plt.subplot(2, 2, 3)
plt.imshow(img3d[img_shape[0]//2, :, :].T)
a3.set_title("coronal view")
a3.set_aspect(cor_aspect)
plt.show()

For reading multiple dicom files from a folder you can use the code below.
import os
from pathlib import Path
import pydicom
dir_path = r"path\to\dicom\files"
dicom_set = []
for root, _, filenames in os.walk(dir_path):
for filename in filenames:
dcm_path = Path(root, filename)
if dcm_path.suffix == ".dcm":
try:
dicom = pydicom.dcmread(dcm_path, force=True)
except IOError as e:
print(f"Can't import {dcm_path.stem}")
else:
dicom_set.append(dicom)
I have leveraged the pathlib library which I strongly suggest to use whenever dealing with folder/file paths. I have also added an exception, but you can modify it to meet your needs.

Reading the huge image data for training classifiers

I am new to python and Machine Learning. I have a huge image dataset of cars having more than 27000 images and labels. I am trying to create a dataset so I can use it in my training classifier, but ofcourse handling this amount of data will be a real pain for the Memory, and that's where I am stuck. At first I was trying to do something like this.
import os
import matplotlib.pyplot as plt
import matplotlib.image as mpg
import cv2
import gc
import numpy as np
from sklearn.preprocessing import normalize
import gc
import resource
import h5py
bbox = "/run/media/fdai5182/LAMAMADAN/Morethan4000samples/data/labels"
imagepath = "/run/media/fdai5182/LAMAMADAN/Morethan4000samples/data/image"
training_data = []
training_labels = []
count = 0
for root, _, files in os.walk(bbox):
cdp = os.path.abspath(root)
for rootImage , _ , fileImage in os.walk(imagepath):
cdpimg = os.path.abspath(r)
for f in files:
ct = 0
name,ext = os.path.splitext(f)
for fI in fileImage:
n , e = os.path.splitext(fI)
if name == n and ext == ".txt" and e == ".jpg":
cip = os.path.join(cdp,f)
cipimg = os.path.join(cdpimg,fI)
txt = open(cip,"r")
for q in txt:
ct = ct + 1
if ct == 3:
x1 = int(q.rsplit(' ')[0])
y1 = int(q.rsplit(' ')[1])
x2 = int(q.rsplit(' ')[2])
y2 = int(q.rsplit(' ')[3])
try:
read_img = mpg.imread(cipimg)
read_img = read_img.astype('float32')
read_img_bbox = read_img[y1:y2, x1:x2,:]
resize_img = cv2.cv2.resize(read_img_bbox,(300,300))
resize_img /= 255.0
training_labels.append(int(cipimg.split('\\')[4]))
training_data.append(resize_img)
print("len Of Training_data",len(training_data))
training_labels.append(int(cipimg.split('/')[8]))
del resize_img
print("len Of Training Labels", len(training_labels))
gc.collect()
except Exception as e:
print("Error",str(e), cip)
count = count + 1
print(count)
txt.flush()
txt.close()
np.save('/run/media/fdai5182/LAMA MADAN/Training_Data_4000Samples',training_data)
np.save('/run/media/fdai5182/LAMA MADAN/Training_Labels_4000Samples',training_labels)
print("DONE")
But it always gives me a huge Memory error after reading images even on 32gb RAM.
So, for that I want to do some other steps which may be useful taking less memory and get this working.
The Steps I want to do are as follows.
allocate np array X of shape N,150,150,3/300,300,3 of type
float32 (not astype)
iterate through images and fill each row of array X with 150,150,3 image pixels
normalize in-place: X /= 255
Write in file (.npy format)
What I did till now is
import cv2
import matplotlib.pyplot as plt
import matplotlib.iamge as mpg
import numpy as np
bbox = "/run/media/fdai5182/LAMAMADAN/Morethan4000samples/data/labels"
imagepath = "/run/media/fdai5182/LAMAMADAN/Morethan4000samples/data/image"
for root, _, files in os.walk(bbox):
cdp = os.path.abspath(root)
for rootImage, _, fileImage in os.walk(imagepath):
cdpimg = os.path.abspath(rootImage)
for f in files:
ct = 0
name,ext = os.path.splitext(f)
for fI in fileImage:
n , e = os.path.splitext(fI)
if name == n and ext == ".txt" and e == ".jpg":
nparrayX = np.zeros((150,150,3)).view('float32')
cip = os.path.join(cdp,f)
cipImg = os.path.join(cdpimg,fI)
read_image = mpg.imread(cip)
resize_image = cv2.cv2.resize(read_image,(150,150))
Am I on the right path?
Also, How can I fill each row of imageformat with 150,150,3 image pixels. I don't want to use list anymore as they take more Memory and time consuming.
Please help me through this.
Also, as a new member if the question is not obeying the rules and regulations of StackOverflow please tell me and I will edit it more.
Thank you,

Both tensorflow/keras and pytorch provide data set / generator classes, which you can use to construct memory efficient data loaders.
For tensorflow/keras there is an excellent tutorial created by Stanford's Shervine Amidi.
For pytorch you find a good tutorial on the project's man page.
I would strongly suggest to make use of these frameworks for your implementation since they allow you to avoid writing boiler-plate code and make your training scalable.

Thank you for your help . But I wanted to do it manually to check How can we do it without using other generators. Below is my Code.
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpg
import numpy as np
import os
N = 0
training_labels = []
bbox = "D:/Morethan4000samples/data/labels"
imagepath = "D:/Morethan4000samples/data/image/"
for root, _, files in os.walk(imagepath):
cdp = os.path.abspath(root)
for f in files:
name, ext = os.path.splitext(f)
if ext == ".jpg":
cip = os.path.join(cdp,f)
N += 1
print(N)
imageX = np.zeros((N,227,227,3), dtype='float32')
i = 0
for root, _ , files in os.walk(imagepath):
cdp = os.path.abspath(root)
print(cdp)
for f in files:
ct = 0
name, ext = os.path.splitext(f)
if ext == ".jpg":
cip = os.path.join(cdp,f)
read = mpg.imread(cip)
cipLabel = cip.replace('image','labels')
cipLabel = cipLabel.replace('.jpg','.txt')
nameL , extL = os.path.splitext(cipLabel)
if extL == '.txt':
boxes = open(cipLabel, 'r')
for q in boxes:
ct = ct + 1
if ct == 3:
x1 = int(q.rsplit(' ')[0])
y1 = int(q.rsplit(' ')[1])
x2 = int(q.rsplit(' ')[2])
y2 = int(q.rsplit(' ')[3])
readimage = read[y1:y2, x1:x2]
resize = cv2.cv2.resize(readimage,(227,227))
resize = cv2.cv2.GaussianBlur(resize, (5,5),0)
imageX[i] = resize
#training_labels.append(int(cip.split('\\')[4]))
training_labels.append(int(cip.split('/')[8]))
print(len(training_labels), len(imageX))
i += 1
print(i)
imageX /= 255.0
plt.imshow(imageX[10])
plt.show()
print(imageX.shape)
print(len(training_labels))
np.save("/run/media/fdai5182/LAMA MADAN/Morethan4000samples/227227/training_images", imageX)
np.save("/run/media/fdai5182/LAMA MADAN/Morethan4000samples/227227/trainin_labels",training_labels)
To save each of your image in a row of matrix of same dimensions is the most efficient way to do that.

How can I add a list of saved images into an existing dataframe in pandas?

I was hoping somebody would be able to help me. I am trying to store a list of saved images from MatPlotLib as a dataframe (or a list) and then add it to an existing dataframe (effectively creating small barcharts for each entry in the dataframe e.g. databars).
I have managed to save the images successfully with a loop. There are 242 images. How can I show these images in a column in a dataframe. I want it to be easy to append it to my existing dataframe to show visually the number of zero values in this dataset. My code gives errors that it NoneType object is not iterable.
This is my code. (Top half just here for clarification as to what q1 and q2 are.)
Thanks.
import csv
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import sys
q1 = pandas.read_csv("data\q1.csv") #dataframe
q1.apply(lambda x: x.str.strip() if x.dtype == "object" else x) #strip whitespace
q1 = q1.dropna()
code = q1.loc[:,"Code"]
region = q1.loc[:,"Region"]
name = q1.loc[:,"Name"]
heads = list(q1.columns.values) #creates list of header values
nz = (q1 == 0).sum(axis=1) #count number of zero values in rows
q2 = q1[['Code','Region','Name']]
q2 = q2.assign(nz=nz.values)
samples=[]
y=1
for val in q2['nz']:
val = val/q2['nz'].max() * 100
plt.barh(val, width = val, color="blue")
plt.xlim((0,100))
plt.yticks([0])
plt.axis('off')
x = plt.savefig("value" + str(y) + ".png", bbox_inches='tight')
samples.append(x)
plt.close()
y = y + 1
imgdf = pandas.DataFrame.from_records(samples)
q3 = q2.append(imgdf)

If you are working in a jupyter notebook, then you can use the HTML display to show the images.
# Some imports
import base64
import pandas as pd
from PIL import Image
from io import BytesIO
from IPython.display import HTML
pd.set_option('display.max_colwidth', -1)
def get_thumbnail(path):
"""
Output a 150x150 sized PIL Image
"""
i = Image.open(path)
i.thumbnail((150, 150), Image.LANCZOS)
return i
def image_base64(im):
"""
Convert to base64 to be given as the src field of img in HTML
"""
if isinstance(im, str):
im = get_thumbnail(im)
with BytesIO() as buffer:
im.save(buffer, 'jpeg')
return base64.b64encode(buffer.getvalue()).decode()
def image_formatter(im):
return f'<img src="data:image/jpeg;base64,{image_base64(im)}">'
# Skipping some of your code
image_paths = []
for val in q2['nz']:
#... Do somethings here
x = plt.savefig("value" + str(y) + ".png", bbox_inches='tight')
plt.close()
image_paths.append("value" + str(y) + ".png")
y = y + 1
q2["images_paths"] = pd.Series(image_paths).values
q2["image"] = q2.image_paths.map(lambda f: get_thumbnail(f))
# Display PIL Images embedded in the dataframe
HTML(q2.to_html(formatters={"image": image_formatter}, escape=False))

Python Code - SVD with numpy

I would like to get some help with my code on Python. I am a novice to Python.
At high level - I read a (.png) file from command line , put into original array , compute svd , find high rank of svd based on command line , multiple with original array and then finally put the file and the array out.
My issue : The generated file is distorted and does not look like the real picture i intended to generate.
My question : I have put the snippet of code I am using , can you please point to what I am doing incorrectly ?
import sys
import os
import numpy
import numpy.linalg
import scipy.misc
def getOutputPngName(path, rank):
filename, ext = os.path.splitext(path)
return filename + '.' + str(rank) + '.png'
def getOutputNpyName(path, rank):
filename, ext = os.path.splitext(path)
return filename + '.' + str(rank) + '.npy'
if len(sys.argv) < 3:
sys.exit('usage: task1.py <PNG inputFile> <rank>')
inputfile = sys.argv[1]
rank = int(sys.argv[2])
outputpng = getOutputPngName(inputfile, rank)
outputnpy = getOutputNpyName(inputfile, rank)
# Import pic.png into array im as command parameter
img = scipy.misc.imread(inputfile)
# Perform SVD on im and obtain individual matrices
P, D, Q = numpy.linalg.svd(img, full_matrices=False)
# Compute overall SVD matrix based on individual matrices
svd_decomp = numpy.dot(numpy.dot(P, numpy.diag(D)), Q)
# Keep Top entries in svd_decomp
initial = svd_decomp.argsort()
temp = numpy.array(initial)
svd_final = numpy.argpartition(temp,-rank)[-rank:]
# Multiply to obtain the best rank-k approximation of the original array
img = numpy.transpose(img)
final = (numpy.dot(svd_final,img))
#Saving the approximated array as a binary array file(1) and as a PNG file(2)
numpy.save(outputnpy, final)
scipy.misc.imsave(outputpng, final)

The biggest issue is the svd_decomp.argsort(). argsort() without any arguments flattens out the whole matrix and sorts it like that, it's not what you want to do.
In fact, you don't need to do any sorting, because linalg's svd() function does it for you. See the documentation.
The singular values for every matrix, sorted in descending order.
So you just have to do the following
import sys
import os
import numpy
import numpy.linalg
import scipy.misc
def getOutputPngName(path, rank):
filename, ext = os.path.splitext(path)
return filename + '.' + str(rank) + '.png'
def getOutputNpyName(path, rank):
filename, ext = os.path.splitext(path)
return filename + '.' + str(rank) + '.npy'
if len(sys.argv) < 3:
sys.exit('usage: task1.py <PNG inputFile> <rank>')
inputfile = sys.argv[1]
rank = int(sys.argv[2])
outputpng = getOutputPngName(inputfile, rank)
outputnpy = getOutputNpyName(inputfile, rank)
# Import pic.png into array im as command parameter
img = scipy.misc.imread(inputfile)
# Perform SVD on im and obtain individual matrices
P, D, Q = numpy.linalg.svd(img, full_matrices=True)
# Select top "rank" singular values
svd_decomp = numpy.matrix(P[:, :rank]) * numpy.diag(D[:rank]) * numpy.matrix(Q[:rank, :])
# Save the output
numpy.save(outputnpy, svd_decomp)
scipy.misc.imsave(outputpng, svd_decomp)
Notice that all we do is select "rank" singular values, no need to sort.
Example outputs:
Base Image:
Rank = 1
Rank = 10

No Need to sort. Just compute your matrix over rank
svd_decomp = np.zeros((len(P), len(Q)))
for i in range(rank):
svd_decomp += D[i] * np.outer(P.T[i], Q[i])

Image segmentation in Python + Processing

I'm running into difficulties trying to run this image segmentation code.
The idea I have is to take an image such as:
http://imgur.com/a/AmcKq
and extract all the black squigglies and save each individual squiggly as its own image.
It seems like the code is working, but it's not segmenting my images for some reason.
The error I am getting is: ('segments detected:', 0)
This is the code Im using:
import os, sys
import numpy as np
from scipy import ndimage as ndi
from scipy.misc import imsave
import matplotlib.pyplot as plt
from skimage.filters import sobel, threshold_local
from skimage.morphology import watershed
from skimage import io
def open_image(name):
filename = os.path.join(os.getcwd(), name)
return io.imread(filename, as_grey=True)
def adaptive_threshold(image):
print(type(image))
print(image)
block_size = 41
binary_adaptive = threshold_local(image, block_size, offset=10)
binary_adaptive = np.asarray(binary_adaptive, dtype=int)
return np.invert(binary_adaptive) * 1.
def segmentize(image):
# make segmentation using edge-detection and watershed
edges = sobel(image)
markers = np.zeros_like(image)
foreground, background = 1, 2
markers[image == 0] = background
markers[image == 1] = foreground
ws = watershed(edges, markers)
return ndi.label(ws == foreground)
def find_segment(segments, index):
segment = np.where(segments == index)
shape = segments.shape
minx, maxx = max(segment[0].min() - 1, 0), min(segment[0].max() + 1, shape[0])
miny, maxy = max(segment[1].min() - 1, 0), min(segment[1].max() + 1, shape[1])
im = segments[minx:maxx, miny:maxy] == index
return (np.sum(im), np.invert(im))
def run(f):
print('Processing:', f)
image = open_image(f)
processed = adaptive_threshold(image)
segments = segmentize(processed)
print('Segments detected:', segments[1])
seg = []
for s in range(1, segments[1]):
seg.append(find_segment(segments[0], s))
seg.sort(key=lambda s: -s[0])
for i in range(len(seg)):
imsave('segments/' + f + '_' + str(i) + '.png', seg[i][1])
folder = os.path.join(os.getcwd(), 'segments')
os.path.isfile(folder) and os.remove(folder)
os.path.isdir(folder) or os.mkdir(folder)
for f in sys.argv[1:]:
run(f)
I'll also mention I'm running this Python script from within Processing 3.3.5 using this as my sketch file:
import deadpixel.command.Command;
static final String BASH =
platform == WINDOWS? "cmd /C " :
platform == MACOSX? "open" : "xdg-open";
static final String CD = "cd ", PY_APP = "python ";
static final String AMP = " && ", SPC = " ";
static final String PY_DIR = "scripts/";
//static final String PY_FILE = PY_DIR + "abc.py";
static final String PY_FILE = PY_DIR + "segmenting.py";
static final String PICS_DIR = "images/";
static final String PICS_EXTS = "extensions=,png,jpg,jpeg,gif";
void setup() {
final String dp = dataPath(""), py = dataPath(PY_FILE);
final String prompt = BASH + CD + dp + AMP + PY_APP + py;
final String pd = dataPath(PICS_DIR);
final String pics = join(listPaths(pd, PICS_EXTS), SPC);
final Command cmd = new Command(prompt + SPC + pics);
println(cmd.command, ENTER);
println("Successs:", cmd.run(), ENTER);
printArray(cmd.getOutput());
exit();
}
And this in a new tab in processing:
https://github.com/GoToLoop/command/blob/patch-1/src/deadpixel/command/Command.java

A quick investigation reveals the problem: this function here
def adaptive_threshold(image):
print(type(image))
print(image)
block_size = 41
binary_adaptive = threshold_local(image, block_size, offset=10)
binary_adaptive = np.asarray(binary_adaptive, dtype=int)
return np.invert(binary_adaptive) * 1.
is supposed to create a mask of the image by adaptive thresholding - but this goes (very) wrong.
The main reason seems to be a misunderstanding of how threshold_local works: this code expects it to return a binarized segmented version of the input image, when in reality it returns a threshold image, see explanation here.
This is not the only problem, however. For images like the one in your example, offset=10 reduces the threshold produced by threshold_local way too far, so the entire image would be above the threshold.
Here's a working version of the function:
def adaptive_threshold(image):
# Create threshold image
# Offset is not desirable for these images
block_size = 41
threshold_img = threshold_local(image, block_size)
# Binarize the image with the threshold image
binary_adaptive = image < threshold_img
# Convert the mask (which has dtype bool) to dtype int
# This is required for the code in `segmentize` (below) to work
binary_adaptive = binary_adaptive.astype(int)
# Return the binarized image
return binary_adaptive
If the code is run with this function (with python; this problem has nothing to do with Processing, as far as I can tell), it returns Segments detected: 108 and it produces a nice segmentation:
plt.imshow(segments[0],interpolation='none')
plt.show()
Side note: based on how you phrased your question, am I correct to assume that you did not write this code yourself and that you perhaps have limited expertise in this field?
If so, you may be interested in learning a bit more about python-based image processing and segmentation. I recently ran a short course on this topic that includes a completely self-explanatory hands-on tutorial of a pipeline similar to the one you are using here. The materials are openly accessible, so feel free to have a look.
Edit:
As per your comment, here is a solution that should allow the program to run with full paths as input.
First, remove all this:
folder = os.path.join(os.getcwd(), 'segments')
os.path.isfile(folder) and os.remove(folder)
os.path.isdir(folder) or os.mkdir(folder)
so that only this remains:
for f in sys.argv[1:]:
run(f)
Next, replace this:
for i in range(len(seg)):
imsave('segments/' + f + '_' + str(i) + '.png', seg[i][1])
by this:
# Get the directory name (if a full path is given)
folder = os.path.dirname(f)
# Get the file name
filenm = os.path.basename(f)
# If it doesn't already exist, create a new dir "segments"
# to save the PNGs
segments_folder = os.path.join(folder,"segments")
os.path.isdir(segments_folder) or os.mkdir(segments_folder)
# Save the segments to the "segments" directory
for i in range(len(seg)):
imsave(os.path.join(segments_folder, filenm + '_' + str(i) + '.png'), seg[i][1])
This solution can handle both files-only input (e.g 'test.png') and path input (e.g. 'C:\Users\Me\etc\test.png').
Edit 2:
For transparency, scipy.misc.imsave allows an alpha layer if arrays are saved as RGBA (MxNx4), see here.
Replace this
imsave(os.path.join(segments_folder, filenm + '_' + str(i) + '.png'), seg[i][1])
by this
# Create an MxNx4 array (RGBA)
seg_rgba = np.zeros((seg[i][1].shape[0],seg[i][1].shape[1],4),dtype=np.bool)
# Fill R, G and B with copies of the image
for c in range(3):
seg_rgba[:,:,c] = seg[i][1]
# For A (alpha), use the invert of the image (so background is 0=transparent)
seg_rgba[:,:,3] = ~seg[i][1]
# Save image
imsave(os.path.join(segments_folder, filenm + '_' + str(i) + '.png'), seg_rgba)
Edit 3:
For saving into a different target folder with individual subfolders for each segmented image:
Instead of this line
folder = os.path.dirname(f)
you can specify the target folder, for example
folder = r'C:\Users\Dude\Desktop'
(Note the r'...' formatting, which produces a raw string literal.)
Next, replace this
segments_folder = os.path.join(folder,"segments")
by this
segments_folder = os.path.join(folder,filenm[:-4]+"_segments")
and to be extra-clean replace this
imsave(os.path.join(segments_folder, filenm + '_' + str(i) + '.png'), seg_rgba)
by this
imsave(os.path.join(segments_folder, filenm[:-4] + '_' + str(i) + '.png'), seg_rgba)