We Keep Coding

Valueerror for reshaping array for image entropy

I have a code for finding entropy info of an image by resizing the image and dividing it into its RGB channels.
import os
from PIL import Image
import numpy as np
from scipy.misc import imread
import cv2
import imageio
#读取RGB图像
def openRGB(image_path):
f = open(image_path,"rb")
data = f.read()
f.close()
data = [int(x) for x in data]
data = np.array(data).reshape((256*256, 3)).astype(np.uint8)
return data
def entropy(X):
n = len(X)
counts = np.bincount(X)
probs = counts[np.nonzero(counts)] / n
en = 0
for i in range(len(probs)):
en = en - probs[i] * np.log(probs[i])/np.log(2)
return en
def getEntropy(image_path):
data =openRGB(image_path)
data_B = data[:, 0]
data_G = data[:, 1]
data_R = data[:, 2]
B = entropy(data_B)
G = entropy(data_G)
R = entropy(data_R)
return (R+B+G)/2;
However, whenever I run the getentropy() function on a given image it keeps giving back this error
ValueError: cannot reshape array of size 37048 into shape (65536,3)
Any idea how I can reformat the image to fit that array shape?

There is a simple explanation: image_path length in bytes is only 37048.
When using np.array(data).reshape((256*256, 3)), the length of data must be 256*256*3 = 196608 bytes.
You are getting an exception because the lengths do not match.
It's simple to reproduce the problem.
Create an input sample file in size 196608 bytes, and there is no exception.
Create an input sample file in size 37048 bytes, get an exception.
Here is a code sample that reproduces the problem:
import os
#from PIL import Image
import numpy as np
#from scipy.misc import imread
#import cv2
#import imageio
def openRGB(image_path):
f = open(image_path, "rb")
data = f.read()
f.close()
data = [int(x) for x in data]
data = np.array(data).reshape((256*256, 3)).astype(np.uint8)
return data
def entropy(X):
n = len(X)
counts = np.bincount(X)
probs = counts[np.nonzero(counts)] / n
en = 0
for i in range(len(probs)):
en = en - probs[i] * np.log(probs[i])/np.log(2)
return en
def getEntropy(image_path):
data = openRGB(image_path)
data_B = data[:, 0]
data_G = data[:, 1]
data_R = data[:, 2]
B = entropy(data_B)
G = entropy(data_G)
R = entropy(data_R)
return (R+B+G)/2
# Create a binary file with random bytes
image_path = 'tmp.bin'
# When n_bytes=196608, there is no exception.
################################################################################
n_bytes = 256*256*3
tmp = np.random.randint(0, 255, n_bytes, np.uint8) # Build random array of n_bytes bytes
with open(image_path, 'wb') as f:
f.write(tmp) # Write tmp to a binary file
file_size_in_bytes = os.path.getsize(image_path)
print('file_size_in_bytes = ' + str(file_size_in_bytes))
res = getEntropy(image_path)
print(res)
################################################################################
# When n_bytes=37048, an exception is raised: ValueError: cannot reshape array of size 37048 into shape (65536,3)
################################################################################
n_bytes = 37048
tmp = np.random.randint(0, 255, n_bytes, np.uint8) # Build random array of n_bytes bytes
with open(image_path, 'wb') as f:
f.write(tmp) # Write tmp to a binary file
file_size_in_bytes = os.path.getsize(image_path)
print('file_size_in_bytes = ' + str(file_size_in_bytes))
res = getEntropy(image_path)
print(res)
################################################################################
Why are you reading 37048 bytes instead of 196608 bytes?
image_path is a JPEG image file, and you are reading the image using f = open(image_path, "rb") and data = f.read().
You may read and reshape the image as follows:
import cv2
def openRGB(image_path):
# For example: image_path = 'img.jpg'
img = cv2.imread(image_path) # Read image in BGR color format.
data = np.array(img).reshape(img.shape[0]*img.shape[1], 3) # Reshape to rows x cols x 3 (Blue in data[:, 0], green in data[:, 1], red in data[:, 2]).
return data
In the above example I used img.shape[0]*img.shape[1], the image resolution is:
height = img.shape[0]
width = img.shape[1]

Why converting npy files (containing video frames) to tfrecords consumes too much disk space?

I am working on a violence detection service. I am trying to develop software based on the code in this repo. My dataset consists of videos resided in two directories "Violence" and "Non-Violence".
I used this code to generate npy files out of RGB channels and optical flow features. The output of this part would be 2 folders containing npy array with 244x244x5 shape. (np.float32 dtype). so it's like I have video frames in RGB in the first 3 channels (npy[...,:3]) and optical flow features in the next two channels (npy[..., 3:]).
Now I am trying to convert them to tfrecords and use tf.data.tfrecorddataset to speed up the training process. Since my model input has to be a cube tensor, my training elements has to be 64 frames of each video. It means the data point shape has to be 64x244x244x5.
So I used this code to convert the npy files to tfrecords.
from pathlib import Path
from os.path import join
import tensorflow as tf
import numpy as np
import cv2
from tqdm import tqdm
def normalize(data):
mean = np.mean(data)
std = np.std(data)
return (data - mean) / std
def random_flip(video, prob):
s = np.random.rand()
if s < prob:
video = np.flip(m=video, axis=2)
return video
def color_jitter(video):
# range of s-component: 0-1
# range of v component: 0-255
s_jitter = np.random.uniform(-0.2, 0.2)
v_jitter = np.random.uniform(-30, 30)
for i in range(len(video)):
hsv = cv2.cvtColor(video[i], cv2.COLOR_RGB2HSV)
s = hsv[..., 1] + s_jitter
v = hsv[..., 2] + v_jitter
s[s < 0] = 0
s[s > 1] = 1
v[v < 0] = 0
v[v > 255] = 255
hsv[..., 1] = s
hsv[..., 2] = v
video[i] = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
return video
def uniform_sample(video: str, target_frames: int = 64) -> np.ndarray:
"""
gets video and outputs n_frames number of frames in video.
Args:
video:
target_frames:
Returns:
"""
len_frames = int(len(data))
interval = int(np.ceil(len_frames / target_frames))
# init empty list for sampled video and
sampled_video = []
for i in range(0, len_frames, interval):
sampled_video.append(video[i])
# calculate number of padded frames and fix it
num_pad = target_frames - len(sampled_video)
if num_pad > 0:
padding = [video[i] for i in range(-num_pad, 0)]
sampled_video += padding
return np.array(sampled_video, dtype=np.float32)
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
if __name__ == '__main__':
path = Path('transformed/')
npy_files = list(path.rglob('*.npy'))[:100]
aug = True
# one_hots = to_categorical(range(2), dtype=np.int8)
path_to_save = 'data_tfrecords'
tfrecord_path = join(path_to_save, 'all_data.tfrecord')
with tf.io.TFRecordWriter(tfrecord_path) as writer:
for file in tqdm(npy_files, desc='files converted'):
# load npy files
npy = np.load(file.as_posix(), mmap_mode='r')
data = np.float32(npy)
del npy
# Uniform sampling
data = uniform_sample(data, target_frames=64)
# Add augmentation
if aug:
data[..., :3] = color_jitter(data[..., :3])
data = random_flip(data, prob=0.5)
# Normalization
data[..., :3] = normalize(data[..., :3])
data[..., 3:] = normalize(data[..., 3:])
# Label one hot encoding
label = 1 if file.parent.stem.startswith('F') else 0
# label = one_hots[label]
feature = {'image': _bytes_feature(tf.compat.as_bytes(data.tobytes())),
'label': _int64_feature(int(label))}
example = tf.train.Example(features=tf.train.Features(feature=feature))
writer.write(example.SerializeToString())
The code works fine, but the real problem is that it consumes too much disk drive. my whole dataset consisting of 2000 videos takes 12 GB, when I converted them to npy files, it became around 80 GB, and now using tfrecords It became over 120 GB or so. How can I convert them in an efficient way to reduce the space required to store them?

The answer might be too late. But I see you are still saving the video frame in your tfrecords file.
Try removing the "image" feature from your features list. And saving per frame as their Height, Width, Channels, and so forth.
feature = {'label': _int64_feature(int(label))}
Which is why the file is taking more space.

convert numpy array to uint8 using python

My code below is intended to get a batch of images and convert them to RGB. But I keep getting an error which says to convert to type uint8. I have seen other questions regarding the conversion to uint8, but none directly from an array to uint8. Any advice on how to make that happen is welcome, thank you!
from skimage import io
import numpy as np
import glob, os
from tkinter import Tk
from tkinter.filedialog import askdirectory
import cv2
# wavelength in microns
MWIR = 4.5
R = .692
G = .582
B = .140
rgb_sum = R + G + B;
NRed = R/rgb_sum;
NGreen = G/rgb_sum;
NBlue = B/rgb_sum;
path = askdirectory(title='Select PNG Folder') # shows dialog box and return the path
outpath = askdirectory(title='Select SAVE Folder')
for file in os.listdir(path):
if file.endswith(".png"):
imIn = io.imread(os.path.join(path, file))
imOut = np.zeros(imIn.shape)
for i in range(imIn.shape[0]): # Assuming Rayleigh-Jeans law
for j in range(imIn.shape[1]):
imOut[i,j,0] = imIn[i,j,0]/((NRed/MWIR)**4)
imOut[i,j,1] = imIn[i,j,0]/((NGreen/MWIR)**4)
imOut[i,j,2] = imIn[i,j,0]/((NBlue/MWIR)**4)
io.imsave(os.path.join(outpath, file) + '_RGB.png', imOut)
the code I am trying to integrate into my own (found in another thread, used to convert type to uint8) is:
info = np.iinfo(data.dtype) # Get the information of the incoming image type
data = data.astype(np.float64) / info.max # normalize the data to 0 - 1
data = 255 * data # Now scale by 255
img = data.astype(np.uint8)
cv2.imshow("Window", img)
thank you!

Normally imInt is of type uint8, after your normalisation it is of type float32 because of the casting cause by the division. you must convert back to uint8 before saving to PNG file:
io.imsave(os.path.join(outpath, file) + '_RGB.png', imOut.astype(np.uint8))
Note that the two loops are not necessary, you can use numpy vector operations instead:
MWIR = 4.5
R = .692
G = .582
B = .140
vector = [R, G, B]
vector = vector / vector.sum()
vector = vector / MWIR
vector = np.pow(vector, 4)
for file in os.listdir(path):
if file.endswith((".png"):
imgIn = ...
imgOut = imgIn * vector
io.imsave(
os.path.join(outpath, file) + '_RGB.png',
imgOut.astype(np.uint8))

ValueError: need at least one array to concatenate

I am having issues with
ValueError: need at least one array to concatenate
Below is the whole error message.
Training mode
Traceback (most recent call last):
File "bcf.py", line 342, in <module>
bcf.train()
File "bcf.py", line 321, in train
self._learn_codebook()
File "bcf.py", line 142, in _learn_codebook
feats_sc = np.concatenate(feats_sc, axis=1).transpose()
ValueError: need at least one array to concatenate
Below is the area of the problem.
def _learn_codebook(self):
MAX_CFS = 800 # max number of contour fragments per image; if above, sample randomly
CLUSTERING_CENTERS = 1500
feats_sc = []
for image in self.data.values():
feats = image['cfs']
feat_sc = feats[1]
if feat_sc.shape[1] > MAX_CFS:
# Sample MAX_CFS from contour fragments
rand_indices = np.random.permutation(feat_sc.shape[1])
feat_sc = feat_sc[:, rand_indices[:MAX_CFS]]
feats_sc.append(feat_sc)
feats_sc = np.concatenate(feats_sc, axis=1).transpose()
print("Running KMeans...")
self.kmeans = sklearn.cluster.KMeans(min(CLUSTERING_CENTERS, feats_sc.shape[0]), n_jobs=-1, algorithm='elkan').fit(feats_sc)
print("Saving codebook...")
self._save_kmeans(self.kmeans)
return self.kmeans
Below is the complete CLASS
class BCF():
def __init__(self):
self.DATA_DIR = "/Users/minniemouse/TRAIN/bcf-master5/data/cuauv/"
self.PERC_TRAINING_PER_CLASS = 0.5
self.CODEBOOK_FILE = "codebook.data"
self.CLASSIFIER_FILE = "classifier"
self.LABEL_TO_CLASS_MAPPING_FILE = "labels_to_classes.data"
self.classes = defaultdict(list)
self.data = defaultdict(dict)
self.counter = defaultdict(int)
self.kmeans = None
self.clf = None
self.label_to_class_mapping = None
def _load_classes(self):
for dir_name, subdir_list, file_list in os.walk(self.DATA_DIR):
if subdir_list:
continue
for f in sorted(file_list, key=hash):
self.classes[dir_name.split('/')[-1]].append(os.path.join(dir_name, f))
def _load_training(self):
for cls in self.classes:
images = self.classes[cls]
for image in images[:int(len(images) * self.PERC_TRAINING_PER_CLASS)]:
image_id = self._get_image_identifier(cls)
self.data[image_id]['image'] = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
if self.data[image_id]['image'] is None:
print("Failed to load " + image)
def _load_testing(self):
for cls in self.classes:
images = self.classes[cls]
for image in images[int(len(images) * self.PERC_TRAINING_PER_CLASS):]:
image_id = self._get_image_identifier(cls)
self.data[image_id]['image'] = cv2.imread(image, cv2.IMREAD_GRAYSCALE)
if self.data[image_id]['image'] is None:
print("Failed to load " + image)
def _load_single(self, image):
# Load single image data
self.data.clear()
image_id = self._get_image_identifier(None)
self.data[image_id]['image'] = image
def _save_label_to_class_mapping(self):
self.label_to_class_mapping = {hash(cls): cls for cls in self.classes}
with open(self.LABEL_TO_CLASS_MAPPING_FILE, 'wb') as out_file:
pickle.dump(self.label_to_class_mapping, out_file, -1)
def _load_label_to_class_mapping(self):
if self.label_to_class_mapping is None:
with open(self.LABEL_TO_CLASS_MAPPING_FILE, 'rb') as in_file:
self.label_to_class_mapping = pickle.load(in_file)
return self.label_to_class_mapping
def _normalize_shapes(self):
for (cls, idx) in self.data.keys():
image = self.data[(cls, idx)]['image']
# Remove void space
y, x = np.where(image > 50)
max_y = y.max()
min_y = y.min()
max_x = x.max()
min_x = x.min()
trimmed = image[min_y:max_y, min_x:max_x] > 50
trimmed = trimmed.astype('uint8')
trimmed[trimmed > 0] = 255
self.data[(cls, idx)]['normalized_image'] = trimmed
def _extract_cf(self):
for (cls, idx) in self.data.keys():
image = self.data[(cls, idx)]['normalized_image']
images,contours, _ = cv2.findContours(image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contour = sorted(contours, key=len)[-1]
mat = np.zeros(image.shape, np.int8)
cv2.drawContours(mat, [contour], -1, (255, 255, 255))
#self.show(mat)
MAX_CURVATURE = 1.5
N_CONTSAMP = 50
N_PNTSAMP = 10
C = None
for pnt in contour:
if C is None:
C = np.array([[pnt[0][0], pnt[0][1]]])
else:
C = np.append(C, [[pnt[0][0], pnt[0][1]]], axis=0)
cfs = self._extr_raw_points(C, MAX_CURVATURE, N_CONTSAMP, N_PNTSAMP)
tmp = mat.copy()
for cf in cfs:
for pnt in cf:
cv2.circle(tmp, (pnt[0], pnt[1]), 2, (255, 0, 0))
#self.show(tmp)
num_cfs = len(cfs)
print("Extracted %s points" % (num_cfs))
feat_sc = np.zeros((300, num_cfs))
xy = np.zeros((num_cfs, 2))
for i in range(num_cfs):
cf = cfs[i]
sc, _, _, _ = shape_context(cf)
# shape context is 60x5 (60 bins at 5 reference points)
sc = sc.flatten(order='F')
sc /= np.sum(sc) # normalize
feat_sc[:, i] = sc
# shape context descriptor sc for each cf is 300x1
# save a point at the midpoint of the contour fragment
xy[i, 0:2] = cf[np.round(len(cf) / 2. - 1).astype('int32'), :]
sz = image.shape
self.data[(cls, idx)]['cfs'] = (cfs, feat_sc, xy, sz)
def _learn_codebook(self):
MAX_CFS = 800 # max number of contour fragments per image; if above, sample randomly
CLUSTERING_CENTERS = 1500
feats_sc = []
for image in self.data.values():
feats = image['cfs']
feat_sc = feats[1]
if feat_sc.shape[1] > MAX_CFS:
# Sample MAX_CFS from contour fragments
rand_indices = np.random.permutation(feat_sc.shape[1])
feat_sc = feat_sc[:, rand_indices[:MAX_CFS]]
feats_sc.append(feat_sc)
feats_sc = np.concatenate(feats_sc, axis=1).transpose()
print("Running KMeans...")
self.kmeans = sklearn.cluster.KMeans(min(CLUSTERING_CENTERS, feats_sc.shape[0]), n_jobs=-1, algorithm='elkan').fit(feats_sc)
print("Saving codebook...")
self._save_kmeans(self.kmeans)
return self.kmeans
I have read through the various posts on ValueError already described, but I am not having much luck on figuring it out. I have now attached the CLASS and full error message information.
Please, can someone point out what I am missing?
Thank you

the problem comes from the lenght of your array. Check if your array/list is longer than to 0 print(len(feats_sc)).
Don't forget to checkout the documentation numpy.concatenate — NumPy v1.16 Manual

The problem seems to be in np.concatenate where it expects an array of arrays and it's not receiving that.
Refer: Scipy docs
numpy.concatenate((a1, a2, ...), axis=0, out=None)
Join a sequence of arrays along an existing axis.
Parameters:
a1, a2, … : sequence of array_like The arrays must have
the same shape, except in the dimension corresponding to axis (the
first, by default).
axis : int, optional The axis along which the arrays will be joined.
If axis is None, arrays are flattened before use. Default is 0.
out : ndarray, optional If provided, the destination to place the
result. The shape must be correct, matching that of what concatenate
would have returned if no out argument were specified.
Returns: res : ndarray The concatenated array.
In your case, check what feats_sc contains.
You can debug using pdb
python -m pdb <your-code>.py
(pdb) b fullpath/to/your-code.py:line-number-to-break
(pdb) c
c will continue until break point in encountered
n will move to next line
b is to set break point
q is to quit

Just to make it clearer, running the following piece of code throws the same ValueError: need at least one array to concatenate error.
import numpy as np
feats_sc = np.array([])
feats_sc = np.concatenate(feats_sc, axis=1)
whereas the following code does not.
import numpy as np
feats_sc = np.array(([[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [1 ,2 ,3]]))
feats_sc = np.concatenate(feats_sc, axis=1)
The reason is that in the former, the numpy array is empty, and in the latter, it is not.

Convert mha 3d images into 2d images in Python (2015 brats challenge dataset)

I want to use SimpleITK or wedpy to convert the 3d images into 2d images.
Or i want to get a three-dimensional matrix, and then i divide the three-dimensional matrix into some two-dimensional matrices.
import SimpleITK as ITK
import numpy as np
#from medpy.io import load
url=r'G:\path\to\my.mha'
image = ITK.ReadImage(url)
frame_num, width, height = image_array.shape
print(frame_num,width,height)
Then only get it:155 240 240
but i want to get [[1,5,2,3,1...],[54,1,3,5...],[5,8,9,6....]]

Just to add to Dave Chen's answer, as it is unclear if you want to get a set of 2D SimpleITK images or numpy arrays. The following code covers all three available options:
import SimpleITK as sitk
import numpy as np
url = "my_file.mha"
image = sitk.ReadImage(url)
max_index = image.GetDepth() # or image.GetWidth() or image.GetHeight() depending on the axis along which you want to extract
# As list of 2D SimpleITK images
list_of_2D_images = [image[:,:,i] for i in range(max_index)]
# As list of 2D numpy arrays which cannot be modified (no data copied)
list_of_2D_images_np_view = [sitk.GetArrayViewFromImage(image[:,:,i]) for i in range(max_index)]
# As list of 2D numpy arrays (data copied to numpy array)
list_of_2D_images_np = [sitk.GetArrayFromImage(image[:,:,i]) for i in range(max_index)]
Also, if you really want to work with URLs and not local files I would suggest looking at the remote download approach used in the SimpleITK notebooks repository, the relevant file is downloaddata.py.

That's not a big deal.
CT images have originally all numbers in int16 type so you don't need to handle float numbers.. In this case, we can think that we can easily change from int16 to uint16 only removing negative values in the image (CT images have some negative numbers as pixel values). Note that we really need uint16, or uint8 type so that OpenCV can handle it... as we have a lot of values in the CT image array, the best choice is uint16, so that we don't lose too much precision.
Ok, now you just need to do as follows:
import SimpleITK as sitk
import numpy as np
import cv2
mha = sitk.ReadImage('/mha/directory') #Importing mha file
array = sitk.GetArrayFromImage(mha) #Converting to array int16 (default)
#Translating each slice to the positive side
for m in range(array.shape[0]):
array[m] = array[m] + abs(np.min(array[m]))
array = np.around(array, decimals=0) #remove any float numbers if exists.. probably not
array = np.asarray(array, dtype='uint16') #From int16 to uint16
After these steps the array is just ready to be saved as png images using opencv.imwrite module:
for image in array:
cv2.imwrite('/dir/to/save/'+'name_image.png', image)
Note that by default SimpleITK handles .mha files by the axial view. I really don't know how to change it because I've never needed it before. Anyway, in this case with some searches you can find something.

I'm not sure exactly what you want to get. But it's easy to extract a 2d slice from a 3d image in SimpleITK.
To get a Z slice where Z=100 you can do this:
zslice = image[100]
To get a Y slice for Y=100:
yslice = image[:, 100]
And a X slice for X=100:
xslice = image[:, :, 100]

#zivy#Dave Chen
I've solved my problem.In fact, running this code will give you 150 240*240 PNG pictures.It's i want to get.
# -*- coding:utf-8 -*-
import numpy as np
import subprocess
import random
import progressbar
from glob import glob
from skimage import io
np.random.seed(5) # for reproducibility
progress = progressbar.ProgressBar(widgets=[progressbar.Bar('*', '[', ']'), progressbar.Percentage(), ' '])
class BrainPipeline(object):
'''
A class for processing brain scans for one patient
INPUT: (1) filepath 'path': path to directory of one patient. Contains following mha files:
flair, t1, t1c, t2, ground truth (gt)
(2) bool 'n4itk': True to use n4itk normed t1 scans (defaults to True)
(3) bool 'n4itk_apply': True to apply and save n4itk filter to t1 and t1c scans for given patient. This will only work if the
'''
def __init__(self, path, n4itk = True, n4itk_apply = False):
self.path = path
self.n4itk = n4itk
self.n4itk_apply = n4itk_apply
self.modes = ['flair', 't1', 't1c', 't2', 'gt']
# slices=[[flair x 155], [t1], [t1c], [t2], [gt]], 155 per modality
self.slices_by_mode, n = self.read_scans()
# [ [slice1 x 5], [slice2 x 5], ..., [slice155 x 5]]
self.slices_by_slice = n
self.normed_slices = self.norm_slices()
def read_scans(self):
'''
goes into each modality in patient directory and loads individual scans.
transforms scans of same slice into strip of 5 images
'''
print('Loading scans...')
slices_by_mode = np.zeros((5, 155, 240, 240))
slices_by_slice = np.zeros((155, 5, 240, 240))
flair = glob(self.path + '/*Flair*/*.mha')
t2 = glob(self.path + '/*_T2*/*.mha')
gt = glob(self.path + '/*more*/*.mha')
t1s = glob(self.path + '/**/*T1*.mha')
t1_n4 = glob(self.path + '/*T1*/*_n.mha')
t1 = [scan for scan in t1s if scan not in t1_n4]
scans = [flair[0], t1[0], t1[1], t2[0], gt[0]] # directories to each image (5 total)
if self.n4itk_apply:
print('-> Applyling bias correction...')
for t1_path in t1:
self.n4itk_norm(t1_path) # normalize files
scans = [flair[0], t1_n4[0], t1_n4[1], t2[0], gt[0]]
elif self.n4itk:
scans = [flair[0], t1_n4[0], t1_n4[1], t2[0], gt[0]]
for scan_idx in xrange(5):
# read each image directory, save to self.slices
slices_by_mode[scan_idx] = io.imread(scans[scan_idx], plugin='simpleitk').astype(float)
for mode_ix in xrange(slices_by_mode.shape[0]): # modes 1 thru 5
for slice_ix in xrange(slices_by_mode.shape[1]): # slices 1 thru 155
slices_by_slice[slice_ix][mode_ix] = slices_by_mode[mode_ix][slice_ix] # reshape by slice
return slices_by_mode, slices_by_slice
def norm_slices(self):
'''
normalizes each slice in self.slices_by_slice, excluding gt
subtracts mean and div by std dev for each slice
clips top and bottom one percent of pixel intensities
if n4itk == True, will apply n4itk bias correction to T1 and T1c images
'''
print('Normalizing slices...')
normed_slices = np.zeros((155, 5, 240, 240))
for slice_ix in xrange(155):
normed_slices[slice_ix][-1] = self.slices_by_slice[slice_ix][-1]
for mode_ix in xrange(4):
normed_slices[slice_ix][mode_ix] = self._normalize(self.slices_by_slice[slice_ix][mode_ix])
print('Done.')
return normed_slices
def _normalize(self, slice):
'''
INPUT: (1) a single slice of any given modality (excluding gt)
(2) index of modality assoc with slice (0=flair, 1=t1, 2=t1c, 3=t2)
OUTPUT: normalized slice
'''
b, t = np.percentile(slice, (0.5,99.5))
slice = np.clip(slice, b, t)
if np.std(slice) == 0:
return slice
else:
return (slice - np.mean(slice)) / np.std(slice)
def save_patient(self, reg_norm_n4, patient_num):
'''
INPUT: (1) int 'patient_num': unique identifier for each patient
(2) string 'reg_norm_n4': 'reg' for original images, 'norm' normalized images, 'n4' for n4 normalized images
OUTPUT: saves png in Norm_PNG directory for normed, Training_PNG for reg
'''
print('Saving scans for patient {}...'.format(patient_num))
progress.currval = 0
if reg_norm_n4 == 'norm': #saved normed slices
for slice_ix in progress(xrange(155)): # reshape to strip
strip = self.normed_slices[slice_ix].reshape(1200, 240)
if np.max(strip) != 0: # set values < 1
strip /= np.max(strip)
if np.min(strip) <= -1: # set values > -1
strip /= abs(np.min(strip))
# save as patient_slice.png
io.imsave('Norm_PNG/{}_{}.png'.format(patient_num, slice_ix), strip)
elif reg_norm_n4 == 'reg':
for slice_ix in progress(xrange(155)):
strip = self.slices_by_slice[slice_ix].reshape(1200, 240)
if np.max(strip) != 0:
strip /= np.max(strip)
io.imsave('Training_PNG/{}_{}.png'.format(patient_num, slice_ix), strip)
else:
for slice_ix in progress(xrange(155)): # reshape to strip
strip = self.normed_slices[slice_ix].reshape(1200, 240)
if np.max(strip) != 0: # set values < 1
strip /= np.max(strip)
if np.min(strip) <= -1: # set values > -1
strip /= abs(np.min(strip))
# save as patient_slice.png
io.imsave('n4_PNG/{}_{}.png'.format(patient_num, slice_ix), strip)
def n4itk_norm(self, path, n_dims=3, n_iters='[20,20,10,5]'):
'''
INPUT: (1) filepath 'path': path to mha T1 or T1c file
(2) directory 'parent_dir': parent directory to mha file
OUTPUT: writes n4itk normalized image to parent_dir under orig_filename_n.mha
'''
output_fn = path[:-4] + '_n.mha'
# run n4_bias_correction.py path n_dim n_iters output_fn
subprocess.call('python n4_bias_correction.py ' + path + ' ' + str(n_dims) + ' ' + n_iters + ' ' + output_fn, shell = True)
def save_patient_slices(patients, type):
'''
INPUT (1) list 'patients': paths to any directories of patients to save. for example- glob("Training/HGG/**")
(2) string 'type': options = reg (non-normalized), norm (normalized, but no bias correction), n4 (bias corrected and normalized)
saves strips of patient slices to approriate directory (Training_PNG/, Norm_PNG/ or n4_PNG/) as patient-num_slice-num
'''
for patient_num, path in enumerate(patients):
a = BrainPipeline(path)
a.save_patient(type, patient_num)
def s3_dump(directory, bucket):
'''
dump files from a given directory to an s3 bucket
INPUT (1) string 'directory': directory containing files to save
(2) string 'bucket': name od s3 bucket to dump files
'''
subprocess.call('aws s3 cp' + ' ' + directory + ' ' + 's3://' + bucket + ' ' + '--recursive')
def save_labels(fns):
'''
INPUT list 'fns': filepaths to all labels
'''
progress.currval = 0
for label_idx in progress(range(len(labels))):
slices = io.imread(labels[label_idx], plugin = 'simpleitk')
for slice_idx in range(len(slices)):
io.imsave(r'{}_{}L.png'.format(label_idx, slice_idx), slices[slice_idx])
if __name__ == '__main__':
url = r'G:\work\deeplearning\BRATS2015_Training\HGG\brats_2013_pat0005_1\VSD.Brain.XX.O.MR_T1.54537\VSD.Brain.XX.O.MR_T1.54537.mha'
labels = glob(url)
save_labels(labels)
# patients = glob('Training/HGG/**')
# save_patient_slices(patients, 'reg')
# save_patient_slices(patients, 'norm')
# save_patient_slices(patients, 'n4')
# s3_dump('Graveyard/Training_PNG/', 'orig-training-png')