We Keep Coding

Error when running ImageDataGenerator.next() code

I am currently running python 3.8.6, and when running the following code from https://machinelearningmastery.com/how-to-configure-image-data-augmentation-when-training-deep-learning-neural-networks/ I get an error saying that scipy is not defined.
# example of random rotation image augmentation
from numpy import expand_dims
from keras.preprocessing.image import ImageDataGenerator
from matplotlib import pyplot
# load the image
img = tf.keras.preprocessing.image.load_img('campaign_data/campaign2/0000/image_0000500.png')
# convert to numpy array
data = tf.keras.preprocessing.image.img_to_array(img)
# expand dimension to one sample
samples = expand_dims(data, 0)
# create image data augmentation generator
datagen = ImageDataGenerator(rotation_range=90)
# prepare iterator
it = datagen.flow(samples, batch_size=1)
# generate samples and plot
for i in range(9):
# define subplot
pyplot.subplot(330 + 1 + i)
# generate batch of images
batch = it.next()
# convert to unsigned integers for viewing
image = batch[0].astype('uint8')
# plot raw pixel data
pyplot.imshow(image)
# show the figure
pyplot.show()
'''
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Input In [131], in <cell line: 18>()
20 pyplot.subplot(330 + 1 + i)
21 # generate batch of images
---> 22 batch = it.next()
23 # convert to unsigned integers for viewing
24 image = batch[0].astype('uint8')
File ~/jupyter-venv/lib/python3.8/site-packages/keras/preprocessing/image.py:160, in Iterator.next(self)
157 index_array = next(self.index_generator)
158 # The transformation of images is not under thread lock
159 # so it can be done in parallel
--> 160 return self._get_batches_of_transformed_samples(index_array)
File ~/jupyter-venv/lib/python3.8/site-packages/keras/preprocessing/image.py:709, in NumpyArrayIterator._get_batches_of_transformed_samples(self, index_array)
707 x = self.x[j]
708 params = self.image_data_generator.get_random_transform(x.shape)
--> 709 x = self.image_data_generator.apply_transform(
710 x.astype(self.dtype), params)
711 x = self.image_data_generator.standardize(x)
712 batch_x[i] = x
File ~/jupyter-venv/lib/python3.8/site-packages/keras/preprocessing/image.py:1800, in ImageDataGenerator.apply_transform(self, x, transform_parameters)
1797 img_col_axis = self.col_axis - 1
1798 img_channel_axis = self.channel_axis - 1
-> 1800 x = apply_affine_transform(
1801 x,
1802 transform_parameters.get('theta', 0),
1803 transform_parameters.get('tx', 0),
1804 transform_parameters.get('ty', 0),
1805 transform_parameters.get('shear', 0),
1806 transform_parameters.get('zx', 1),
1807 transform_parameters.get('zy', 1),
1808 row_axis=img_row_axis,
1809 col_axis=img_col_axis,
1810 channel_axis=img_channel_axis,
1811 fill_mode=self.fill_mode,
1812 cval=self.cval,
1813 order=self.interpolation_order)
1815 if transform_parameters.get('channel_shift_intensity') is not None:
1816 x = apply_channel_shift(x,
1817 transform_parameters['channel_shift_intensity'],
1818 img_channel_axis)
File ~/jupyter-venv/lib/python3.8/site-packages/keras/preprocessing/image.py:2244, in apply_affine_transform(x, theta, tx, ty, shear, zx, zy, row_axis, col_axis, channel_axis, fill_mode, cval, order)
2212 #keras_export('keras.preprocessing.image.apply_affine_transform')
2213 def apply_affine_transform(x, theta=0, tx=0, ty=0, shear=0, zx=1, zy=1,
2214 row_axis=1, col_axis=2, channel_axis=0,
2215 fill_mode='nearest', cval=0., order=1):
2216 """Applies an affine transformation specified by the parameters given.
2217
2218 Args:
(...)
2242 ImportError: if SciPy is not available.
2243 """
-> 2244 if scipy is None:
2245 raise ImportError('Image transformations require SciPy. '
2246 'Install SciPy.')
2248 # Input sanity checks:
2249 # 1. x must 2D image with one or more channels (i.e., a 3D tensor)
2250 # 2. channels must be either first or last dimension
NameError: name 'scipy' is not defined
'''
I cannot seem to find any solution on how to solve this bug, but any help will be greatly appreciated.

I got the same error when scipy was not installed. After installing scipy it worked. Please use:
pip install scipy.

Feature analysis using histograms -- LinAlgError: singular matrix

I am trying to create histograms for feature analysis to see how similar high grade tumor and low grade tumor characteristics are. Something similar to the histogram shown below:
Matrix "Z" looks like this and contains 106 features
Matrix "Z"
The code I am using is:
#Import data
data = pd.read_csv("/Users/MLuser/Desktop/Data table - CM_Transposed.csv")
# Prproccess data
bins = (0,2,4)
group_names = ['low_grade','high_grade']
data['FGrade'] = pd.cut(data['FGrade'], bins=bins, labels=group_names)
label_grade = LabelEncoder()
data['FGrade'] = label_grade.fit_transform(data['FGrade'])
Z=data
# Separate the dataset as response variable and feature variables
Z = Z.drop('Feature Name', axis = 1)
Z.columns
low_grade=Z[Z.FGrade==0] # define malignant
high_grade=Z[Z.FGrade==1] # define benign
for i, col in enumerate(Z.columns):
plt.figure(i)
sns.distplot(low_grade.iloc[:,i], color='red')
sns.distplot(high_grade.iloc[:,i], color='g')
Unfortunately when I run the above, I get a single graph and the following error:
---------------------------------------------------------------------------
LinAlgError Traceback (most recent call last)
<ipython-input-25-79d6e8cd51a8> in <module>
27 for i, col in enumerate(Z.columns):
28 plt.figure(i)
---> 29 sns.distplot(low_grade.iloc[:,i], color='red')
30 sns.distplot(high_grade.iloc[:,i], color='g')
31
~/env/lib/python3.6/site-packages/seaborn/distributions.py in distplot(a, bins, hist, kde, rug, fit, hist_kws, kde_kws, rug_kws, fit_kws, color, vertical, norm_hist, axlabel, label, ax)
229 if kde:
230 kde_color = kde_kws.pop("color", color)
--> 231 kdeplot(a, vertical=vertical, ax=ax, color=kde_color, **kde_kws)
232 if kde_color != color:
233 kde_kws["color"] = kde_color
~/env/lib/python3.6/site-packages/seaborn/distributions.py in kdeplot(data, data2, shade, vertical, kernel, bw, gridsize, cut, clip, legend, cumulative, shade_lowest, cbar, cbar_ax, cbar_kws, ax, **kwargs)
689 ax = _univariate_kdeplot(data, shade, vertical, kernel, bw,
690 gridsize, cut, clip, legend, ax,
--> 691 cumulative=cumulative, **kwargs)
692
693 return ax
~/env/lib/python3.6/site-packages/seaborn/distributions.py in _univariate_kdeplot(data, shade, vertical, kernel, bw, gridsize, cut, clip, legend, ax, cumulative, **kwargs)
292 "only implemented in statsmodels."
293 "Please install statsmodels.")
--> 294 x, y = _scipy_univariate_kde(data, bw, gridsize, cut, clip)
295
296 # Make sure the density is nonnegative
~/env/lib/python3.6/site-packages/seaborn/distributions.py in _scipy_univariate_kde(data, bw, gridsize, cut, clip)
364 """Compute a univariate kernel density estimate using scipy."""
365 try:
--> 366 kde = stats.gaussian_kde(data, bw_method=bw)
367 except TypeError:
368 kde = stats.gaussian_kde(data)
~/env/lib/python3.6/site-packages/scipy/stats/kde.py in __init__(self, dataset, bw_method, weights)
206 self._neff = 1/sum(self._weights**2)
207
--> 208 self.set_bandwidth(bw_method=bw_method)
209
210 def evaluate(self, points):
~/env/lib/python3.6/site-packages/scipy/stats/kde.py in set_bandwidth(self, bw_method)
552 raise ValueError(msg)
553
--> 554 self._compute_covariance()
555
556 def _compute_covariance(self):
~/env/lib/python3.6/site-packages/scipy/stats/kde.py in _compute_covariance(self)
564 bias=False,
565 aweights=self.weights))
--> 566 self._data_inv_cov = linalg.inv(self._data_covariance)
567
568 self.covariance = self._data_covariance * self.factor**2
~/env/lib/python3.6/site-packages/scipy/linalg/basic.py in inv(a, overwrite_a, check_finite)
972 inv_a, info = getri(lu, piv, lwork=lwork, overwrite_lu=1)
973 if info > 0:
--> 974 raise LinAlgError("singular matrix")
975 if info < 0:
976 raise ValueError('illegal value in %d-th argument of internal '
LinAlgError: singular matrix
OS: 10.14.6
Python: 3.6.8.final.0
pd: 0.25.1
np: 1.17.2
sns: 0.9.0

Unable to plot linear regression scatter plot and predicting line

I have tried lot of methods and went through many questions that are already answered over here but in vain. None of it could solve my problem. So please if you can't solve this problem at least don't mark it as 'duplicate' because am desparately looking forward to make my model work and am stuck at this stupid syntactical error.
So, my query is pretty simple, I have a data frame that consists of 2 columns 1st is 'sqft_living' and the second is 'price'. So, I have used linear regression to predict price based on its sqft area. I want to visualize this, as scatter plot followed by a best fit line. However, am getting this error:
TypeError: unhashable type: 'numpy.ndarray'
I have already converted the dataframe into series and corrected dimensions as well still am getting this error.
Please provide me with solution code along with exploitation.
Any help will be highly appreciated as I am stuck with this and can't complete my assignment.
below is the exact code and error i am getting.
import numpy as np
import matplotlib.pyplot as plt # To visualize
import pandas as pd # To read data
from sklearn.linear_model import LinearRegression
X = poly1_data.iloc[:, 0].values.reshape(-1, 1) # values converts it into a numpy array
Y = poly1_data.iloc[:, 1].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column
linear_regressor = LinearRegression() # create object for the class
linear_regressor.fit(X, Y) # perform linear regression
Y_pred = linear_regressor.predict(X) # make predictions
plt.scatter(X, Y)
plt.plot(X, Y_pred, color='red')
plt.show()
--------------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-245-96227c9411b1> in <module>
9 linear_regressor.fit(X, Y) # perform linear regression
10 Y_pred = linear_regressor.predict(X) # make predictions
---> 11 plt.scatter(X, Y)
12 plt.plot(X, Y_pred, color='red')
13 plt.show()
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\pyplot.py in scatter(x, y, s, c, marker, cmap, norm, vmin, vmax, alpha, linewidths, verts, edgecolors, data, **kwargs)
2860 vmin=vmin, vmax=vmax, alpha=alpha, linewidths=linewidths,
2861 verts=verts, edgecolors=edgecolors, **({"data": data} if data
-> 2862 is not None else {}), **kwargs)
2863 sci(__ret)
2864 return __ret
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\__init__.py in inner(ax, data, *args, **kwargs)
1808 "the Matplotlib list!)" % (label_namer, func.__name__),
1809 RuntimeWarning, stacklevel=2)
-> 1810 return func(ax, *args, **kwargs)
1811
1812 inner.__doc__ = _add_data_doc(inner.__doc__,
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\axes\_axes.py in scatter(self, x, y, s, c, marker, cmap, norm, vmin, vmax, alpha, linewidths, verts, edgecolors, **kwargs)
4170 edgecolors = 'face'
4171
-> 4172 self._process_unit_info(xdata=x, ydata=y, kwargs=kwargs)
4173 x = self.convert_xunits(x)
4174 y = self.convert_yunits(y)
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in _process_unit_info(self, xdata, ydata, kwargs)
2133 return kwargs
2134
-> 2135 kwargs = _process_single_axis(xdata, self.xaxis, 'xunits', kwargs)
2136 kwargs = _process_single_axis(ydata, self.yaxis, 'yunits', kwargs)
2137 return kwargs
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in _process_single_axis(data, axis, unit_name, kwargs)
2116 # We only need to update if there is nothing set yet.
2117 if not axis.have_units():
-> 2118 axis.update_units(data)
2119
2120 # Check for units in the kwargs, and if present update axis
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\axis.py in update_units(self, data)
1471 neednew = self.converter != converter
1472 self.converter = converter
-> 1473 default = self.converter.default_units(data, self)
1474 if default is not None and self.units is None:
1475 self.set_units(default)
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\category.py in default_units(data, axis)
101 # default_units->axis_info->convert
102 if axis.units is None:
--> 103 axis.set_units(UnitData(data))
104 else:
105 axis.units.update(data)
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\category.py in __init__(self, data)
167 self._counter = itertools.count()
168 if data is not None:
--> 169 self.update(data)
170
171 def update(self, data):
C:\ProgramData\Anaconda3\lib\site-packages\matplotlib\category.py in update(self, data)
184 data = np.atleast_1d(np.array(data, dtype=object))
185
--> 186 for val in OrderedDict.fromkeys(data):
187 if not isinstance(val, (str, bytes)):
188 raise TypeError("{val!r} is not a string".format(val=val))
TypeError: unhashable type: 'numpy.ndarray'
````[I am getting this image without any scatter plot and best fit line][1]
[1]: https://i.stack.imgur.com/9jccu.png

How can I generate and display a grid of images in PyTorch with plt.imshow and torchvision.utils.make_grid?

I am trying to understand how torchvision interacts with mathplotlib to produce a grid of images. It's easy to generate images and display them iteratively:
import torch
import torchvision
import matplotlib.pyplot as plt
w = torch.randn(10,3,640,640)
for i in range (0,10):
z = w[i]
plt.imshow(z.permute(1,2,0))
plt.show()
However, displaying these images in a grid does not seem to be as straightforward.
w = torch.randn(10,3,640,640)
grid = torchvision.utils.make_grid(w, nrow=5)
plt.imshow(grid)
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-61-1601915e10f3> in <module>()
1 w = torch.randn(10,3,640,640)
2 grid = torchvision.utils.make_grid(w, nrow=5)
----> 3 plt.imshow(grid)
/anaconda3/lib/python3.6/site-packages/matplotlib/pyplot.py in imshow(X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, shape, filternorm, filterrad, imlim, resample, url, hold, data, **kwargs)
3203 filternorm=filternorm, filterrad=filterrad,
3204 imlim=imlim, resample=resample, url=url, data=data,
-> 3205 **kwargs)
3206 finally:
3207 ax._hold = washold
/anaconda3/lib/python3.6/site-packages/matplotlib/__init__.py in inner(ax, *args, **kwargs)
1853 "the Matplotlib list!)" % (label_namer, func.__name__),
1854 RuntimeWarning, stacklevel=2)
-> 1855 return func(ax, *args, **kwargs)
1856
1857 inner.__doc__ = _add_data_doc(inner.__doc__,
/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_axes.py in imshow(self, X, cmap, norm, aspect, interpolation, alpha, vmin, vmax, origin, extent, shape, filternorm, filterrad, imlim, resample, url, **kwargs)
5485 resample=resample, **kwargs)
5486
-> 5487 im.set_data(X)
5488 im.set_alpha(alpha)
5489 if im.get_clip_path() is None:
/anaconda3/lib/python3.6/site-packages/matplotlib/image.py in set_data(self, A)
651 if not (self._A.ndim == 2
652 or self._A.ndim == 3 and self._A.shape[-1] in [3, 4]):
--> 653 raise TypeError("Invalid dimensions for image data")
654
655 if self._A.ndim == 3:
TypeError: Invalid dimensions for image data
Even though PyTorch's documentation indicates that w is the correct shape, Python says that it isn't. So I tried to permute the indices of my tensor:
w = torch.randn(10,3,640,640)
grid = torchvision.utils.make_grid(w.permute(0,2,3,1), nrow=5)
plt.imshow(grid)
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-62-6f2dc6313e29> in <module>()
1 w = torch.randn(10,3,640,640)
----> 2 grid = torchvision.utils.make_grid(w.permute(0,2,3,1), nrow=5)
3 plt.imshow(grid)
/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/utils.py in make_grid(tensor, nrow, padding, normalize, range, scale_each, pad_value)
83 grid.narrow(1, y * height + padding, height - padding)\
84 .narrow(2, x * width + padding, width - padding)\
---> 85 .copy_(tensor[k])
86 k = k + 1
87 return grid
RuntimeError: The expanded size of the tensor (3) must match the existing size (640) at non-singleton dimension 0
What's happening here? How can I place a bunch of randomly generated images into a grid and display them?

There's a small mistake in your code. torchvision.utils.make_grid() returns a tensor which contains the grid of images. But the channel dimension has to be moved to the end since that's what matplotlib recognizes. Below is the code that works fine:
In [107]: import torchvision
# sample input (10 RGB images containing just Gaussian Noise)
In [108]: batch_tensor = torch.randn(*(10, 3, 256, 256)) # (N, C, H, W)
# make grid (2 rows and 5 columns) to display our 10 images
In [109]: grid_img = torchvision.utils.make_grid(batch_tensor, nrow=5)
# check shape
In [110]: grid_img.shape
Out[110]: torch.Size([3, 518, 1292])
# reshape and plot (because matplotlib needs channel as the last dimension)
In [111]: plt.imshow(grid_img.permute(1, 2, 0))
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Out[111]: <matplotlib.image.AxesImage at 0x7f62081ef080>
which shows the output as:

You have to convert to numpy first
import numpy as np
def show(img):
npimg = img.numpy()
plt.imshow(np.transpose(npimg, (1,2,0)), interpolation='nearest')
w = torch.randn(10,3,640,640)
grid = torchvision.utils.make_grid(w, nrow=10, padding=100)
show(grid)

plt.hist errors on subsetted data

I'm very new to Python so please excuse any stupidity on my part.
I'm running a histogram with matplotlib and getting errors when I use subset data, the code works perfectly if I use the full dataset, hence my confusion.
Perhaps I'm not subsetting correctly?
My code is below and related errors are below, thanks.
For awareness, this was written in Python 3.
Import required packages:
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
Read the data:
mlb=pd.read_csv('C:\Users\ocmh\Desktop\Python\Batting.csv')
View a sample of the data:
mlb.head()
Subset the data to return just Boston data:
mlb_bos=mlb[(mlb['teamID'] == 'BOS')]
View a sample of the subset data:
mlb_bos.head()
Plot a histogram of the original data: And works perfectly
plt.hist(mlb.AB.dropna, color= sns.desaturate("indianred",1))
Plot a histogram of the subset data: Returns errors (Errors are below)
plt.hist(mlb_bos.AB.dropna, color= sns.desaturate("indianred",1))
If you don't have the seaborn package installed, you can just drop color= sns.desaturate("indianred",1) as this was purely for aesthetics.
Errors below:
KeyError Traceback (most recent call last)
<ipython-input-11-1484047d7ac6> in <module>()
----> 1 plt.hist(mlb_bos.AB, color=color)
/Users/mattoconnell/anaconda/lib/python3.4/site-packages/matplotlib/pyplot.py in hist(x, bins, range, normed, weights, cumulative, bottom, histtype, align, orientation, rwidth, log, color, label, stacked, hold, **kwargs)
2894 histtype=histtype, align=align, orientation=orientation,
2895 rwidth=rwidth, log=log, color=color, label=label,
-> 2896 stacked=stacked, **kwargs)
2897 draw_if_interactive()
2898 finally:
/Users/mattoconnell/anaconda/lib/python3.4/site-packages/matplotlib/axes/_axes.py in hist(self, x, bins, range, normed, weights, cumulative, bottom, histtype, align, orientation, rwidth, log, color, label, stacked, **kwargs)
5602 # Massage 'x' for processing.
5603 # NOTE: Be sure any changes here is also done below to 'weights'
-> 5604 if isinstance(x, np.ndarray) or not iterable(x[0]):
5605 # TODO: support masked arrays;
5606 x = np.asarray(x)
/Users/mattoconnell/anaconda/lib/python3.4/site-packages/pandas/core/series.py in __getitem__(self, key)
512 def __getitem__(self, key):
513 try:
--> 514 result = self.index.get_value(self, key)
515
516 if not np.isscalar(result):
/Users/mattoconnell/anaconda/lib/python3.4/site-packages/pandas/core/index.py in get_value(self, series, key)
1458
1459 try:
-> 1460 return self._engine.get_value(s, k)
1461 except KeyError as e1:
1462 if len(self) > 0 and self.inferred_type in ['integer','boolean']:
pandas/index.pyx in pandas.index.IndexEngine.get_value (pandas/index.c:3113)()
pandas/index.pyx in pandas.index.IndexEngine.get_value (pandas/index.c:2844)()
pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:3704)()
pandas/hashtable.pyx in pandas.hashtable.Int64HashTable.get_item (pandas/hashtable.c:7255)()
pandas/hashtable.pyx in pandas.hashtable.Int64HashTable.get_item (pandas/hashtable.c:7193)()
KeyError: 0