We Keep Coding

Creating a neural network with my own dataset - ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type numpy.ndarray)

Right now my code is throwing that error and I'm not sure why I have tried converting the arrays into 3d arrays the error is being thrown in the model.fit section I believe my error may be with creating my dataset I dont believe its a type error i think it be most likely that there is something wrong with the dimension of my array but im really not sure if anyone could help that would be great heres my code -
from PIL import Image
import numpy as np
from numpy import asarray
import os
import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
black_people = os.path.join(os.curdir, 'black people')
numpydata_black = []
for root, _, files in os.walk(black_people):
# make sure to only get the imgs (ignore .DS_Store)
for file in filter(lambda f: str(f).endswith(".png"), files):
img = Image.open(os.path.join(root, file))
numpydata_black.append(np.array(img))
numpydata_black = asarray(numpydata_black, dtype=object)
white_people = os.path.join(os.curdir, 'black people')
numpydata_white = []
for root, _, files in os.walk(white_people):
# make sure to only get the imgs (ignore .DS_Store)
for file in filter(lambda f: str(f).endswith(".png"), files):
img = Image.open(os.path.join(root, file))
numpydata_white.append(np.array(img))
numpydata_white = asarray(numpydata_white, dtype=object)
twod_array = []
for image in numpydata_white:
white_images = [image/255.0, 'white']
twod_array.append(white_images)
for image in numpydata_black:
black_image = [image/255.0, 'black']
twod_array.append(black_image)
class_label = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
class_names = ['White','Black']
twod_array = np.array([np.array(val) for val in twod_array])
model = keras.Sequential([
keras.layers.Flatten(input_shape=(32, 32)),
keras.layers.Dense(128, activation='relu'),
keras.layers.Dense(10, activation='softmax')
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
twod_array = np.asarray(twod_array)
class_label = np.asarray(class_label)
print(type(twod_array))
model.fit(twod_array, class_label, epochs=5)

Testing a Random Image against a Python Keras/Tensorflow CNN

I've created and CNN and I am trying to figure out how to test a random image against it. I am utilizing Keras and Tensorflow. Lets assume I wanted to test the image found here: https://i.ytimg.com/vi/7I8OeQs7cQA/maxresdefault.jpg.
How would I save the model, load it then test this image against it? Here is some example code I found online that demonstrates what I mean:
https://meta.stackexchange.com/questions/144665/hide-email-address-from-my-profile
Any help is much appreciated, thanks!
import os
import cv2
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import display, Image
from keras.models import Sequential, load_model
from keras.layers import Conv2D, Flatten, MaxPooling2D, Input
from keras.preprocessing.image import ImageDataGenerator
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import models, layers
X = []
y = []
from sklearn.model_selection import train_test_split
labels = os.listdir(r'C:/Users/zF1bo/Desktop/natural_images')
labels
for label in labels:
path = r'C:/Users/zF1bo/Desktop/natural_images/{}/'.format(label)
img_data = os.listdir(path)
for image in img_data:
a = cv2.imread( path + image)
a = cv2.resize(a, (64, 64))
X.append(np.array(a.astype('float32')) / 255)
y.append(label)
buckets = []
for i in y:
if i == 'airplane':
buckets.append(0)
elif i == 'car':
buckets.append(1)
elif i == 'cat':
buckets.append(2)
elif i == 'dog':
buckets.append(3)
elif i == 'flower':
buckets.append(4)
elif i == 'fruit':
buckets.append(5)
elif i == 'motorbike':
buckets.append(6)
elif i == 'person':
buckets.append(7)
y = buckets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, \
random_state = 0)
model = models.Sequential()
model.add(layers.Conv2D(filters=32, kernel_size=(5,5), activation='relu', input_shape=(64,64,3)))
model.add(layers.MaxPool2D(pool_size=(2, 2)))
model.add(layers.Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(layers.MaxPool2D(pool_size=(2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(8, activation='softmax'))
model.compile(optimizer='adam', loss = 'sparse_categorical_crossentropy',metrics=['accuracy'])
y_train = np.array(y_train)
model.fit(X_train, y_train, batch_size=(256), epochs=25)
pred = model.predict(X_test)
diff = []
for i in pred:
diff.append(np.argmax(i))
from sklearn.metrics import accuracy_score
accuracy_score(diff,y_test)

Step 1: Save the model
model.save('model.h5')
Step 2: Load the model
loaded_model = tensorflow.keras.models.load_model('model.h5')
Step 3: Download the image via requests library(answer is taken from: Downloading a picture via urllib and python):
import urllib.request
urllib.request.urlretrieve(url, filename)
Otherwise you can apply the same steps like in the first picture you posted. Do not forget to expand_dims()

you can save your model using mode.save('path to save location')
To input an image you should read it, perform any pre-processing that you did to the training images then use model.predict as shown in the code below.
import tensorflow as tf
from tensorflow import keras
model = keras.models.load_model('path/to/location') # loads the saved model
pred_img =r'path to the img'
img=cv2.imread (pred_img)
img=img/255 # rescale the image
print(img.shape)
img=cv2.resize(img, (64,64)) # resize to same size used in training
print (img.shape)
img=np.expand_dims(img, axis=0)
print (img.shape)
pred=model.predict(img)
print (pred) # will be a list of 8 elements select the element in the list with the highest probability
index=np.argmax(pred)) # this will be the index of the class predicted
class_name=buckets[index] # this will be the name of the class predicted
print (class_name)
NOTE if your read in the training images with cv2 remember the order is bgr not rgb. The model was trained on bgr images. When you read in the image to predict it must be bgr also.

ValueError: Error when checking input: expected dense_11_input to have 3 dimensions, but got array with shape (0, 1)

I am new in machine learning and python and I try to make classification whether a patient cancer or not. I found a piece of code from https://github.com/fahomid/ML-Tensorflow-Medical-Image/blob/master/tensorflow-model.py I have a small dataset. Training and test set have two patient directories that have dicom files just for trying. The code is as following;
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
import os
import pydicom
import numpy as np
import PIL
# Generate data from dicom file
dataset = [];
labels = [];
for root, dirs, files in os.walk("training_data/Cancer"):
for file in files:
if file.endswith(".dcm"):
ds = pydicom.dcmread(os.path.join(root, file))
dataset.append(ds.pixel_array)
labels.append(1);
for root, dirs, files in os.walk("training_data/Normal"):
for file in files:
if file.endswith(".dcm"):
ds = pydicom.dcmread(os.path.join(root, file))
dataset.append(ds.pixel_array)
labels.append(0)
dataset_size = len(dataset)
dataset = np.asarray(dataset)
labels = np.asarray(labels)
# create model
model = Sequential()
model.add(Dense(32, activation='tanh', input_shape=(512, 512)))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=
['accuracy'])
model.fit(dataset, labels, epochs=10, shuffle=True, batch_size=32)
# serialize model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("\n\nModel saved to disk\n\n")
model.summary()
The line of error message is as following;
model.fit(dataset, labels, epochs=10, shuffle=True, batch_size=32)
ValueError: Error when checking input: expected dense_11_input to have 3
dimensions, but got array with shape (0, 1)
Thanks for your help.

The following section of your code looks for files (with.dcm extension) containing the data for Cancer and Normal person. It does NOT find any, So it returns nothing.
# Generate data from dicom file
dataset = [];
labels = [];
for root, dirs, files in os.walk("training_data/Cancer"):
for file in files:
if file.endswith(".dcm"):
ds = pydicom.dcmread(os.path.join(root, file))
dataset.append(ds.pixel_array)
labels.append(1);
for root, dirs, files in os.walk("training_data/Normal"):
for file in files:
if file.endswith(".dcm"):
ds = pydicom.dcmread(os.path.join(root, file))
dataset.append(ds.pixel_array)
labels.append(0)
So the value of dataset variable is 0, and labels variable is 1. And when the model.fit method is called, it expects the input to be 3 dimensions with the shape of (512, 512), but it only gets the input with the shape of (0, 1).

ValueError: Error when checking input: expected flatten_input to have 3 dimensions, but got array with shape (22, 12)

I am new to Tensorflow, I am trying to do text recognize for image(size 12x22) using tensorflow.
Training model seems to be ok, but when do prediction, there is error:
File "C:\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training_utils.py", line 316, in standardize_input_data
'with shape ' + str(data_shape))
ValueError: Error when checking input: expected flatten_input to have 3 dimensions, but got array with shape (22, 12)
I don't know why it needs 3 dimension while I just input a image in 2D.
train_model.py, this is the trainer, export a training model
import cv2
import pickle
import os.path
import numpy as np
from imutils import paths
import tensorflow as tf
# from sklearn.preprocessing import LabelBinarizer
# from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
# from keras.layers.convolutional import Conv2D, MaxPooling2D
from tensorflow.keras.layers import Flatten, Dense
import image_fit
LETTER_IMAGES_FOLDER = "training_generate/divided_sample"
MODEL_FILENAME = "captcha_model.hdf5"
MODEL_LABELS_FILENAME = "model_labels.dat"
# initialize the data and labels
data = []
labels = []
# loop over the input images
print('Read images')
for image_file in paths.list_images(LETTER_IMAGES_FOLDER):
# Load the image and convert it to grayscale
image = cv2.imread(image_file)
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# resize the image as 12, 22
image = image_fit.resize_to_fit(image, 12, 22)
# Grab the name of the letter based on the folder it was in
label = image_file.split(os.sep)[-2]
label = chr(int(label))
# Add the letter image and it's label to our training data
data.append(image)
labels.append(label)
print('Done reading')
data = tf.keras.utils.normalize(np.array(data), axis=1)
# print(data[0])
labels = np.array(labels)
model = Sequential()
model.add(Flatten(input_shape=(22, 12)))
model.add(Dense(128, activation=tf.nn.relu))
model.add(Dense(128, activation=tf.nn.relu))
model.add(Dense(62, activation=tf.nn.softmax))
model.compile(optimizer='Adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.fit(data, labels, epochs=3)
print('Finish training')
# val_loss, val_acc = model.evaluate()
model.save(MODEL_FILENAME)
solve.py, this is the python script that solve the letter.
import cv2
import pickle
import os.path
import numpy as np
from imutils import paths
import tensorflow as tf
import image_fit
LETTER_IMAGES_FOLDER = "training_generate/divided_sample"
MODEL_FILENAME = "captcha_model.hdf5"
MODEL_LABELS_FILENAME = "model_labels.dat"
image_file = 'test_0.bmp'
image = cv2.imread(image_file)
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image = image_fit.resize_to_fit(image, 12, 22)
new_model = tf.keras.models.load_model(MODEL_FILENAME)
predicitions = new_model.predict(image)
print(np.argmax(predicitions[0]))

When you make the prediction with the line
predicitions = new_model.predict(image)
it is expecting the first dimension to be the number of images in the batch. Since you only have one image, the first dimension would be of size one. You could resize the image to be (1,22,12), and this would pass:
predicitions = new_model.predict(image.reshape(1,22,12))

According to Kruger's answer, the code should be look like this:
image_file = 'test_0.bmp'
image = cv2.imread(image_file)
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image = image_fit.resize_to_fit(image, 12, 22)
data = [image]
new_model = tf.keras.models.load_model(MODEL_FILENAME)
predicitions = new_model.predict(np.array(data))
print(np.argmax(predicitions[0]))

Error: Error when checking model input: expected dense_input_6 to have shape (None, 784) but got array with shape (784L, 1L)

I get an error when trying to apply the below code onto the MNIST sample dataset for both training and testing. Please helpe
The following is my code:
import pandas
import numpy
import numpy
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.utils import np_utils
# fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)
# Read in the TRAINING dataset
f = open("C:/Users/USER/Desktop/mnist/mnist_train_100.csv", 'r')
a = f.readlines() # place everythig in a lsit called 'a'
#print(a)
f.close()
# go through the list a and split by comma
output_nodes = 10
for record in a: #go through the big list "a"
all_values = record.split(',')
X_train = (numpy.asfarray(all_values[1:]) / 255.0 * 0.99) + 0.01
y_train = numpy.zeros(output_nodes) + 0.01
y_train[int(all_values[0])] = 0.99
# Read in the TEST data set and then split
f = open("C:/Users/USER/Desktop/mnist/mnist_test_10.csv", 'r')
a = f.readlines() # place everythig in a lsit called 'a'
#print(a)
f.close()
# go through the list a and split by comma
for record in a: #go through the big list "a"
all_values = record.split(',')
X_test = (numpy.asfarray(all_values[1:]) / 255.0 * 0.99) + 0.01
y_test = numpy.zeros(output_nodes) + 0.01
y_test[int(all_values[0])] = 0.99
num_pixels = len(X_train)
# define baseline model
def baseline_model():
# create model
model = Sequential()
model.add(Dense(num_pixels, input_dim=num_pixels, init='normal', activation='relu'))
model.add(Dense(output_nodes, init='normal', activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
## build the model
#model = baseline_model()
## Fit the model
#model.fit(X_train, y_train, validation_data=(X_test, y_test), nb_epoch=10, batch_size=200,verbose=2)
I get the following error:
Exception: Error when checking model input: expected dense_input_6 to have shape (None, 784) but got array with shape (784L, 1L)

I assume you are working with this tutorial.
I would recommend using pandas to read your format:
import pandas as pd
import numpy as np
data = pd.read_csv('mnist_train_100.csv', header=None)
# numpy array of shape (100, 784), type float32
X_train = data.ix[:, 1:].values.astype(np.float32)
# numpy array of shape (100,), type int64
y_train = data.ix[:, 0].values