Related
I am trying to train a LSTM model to reconstruct time series data. I have a data set of ~1800 univariant time-series.
Basically I'm trying to solve a problem similar to this one Anomaly detection in ECG plots, but my time series have different lengths.
I used this approach to deal with variant length:
How to apply LSTM-autoencoder to variant-length time-series data?
and this approach to split the input data based on shape:
Keras misinterprets training data shape
When looping over the data and fitting a model for every shape. is the model eventually only based on the last shape it trained on or is it using all the data to train the final model?
How would I train the model on all input data regardless shape of data?
I know I can add padding but I am trying to use the data as is at this point.
Any suggestions or other approaches to deal with different length on timeseries?
(It is not an issue of time sampling it is more of one timeseries started recording on day X and some only on day X+100)
Here is the code I am using for my autoencoder:
import keras.backend as K
from keras.layers import (Input, Dense, TimeDistributed, LSTM, GRU, Dropout, merge,
Flatten, RepeatVector, Bidirectional, SimpleRNN, Lambda)
def encoder(model_input, layer, size, num_layers, drop_frac=0.0, output_size=None,
bidirectional=False):
"""Encoder module of autoencoder architecture"""
if output_size is None:
output_size = size
encode = model_input
for i in range(num_layers):
wrapper = Bidirectional if bidirectional else lambda x: x
encode = wrapper(layer(size, name='encode_{}'.format(i),
return_sequences=(i < num_layers - 1)))(encode)
if drop_frac > 0.0:
encode = Dropout(drop_frac, name='drop_encode_{}'.format(i))(encode)
encode = Dense(output_size, activation='linear', name='encoding')(encode)
return encode
def repeat(x):
stepMatrix = K.ones_like(x[0][:,:,:1]) #matrix with ones, shaped as (batch, steps, 1)
latentMatrix = K.expand_dims(x[1],axis=1) #latent vars, shaped as (batch, 1, latent_dim)
return K.batch_dot(stepMatrix,latentMatrix)
def decoder(encode, layer, size, num_layers, drop_frac=0.0, aux_input=None,
bidirectional=False):
"""Decoder module of autoencoder architecture"""
decode = Lambda(repeat)([inputs,encode])
if aux_input is not None:
decode = merge([aux_input, decode], mode='concat')
for i in range(num_layers):
if drop_frac > 0.0 and i > 0: # skip these for first layer for symmetry
decode = Dropout(drop_frac, name='drop_decode_{}'.format(i))(decode)
wrapper = Bidirectional if bidirectional else lambda x: x
decode = wrapper(layer(size, name='decode_{}'.format(i),
return_sequences=True))(decode)
decode = TimeDistributed(Dense(1, activation='linear'), name='time_dist')(decode)
return decode
inputs = Input(shape=(None, 1))
encoded = encoder(inputs,LSTM,128, 2, drop_frac=0.0, output_size=None, bidirectional=False)
decoded = decoder(encoded, LSTM, 128, 2, drop_frac=0.0, aux_input=None,
bidirectional=False,)
sequence_autoencoder = Model(inputs, decoded)
sequence_autoencoder.compile(optimizer='adam', loss='mae')
trainByShape = {}
for item in train_data:
if item.shape in trainByShape:
trainByShape[item.shape].append(item)
else:
trainByShape[item.shape] = [item]
for shape in trainByShape:
modelHistory =sequence_autoencoder.fit(
np.asarray(trainByShape[shape]),
np.asarray(trainByShape[shape]),
epochs=100, batch_size=1, validation_split=0.15)
use a bidirectional lstm and increase the number of parameters to gain accuracy. I increased the latent_dim to 1000 and it fit the data closely. More hardware and more memory.
def create_dataset(dataset, look_back=3):
dataX, dataY = [], []
for i in range(len(dataset)-look_back-1):
a = dataset[i:(i+look_back)]
dataX.append(a)
dataY.append(dataset[i + look_back])
return np.array(dataX), np.array(dataY)
COLUMNS=['Open']
dataset=eqix_df[COLUMNS]
scaler = MinMaxScaler(feature_range=(0, 1))
dataset = scaler.fit_transform(np.array(dataset).reshape(-1,1))
train_size = int(len(dataset) * 0.70)
test_size = len(dataset) - train_size
train, test = dataset[0:train_size], dataset[train_size:len(dataset)]
look_back=10
trainX=[]
testX=[]
y_train=[]
trainX, y_train = create_dataset(train, look_back)
testX, y_test = create_dataset(test, look_back)
X_train = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
X_test = np.reshape(testX, (testX.shape[0], 1, testX.shape[1]))
latent_dim=700
n_future=1
model = Sequential()
model.add(Bidirectional(LSTM(units=latent_dim, return_sequences=True,
input_shape=(X_train.shape[1], 1))))
#LSTM 1
model.add(Bidirectional(LSTM(latent_dim,return_sequences=True,dropout=0.4,recurrent_dropout=0.4,name='lstm1')))
#LSTM 2
model.add(Bidirectional(LSTM(latent_dim,return_sequences=True,dropout=0.2,recurrent_dropout=0.4,name='lstm2')))
#LSTM 3
model.add(Bidirectional(LSTM(latent_dim, return_sequences=False,dropout=0.2,recurrent_dropout=0.4,name='lstm3')))
model.add(Dense(units = n_future))
model.compile(optimizer="adam", loss="mean_squared_error", metrics=["acc"])
history=model.fit(X_train, y_train,epochs=50,verbose=0)
plt.plot(history.history['loss'])
plt.title('loss accuracy')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
#print(X_test)
prediction = model.predict(X_test)
# shift train predictions for plotting
trainPredictPlot = np.empty_like(dataset)
trainPredictPlot[:, :] = np.nan
trainPredictPlot[look_back:len(prediction)+look_back, :] = prediction
# shift test predictions for plotting
#plt.plot(scaler.inverse_transform(dataset))
plt.plot(trainPredictPlot, color='red')
#plt.plot(testPredictPlot)
#plt.legend(['Actual','Train','Test'])
x=np.linspace(look_back,len(prediction)+look_back,len(y_test))
plt.plot(x,y_test)
plt.show()
Keras LSTM implementation expect a input of type: (Batch, Timesteps, Features).
One solution would be to set Timesteps = 1 and pass the sequence lengths as the Batch dimensions.
If the sampling procedure is the same (no need for resampling), and the difference in length only comes from when the recording time start (X+100 instead of X), I would try to get rid off the lag in the pre-processing stages to get the section of interest only.
Part 1: Plotting the irregular heartbeat. Part 2 is a DENSE network to classify incoming heartbeat voltage to predict irregular beat patterns. 94% accuracy!
from scipy.io import arff
import pandas as pd
from scipy.misc import electrocardiogram
import matplotlib.pyplot as plt
import numpy as np
data = arff.loadarff('ECG5000_TRAIN.arff')
df = pd.DataFrame(data[0])
#for column in df.columns:
# print(column)
columns=[x for x in df.columns if x!="target"]
print(columns)
#print(df[df.target == "b'1'"].drop(labels='target', axis=1).mean(axis=0).to_numpy())
normal=df.query("target==b'1'").drop(labels='target', axis=1).mean(axis=0).to_numpy()
rOnT=df.query("target==b'2'").drop(labels='target', axis=1).mean(axis=0).to_numpy()
pcv=df.query("target==b'3'").drop(labels='target', axis=1).mean(axis=0).to_numpy()
sp=df.query("target==b'4'").drop(labels='target', axis=1).mean(axis=0).to_numpy()
ub=df.query("target==b'5'").drop(labels='target', axis=1).mean(axis=0).to_numpy()
plt.plot(normal,label="Normal")
plt.plot(rOnT,label="R on T",alpha=.3)
plt.plot(pcv, label="PCV",alpha=.3)
plt.plot(sp, label="SP",alpha=.3)
plt.plot(ub, label="UB",alpha=.3)
plt.legend()
plt.title("ECG")
plt.show()
Frame by frame comparision for normal. There are bands of operation which a normal heart stays with:
def PlotTheFrames(df,title,color):
fig,ax = plt.subplots(figsize=(140,50))
for key,item in df.iterrows():
array=[]
for value in np.array(item).flatten():
array.append(value);
x=np.linspace(0,100,len(array))
ax.plot(x,array,c=color)
plt.title(title)
plt.show()
normal=df.query("target==b'1'").drop(labels='target', axis=1)
PlotTheFrames(normal,"Normal Heart beat",'r')
R on T the valves don't seem to be operating correctly
rOnT=df.query("target==b'2'").drop(labels='target', axis=1)
PlotTheFrames(rOnT,"R on T Heart beat","b")
Use a deep learning dense network instead of LSTM! I used leakyReLU for the smaller gradient descent
X=df[columns]
y=pd.get_dummies(df['target'])
model=Sequential()
model.add(Dense(440, input_shape=(len(columns),),activation='LeakyReLU'))
model.add(Dropout(0.4))
model.add(Dense(280, activation='LeakyReLU'))
model.add(Dropout(0.2))
model.add(Dense(240, activation='LeakyReLU'))
model.add(Dense(32, activation='LeakyReLU'))
model.add(Dense(16, activation='LeakyReLU'))
model.add(Dense(5))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.3, random_state=42)
scaler = StandardScaler()
scaler.fit(X_train)
X_train=scaler.transform(X_train)
X_test=scaler.transform(X_test)
history=model.fit(X_train, y_train,epochs = 1000,verbose=0)
model.evaluate(X_test, y_test)
plt.plot(history.history['loss'])
plt.title('loss accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
I'm a student at the University of Bologna ( Italy) and I'm using the Google Coral USB accelerator for my thesis.
I realized a keras neural network that classifies my data in four classes and the accuracy I get is roughly 97%.
I performed a full integer post- training quantization since keras networks don't support quantization-aware training. I followed the guide on TensorFlow site but I had problems running inference on the edge tpu .
In particular my network undergoes an accuracy loss when the model is converted to a tensorflowlite one ( the accuracy drops to roughly the 25%) . This is due to quantization since the tensorflowlite model without quantization that runs on my pc is not affected by the conversion.
I tried to scale my input data in a range [0, 255] with MinMaxScaler but in this case , even if the accuracy of the tensorflowlite quantized model matches the one of the not converted one , the results are not satisfactory since the accuracy of the network itself is low.
I wonder if you could help me solve this problem. Perhaps the values on my dataset are too low and the quantization fails to convert float32 into uint8 without information loss.
Below you'll find my python code.
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from numpy import loadtxt
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from sklearn.model_selection import train_test_split
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.optimizers import RMSprop
from sklearn.preprocessing import StandardScaler,MinMaxScaler,Normalizer
from sklearn.metrics import confusion_matrix, accuracy_score
from tensorflow.keras.callbacks import EarlyStopping
import os
import time
import tflite_runtime.interpreter as tflite
import collections
import operator
"""Functions to work with classification models."""
Class = collections.namedtuple('Class', ['id', 'score'])
def input_tensor(interpreter):
"""Returns input tensor view as numpy array of shape (height, width, 3)."""
tensor_index = interpreter.get_input_details()[0]['index']
return interpreter.tensor(tensor_index)()[0]
def output_tensor(interpreter):
"""Returns dequantized output tensor."""
output_details = interpreter.get_output_details()[0]
output_data = np.squeeze(interpreter.tensor(output_details['index'])()) #Remove single-dimensional entries from the shape of an array.
scale, zero_point = output_details['quantization']
return scale * (output_data - zero_point)
def set_input(interpreter, data):
"""Copies data to input tensor."""
input_tensor(interpreter)[:] = data
return data
def get_output(interpreter, top_k=1, score_threshold=0.0):
"""Returns no more than top_k classes with score >= score_threshold."""
scores = output_tensor(interpreter)
classes = [
Class(i, scores[i])
for i in np.argpartition(scores, -top_k)[-top_k:]
if scores[i] >= score_threshold
]
return sorted(classes, key=operator.itemgetter(1), reverse=True)
#load the dataset
Modelli_Prova01_Nom01_Acc1L = loadtxt(r'/home/utente/Scrivania/csvtesi/Modelli_Prova01_Nom01_Acc1L.csv',delimiter=',')
Modelli_Prova02_Nom01_Acc1L = loadtxt(r'/home/utente/Scrivania/csvtesi/Modelli_Prova02_Nom01_Acc1L.csv',delimiter=',')
Modelli_Prova03_Nom01_Acc1L = loadtxt(r'/home/utente/Scrivania/csvtesi/Modelli_Prova03_Nom01_Acc1L.csv',delimiter=',')
Modelli_Prova04_Nom01_Acc1L = loadtxt(r'/home/utente/Scrivania/csvtesi/Modelli_Prova04_Nom01_Acc1L.csv',delimiter=',')
Modelli_Prova05_Nom01_Acc1L = loadtxt(r'/home/utente/Scrivania/csvtesi/Modelli_Prova05_Nom01_Acc1L.csv',delimiter=',')
time_start = time.perf_counter()
#split x and y data (train and test)
Acc1L01_train,Acc1L01_test = train_test_split(Modelli_Prova01_Nom01_Acc1L ,test_size=0.015,random_state=42)
Acc1L02_train,Acc1L02_test = train_test_split(Modelli_Prova02_Nom01_Acc1L,test_size=0.3,random_state=42)
Acc1L03_train,Acc1L03_test = train_test_split(Modelli_Prova03_Nom01_Acc1L,test_size=0.3,random_state=42)
Acc1L04_train,Acc1L04_test = train_test_split(Modelli_Prova04_Nom01_Acc1L,test_size=0.3,random_state=42)
Acc1L05_train,Acc1L05_test = train_test_split(Modelli_Prova05_Nom01_Acc1L,test_size=0.15,random_state=42)
Y1_train= np.zeros([len(Acc1L01_train)+len(Acc1L05_train),1])
Y2_train= np.ones([len(Acc1L02_train),1])
Y3_train= np.ones([len(Acc1L03_train),1]) +1
Y4_train= np.ones([len(Acc1L04_train),1]) +2
Y1_test= np.zeros([len(Acc1L01_test)+len(Acc1L05_test),1])
Y2_test= np.ones([len(Acc1L02_test),1])
Y3_test= np.ones([len(Acc1L03_test),1]) +1
Y4_test= np.ones([len(Acc1L04_test),1]) +2
xAcc1L_train = np.concatenate((Acc1L01_train,Acc1L05_train,Acc1L02_train,Acc1L03_train,Acc1L04_train),axis=0)
xAcc1L_train=MinMaxScaler([0,255]).fit_transform(xAcc1L_train)
#xAcc1L_train=StandardScaler().fit_transform(xAcc1L_train)
#xAcc1L_train=Normalizer().fit_transform(xAcc1L_train)
#xAcc1L_train=np.transpose(xAcc1L_train)
yAcc1L_train = np.concatenate((Y1_train,Y2_train,Y3_train,Y4_train),axis=0)
xAcc1L_test = np.concatenate((Acc1L01_test,Acc1L05_test,Acc1L02_test,Acc1L03_test,Acc1L04_test),axis=0)
xAcc1L_test=Normalizer().fit_transform(xAcc1L_test)
#xAcc1L_test=MinMaxScaler([0,255]).fit_transform(xAcc1L_test)
#xAcc1L_test=StandardScaler().fit_transform(xAcc1L_test)
#xAcc1L_test=np.transpose(xAcc1L_test)
yAcc1L_test = np.concatenate((Y1_test,Y2_test,Y3_test,Y4_test),axis=0)
#1 hot encode y
one_hot_labelsAcc1L =to_categorical(yAcc1L_train, num_classes=4)
one_hot_labelsAcc1L_test = to_categorical(yAcc1L_test, num_classes=4)
#fit the model
model = Sequential()
model.add(Dense(300, activation='relu', input_dim=30))
model.add(Dense(4, activation='softmax'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
es1 = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=100)
es2 = EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=100)
history=model.fit(xAcc1L_train, one_hot_labelsAcc1L,validation_data=(xAcc1L_test,one_hot_labelsAcc1L_test),epochs=500, batch_size=30, verbose=1, callbacks=[es1,es2])
#history=model.fit(tf.cast(xAcc1L_train, tf.float32), one_hot_labelsAcc1L,validation_data=(tf.cast(xAcc1L_test, tf.float32),one_hot_labelsAcc1L_test),epochs=500, batch_size=30, verbose=1, callbacks=[es1,es2])
time_elapsed = (time.perf_counter() - time_start)
print ("%5.1f secs " % (time_elapsed))
start=time.monotonic()
_, accuracy = model.evaluate(xAcc1L_test, one_hot_labelsAcc1L_test, batch_size=30, verbose=1)
#_, accuracy = model.evaluate(tf.cast(xAcc1L_test, tf.float32), one_hot_labelsAcc1L_test, batch_size=30, verbose=1)
print(accuracy)
inference_time = time.monotonic() - start
print('%.1fms ' % (inference_time * 1000))
# summarize history for accuracy
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper right')
plt.show()
#predicted labels
predictions = model.predict(xAcc1L_test)
y_pred = (predictions > 0.5)
matrix = confusion_matrix(one_hot_labelsAcc1L_test.argmax(axis=1), y_pred.argmax(axis=1))
print('confusion matrix = \n',matrix)
print("Accuracy:",accuracy_score(one_hot_labelsAcc1L_test.argmax(axis=1), y_pred.argmax(axis=1)))
mod01=model.save('/home/utente/Scrivania/csvtesi/rete_Nom01.h5')
#convert the model
#representative dataset
train_ds = tf.data.Dataset.from_tensor_slices(
(tf.cast(xAcc1L_train, tf.float32))).batch(1)
print(train_ds)
def representative_dataset_gen():
for input_value in train_ds:
yield [input_value]
print(model.layers[0].input_shape)
#integer post-training quantization
converter = tf.compat.v1.lite.TFLiteConverter.from_keras_model_file('/home/utente/Scrivania/csvtesi/rete_Nom01.h5') #all operations mapped on edge tpu
#converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.representative_dataset = representative_dataset_gen
print(converter.representative_dataset)
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.inference_input_type = tf.uint8
converter.inference_output_type = tf.uint8
tflite_quant_model = converter.convert()
open('/home/utente/Scrivania/csvtesi/rete_Nom01_quant.tflite', "wb").write(tflite_quant_model)
#compiler compila il modello quantizzato tflite per edge tpu
os.system("edgetpu_compiler \'/home/utente/Scrivania/csvtesi/rete_Nom01_quant.tflite'")
#interpret the model
interpreter = tf.lite.Interpreter('/home/utente/Scrivania/csvtesi/rete_Nom01_quant_edgetpu.tflite',experimental_delegates=[tflite.load_delegate('libedgetpu.so.1')])
interpreter.allocate_tensors()
idt=print(interpreter.get_input_details())
odt=print(interpreter.get_output_details())
for j in range(5):
start = time.monotonic()
o_test=np.arange(len(xAcc1L_test[:,0]))
o_test=o_test[:,np.newaxis]
for i in range (len(xAcc1L_test[:,0])):
input=set_input(interpreter, xAcc1L_test[i,:])
#print("inference input %s" % input)
interpreter.invoke()
classes = get_output(interpreter,4)
output = interpreter.get_tensor(interpreter.get_output_details()[0]['index'])#/255 con edgetpu
#print("inference output %s" % output)
#print("inference classes %s" % classes)
a=np.array([one_hot_labelsAcc1L_test[i,:].argmax(axis=0)])
b=np.array(output.argmax(axis=1))
o_test[i]=b
#if a==b:
#print('good classification')
#else:
#print('bad classification')
inference_time = time.monotonic() - start
print('%.1fms ' % (inference_time * 1000))
#print(o_test)
print("Accuracy:",accuracy_score(yAcc1L_test,o_test))
My input train dataset is part of csv files and it's a matrix of dimensions = (1756,30) and my input test one is a matrix of (183,30).
This is how the data looks like (first two rows):
[[-0.283589 -0.0831421 -0.199936 -0.144523 -0.215593 -0.199029 0.0300179 -0.0299262 -0.0759612 -0.0349733 0.102882 -0.00470235 -0.14267 -0.116636 -0.0842867 -0.124638 -0.107917 -0.0995006 -0.222817 -0.256093 -0.121859 -0.130829 -0.186091 -0.174511 -0.0715493 -0.0595195 -0.054914 -0.0362971 -0.0286576 -0.0409128],
[-0.226151 -0.0386177 -0.16834 -0.0768908 -0.166611 -0.161028 0.0493133 -0.00515959 -0.0362308 -0.00723895 0.105943 -0.010825 -0.142335 -0.10863 -0.0634201 -0.112928 -0.0927994 -0.0556194 -0.180721 -0.218341 -0.0934449 -0.100047 -0.134569 -0.119806 -0.0265749 -0.044841 -0.0538225 -0.017408 -0.00528171 -0.0248457]]
I have been trying to replicate the same simple network structure in MATLAB and Keras. The problem is the accuracy I get is very different. MATLAB code gets accuracy near 0.84 and loss near 17 and Keras code gets accuracy near 0.63 and loss near 130, with Keras using double epochs to train and the same data. I think the difference is too big to be a matter of implementation, so I think I'm missing something.
The original code is from a MATLAB example in which I have made a little change to avoid normalization in the first layer.
Here is the MATLAB code:
% Load the digit training set as 4-D array data using
% |digitTrain4DArrayData|.
[trainImages,~,trainAngles] = digitTrain4DArrayData;
disp("Train Images:")
disp(trainImages(:,:,:,1))
% Display 20 random sample training digits using |imshow|.
numTrainImages = size(trainImages,4);
figure
idx = randperm(numTrainImages,20);
for i = 1:numel(idx)
subplot(4,5,i)
imshow(trainImages(:,:,:,idx(i)))
drawnow
end
%%
% Combine all the layers together in a |Layer| array.
layers = [ ...
imageInputLayer([28 28 1], 'Normalization', 'none')
convolution2dLayer(12,25)
reluLayer
fullyConnectedLayer(1)
regressionLayer];
%% Train Network'
options = trainingOptions('sgdm','InitialLearnRate',0.001, ...
'MaxEpochs',15)
net = trainNetwork(trainImages,trainAngles,layers,options)
net.Layers
%% Test Network
[testImages,~,testAngles] = digitTest4DArrayData;
predictedTestAngles = predict(net,testImages);
% *Evaluate Performance*
predictionError = testAngles - predictedTestAngles;
thr = 10;
numCorrect = sum(abs(predictionError) < thr);
numTestImages = size(testImages,4);
accuracy = numCorrect/numTestImages
%%
% Use the root-mean-square error (RMSE) to measure the differences between
% the predicted and actual angles of rotation.
squares = predictionError.^2;
rmse = sqrt(mean(squares))
%%
% *Display Box Plot of Residuals for Each Digit Class*
residuals = testAngles - predictedTestAngles;
residualMatrix = reshape(residuals,500,10);
figure
boxplot(residualMatrix, ...
'Labels',{'0','1','2','3','4','5','6','7','8','9'})
xlabel('Digit Class')
ylabel('Degrees Error')
title('Residuals')
idx = randperm(numTestImages,49);
for i = 1:numel(idx)
image = testImages(:,:,:,idx(i));
predictedAngle = predictedTestAngles(idx(i));
imagesRotated(:,:,:,i) = imrotate(image,predictedAngle,'bicubic','crop');
end
figure
subplot(1,2,1)
montage(testImages(:,:,:,idx))
title('Original')
subplot(1,2,2)
montage(imagesRotated)
title('Corrected')
Here is the Keras code:
import numpy as np
import scipy.io
import matplotlib.pyplot as plt
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense, Activation, Conv2D, BatchNormalization, AveragePooling2D
from keras.optimizers import SGD
from keras.utils import np_utils
from keras import regularizers
np.random.seed(1671) # for reproducibility
# network and training
NB_EPOCH = 30
BATCH_SIZE = 128
VERBOSE = 1
NB_CLASSES = 10 # number of outputs = number of digits
OPTIMIZER = SGD() # SGD optimizer, explained later in this chapter
N_HIDDEN = 128
VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION
data = scipy.io.loadmat('RegressionImageData.mat')
XTrain = np.rollaxis(data['XTrain'],3,0)
XTest = np.rollaxis(data['XTest'],3,0)
YTest = np.squeeze(data['YTest'])
YTrain = np.squeeze(data['YTrain'])
print("Train Images:")
print(XTrain.shape)
print(type(XTrain))
print(XTrain)
XTrain_test = np.reshape(XTrain, (5000,28,28))
with open("./test.txt", "a+") as file:
np.set_printoptions(threshold=np.nan)
file.write(np.array2string(XTrain_test[0], max_line_width=np.inf))
model = Sequential()
model.add(Conv2D(25,(12,12), input_shape=(28,28,1), strides=(1,1), activation = "relu"))
model.add(Flatten())
model.add(Dense(1))
model.summary()
sgd = SGD(lr=0.001, decay=0.1, momentum=0.9, nesterov=False)
model.compile(loss='mean_squared_error', optimizer=sgd)
history = model.fit(XTrain, YTrain,
batch_size=BATCH_SIZE, epochs=NB_EPOCH,
verbose=VERBOSE, validation_split=VALIDATION_SPLIT,
shuffle=False)
predictions= model.predict(XTrain)
[np.transpose(predictions[1:50]), np.transpose(YTrain[1:50]), np.abs(np.transpose(predictions[1:50])- np.transpose(YTrain[1:50]))]
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('rmse')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
Ypred_test= model.predict(XTest)
Ypred_test=np.reshape(Ypred_test, (5000,))
predictionError = Ypred_test - YTest
thr = 10;
numCorrect = np.sum((np.abs(predictionError) < thr)*1)
numValidationImages = len(YTest)
accuracy = numCorrect/numValidationImages
print(accuracy)
squares = np.power(predictionError,2)
rmse = np.sqrt(np.mean(squares))
print(rmse)
Anyone know where could be the gap?
I need one or more hints to get over the first pain in transfer-learning.
The following code is a stripped-down version of what I am actually trying to do, but it shows the issues even with one the fake image (A: empty / B: empty + little square) I use there. In the final version, the input will be much more complex images (which justifies the complexity of the applied base model).
The problem looks simple. Input: two types of images, output: binary classification ("square present yes/no"). The modified ResNet50 model is fed with prepared training data via ImageDataGenerator. As I can create any amount of fake data, there is no data augmentation step in the code.
Anyway, when I run the code the displayed loss (for both the Adam and the SDG optimizer) doesn't seem to improve and the accuracy quickly tends to approach the ratio of the number of the examples in the two image classes (i.e. B/A). (Note: during the weekend, I even tried for 500 epochs ... no change.)
For both (most likely connected) issues I haven't been able to spot the reason yet ... could you? Is it one of the hyper-parameters, is there an obvious glitch in the model setup or any other part of the implementation? Probably it's just something stupid, but after chasing it and playing around with different and more and more simplified versions, I am about to run out of ideas regarding what to try next.
import cv2
import matplotlib.pyplot as plt
import numpy as np
from tqdm import tqdm
from random import randint
from keras.layers import Dense, GlobalAveragePooling2D
from keras.optimizers import Adam
from keras.models import Model
from keras.applications import ResNet50
from keras.preprocessing.image import ImageDataGenerator
def modified_resnet_model():
# load ResNet50 model excluding classification layers
basemodel = ResNet50(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
# freeze model weights
for layer in basemodel.layers:
layer.trainable = False
# add new classification head
x = GlobalAveragePooling2D()(basemodel.output)
x = Dense(128, activation='relu')(x)
predictions = Dense(1, activation='softmax')(x)
modresnet50model = Model(inputs=basemodel.input, outputs=predictions)
# return the result
return modresnet50model
def data_set_creator(numsamples, probpos, target_image_size=(224, 224)):
dataset = {}
image_stack = []
immean = np.array([0.0, 0.0, 0.0])
imstat = {}
# first create target labels
lbbuf = np.zeros((numsamples, 1))
lbbuf[:int(probpos*numsamples)] = 1
lbbuf = np.random.permutation(lbbuf)
# second create matching "fake" images according to label stack
for index in tqdm(range(numsamples)):
# zero labeled images are empty
img = np.zeros((target_image_size[0], target_image_size[1], 3)).astype(np.float32)
sh = 10
if lbbuf[index]:
# all others contain a suqare somewhere
xp = randint(sh, target_image_size[0]-1-sh)
yp = randint(sh, target_image_size[1]-1-sh)
randval = 100 # randint(1, 255)
# print('center: ({0:d},{1:d}); value: {2:d}'.format(xp, yp, randval))
img[yp-sh:yp+sh, xp-sh:xp+sh, :] = randval
# else:
# print(' --- ')
# normalize image and add it to the image stack
img /= 255.0 # normalize image
image_stack.append(img)
# update mean vector
immean += cv2.mean(img)[:-1]
# assemple data set
imstat['mean'] = immean/numsamples
image_stack = np.array(image_stack)
dataset['images'] = image_stack
dataset['imstat'] = imstat
dataset['labels'] = lbbuf
# return the result
return dataset
if __name__ == '__main__':
# define some parameters
imagesize = (224, 224)
nsamples = 10000
pos_prob_train = 0.3
probposval = pos_prob_train
valfrac = 0.1 # use 10% of the data for validation
batchsize = 24
epochs = 30
stepsperepoch = 100
validationsteps = 25
# ================================================================================
# create training and validation data sets
nst = int(nsamples*(1-valfrac))
dataset_training = data_set_creator(nst, pos_prob_train, target_image_size=imagesize)
dataset_validation = data_set_creator(nsamples-nst, probposval, target_image_size=imagesize)
# subtract the mean (training data!) from all the images
for ci in range(3):
dataset_training['images'][:, :, :, ci] -= dataset_training['imstat']['mean'][ci]
dataset_validation['images'][:, :, :, ci] -= dataset_training['imstat']['mean'][ci]
# get the (modified) model
model = modified_resnet_model()
theoptimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
model.compile(optimizer=theoptimizer, loss='binary_crossentropy', metrics=['accuracy'])
print(model.summary())
# setup data input generators
train_datagen = ImageDataGenerator()
validation_datagen = ImageDataGenerator()
train_generator = train_datagen.flow(dataset_training['images'],
dataset_training['labels'],
batch_size=batchsize)
validation_generator = validation_datagen.flow(dataset_validation['images'],
dataset_validation['labels'],
batch_size=batchsize)
# train the (modified) model
history = model.fit_generator(train_generator, steps_per_epoch=stepsperepoch,
epochs=epochs, validation_data=validation_generator,
validation_steps=validationsteps)
#visualize the training and validation performance
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
nepochs = range(1, len(acc)+1)
plt.plot(nepochs, acc, 'bo', label='Training acc')
plt.plot(nepochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()
plt.savefig('trainval_acc.png')
plt.figure()
plt.plot(nepochs, loss, 'bo', label='Training loss')
plt.plot(nepochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.savefig('trainval_loss.png')
plt.show()
I have programmed a GAN model using keras but the training didn't go well. The generator model always returns a bare noise image (28x28 size) instead of something similar to mnist dataset. This doesn't give me any error though, when it comes to training discriminator model will become trainable=False, which is not what I want to do.
If this implementation is bad, please let me know. Can anyone help?
import os
import numpy as np
import matplotlib.pyplot as plt
import keras
from keras.models import Sequential
from keras.layers import Dense, Activation, BatchNormalization
from keras.optimizers import SGD, Adam, RMSprop
from keras.datasets import mnist
from keras.regularizers import l1_l2
def plot_generated(noise, Generator):
image_fake = Generator.predict(noise)
plt.figure(figsize=(10,8))
plt.show()
plt.close()
def plot_metircs(metrics, epoch=None):
plt.figure(figsize=(10,8))
plt.plot(metrics['d'], label='discriminative loss', color='b')
plt.legend()
plt.show()
plt.close()
plt.figure(figsize=(10,8))
plt.plot(metrics['g'], label='generative loss', color='r')
plt.legend()
plt.show()
plt.close()
def Generator():
model = Sequential()
LeakyReLU = keras.layers.advanced_activations.LeakyReLU(alpha=0.2)
model.add(Dense(input_dim=100, units=128, activation=LeakyReLU, name='g_input'))
model.add(Dense(input_dim=128, units=784, activation='tanh', name='g_output'))
return model
def Discriminator():
model = Sequential()
LeakyReLU = keras.layers.advanced_activations.LeakyReLU(alpha=0.2)
model.add(Dense(input_dim=784, units=128, activation=LeakyReLU, name='d_input'))
model.add(Dense(input_dim=128, units=1, activation='sigmoid', name='d_output'))
model.compile(loss='binary_crossentropy', optimizer='Adam')
return model
def Generative_Adversarial_Network(Generator, Discriminator):
model = Sequential()
model.add(Generator)
model.add(Discriminator)
# train only generator in the entire GAN architecture
Discriminator.trainable = False
model.compile(loss='binary_crossentropy', optimizer='Adam')
return model
def Training(z_input_size, Generator, Discriminator, GAN, loss_dict, X_train, epoch, batch, smooth):
for e in range(epoch):
# z: noise, used for input of G to generate fake image based on this noise! it's like a seed
noise = np.random.uniform(-1, 1, size=[batch, z_input_size])
image_fake = Generator.predict_on_batch(noise)
# sampled real_image from dataset
rand_train_index = np.random.randint(0, X_train.shape[0], size=batch)
image_real = X_train[rand_train_index, :]
# concatenate real and fake images
"""
X = [
image_real => label : 1 (we can multiply a smoothing factor)
image_fake => label : 0
]
"""
X = np.vstack((image_real, image_fake))
y = np.zeros(len(X))
# putting label "1" to image_real
y[len(image_real):] = 1*(1 - smooth)
y = y.astype(int)
# train only discriminator
d_loss = Discriminator.train_on_batch(x=X, y=y)
# NOTE: remember?? we set discriminator OFF during the training of GAN!
# So, we can safely train only generator, weight of discriminator set fixed!
g_loss = GAN.train_on_batch(x=noise, y=y[len(noise):])
loss_dict['d'].append(d_loss)
loss_dict['g'].append(g_loss)
if e%1000 == 0:
plt.imshow(image_fake)
plt.show()
plot_generated(noise, Generator)
plot_metircs(loss_dict)
return "done!"
Gen = Generator()
Dis = Discriminator()
GAN = Generative_Adversarial_Network(Gen, Dis)
GAN.summary()
Gen.summary()
Dis.summary()
gan_losses = {"d":[], "g":[], "f":[]}
epoch = 30000
batch = 1000
smooth = 0.9
z_input_size = 100
row, col = 28, 28
z_group_matrix = np.random.uniform(0, 1, examples*z_input_size)
z_group_matrix = z_group_matrix.reshape([9, z_input_size])
print(z_group_matrix.shape)
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train, X_test = X_train.reshape(X_train.shape[0], row*col), X_test.reshape(X_test.shape[0], row*col)
X_train.astype('float32')
X_test.astype('float32')
X_train, X_test = X_train/255, X_test/255
print('X_train shape: ', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
Training(z_input_size, Gen, Dis, GAN, loss_dict=gan_losses, X_train=X_train, epoch=epoch, batch=batch, smooth=smooth)
The model itself is correct.
I would suggest a few minor changes:
smooth 0.9 is too much. Make it close to 0.1.
Leak Factor you have is 0.2, usually its a very small decimal close to 0; take around
0.01/0.02.
Batchsize around 400
Epochs around 2000
And finally early stopping with a bit large threshold.