We Keep Coding

Why is my Python Subscriber/Publisher node not publishing anything? Thank you

The instructions for the question are below
Question report 2.7. You are now ready to make a ROS node to perform real-time
extraction of the centroid of the line from the images acquired by the camera of your
robot.
File name: image_segment_node.py
Subscribes to topics:
raspicam_node/image (type sensor_msgs/Image )
Publishes to topics:
• /image/segmented (type sensor_msgs/Image )
• /image/centroid (type geometry_msgs/PointStamped )
Description: This node should import the file image_processing.py to use all the
functions that you have developed so far. For each new image received from the camera,
the node should perform the same operations described in Question provided 2.3, but
without visualization of the results. Instead, the node should:
Publish the segmented image with the line on the topic /image/segmented .
Insert the computed horizontal centroid in the x field of a PointStamped message,
add the current time to the header.stamp field, and then publish the message
on the topic /image/centroid
The code I have for image_segment_node.py is below
#!/usr/bin/env python
"""This is a node to perform real-time extraction of the centroid of the line from images acquired by camera of robot and export data."""
import rospy
import numpy as np
import image_processing as ip
from cv_bridge import CvBridge
from sensor_msgs.msg import Image
from geometry_msgs.msg import PointStamped
import cv2
def callback(msg):
global pub1
global pub2
rospy.loginfo("started")
img = CvBridge.imgmsg_to_cv2(msg, 'bgr8')
img = cv2.imread(img)
lb, ub = ip.classifier_parameters()
img_seg = cv2.inRange(img, lb, ub)
x = ip.image_centroid_horizontal(img_seg)
img_seg = ip.image_one_to_three_channels(img_seg)
img_seg = ip.image_line_vertical(img_seg, x)
pub1.publish(img_seg)
point1 = PointStamped()
point1.point = x
point1.header = rospy.Time.now()
pub2.publish(point1)
def main():
global pub1
global pub2
rospy.init_node('image_segment_node')
rospy.Subscriber("/raspicam_node/image", Image, callback=callback, queue_size=1)
pub1 = rospy.Publisher("/image/segmented", Image, queue_size=10)
pub2 = rospy.Publisher("/image/centroid", PointStamped, queue_size=10)
rate = rospy.Rate(10)
# rospy.spin()
if __name__ == '__main__':
try:
main()
finally:
pass
Terminal tells me this
drand21#roslab:~/ros_ws/src/me416_lab/nodes$ rostopic echo /image/centroid
WARNING: topic [/image/centroid] does not appear to be published yet

Can you try the code below, I changed only the structure. There doesn't seem any problem with it, but warning is clearly about "pub2" not publishing.
#!/usr/bin/env python
"""This is a node to perform real-time extraction of the centroid of the line from images acquired by camera of robot and export data."""
import rospy
import numpy as np
import image_processing as ip
from cv_bridge import CvBridge
from sensor_msgs.msg import Image
from geometry_msgs.msg import PointStamped
import cv2
class image_segment():
# Class function for creating map relations
def __init__(self):
rospy.init_node('image_segment_node')
self.rate = rospy.Rate(10)
# Publishers
self.pub1 = rospy.Publisher("/image/segmented", Image, queue_size=10)
self.pub2 = rospy.Publisher("/image/centroid", PointStamped, queue_size=10)
# Subscribers
rospy.Subscriber("/raspicam_node/image", Image, callback=self.callback_im, queue_size=1)
def callback_im(self, msg):
rospy.loginfo("started")
img = CvBridge.imgmsg_to_cv2(msg, 'bgr8')
img = cv2.imread(img)
lb, ub = ip.classifier_parameters()
img_seg = cv2.inRange(img, lb, ub)
x = ip.image_centroid_horizontal(img_seg)
img_seg = ip.image_one_to_three_channels(img_seg)
img_seg = ip.image_line_vertical(img_seg, x)
self.pub1.publish(img_seg)
point1 = PointStamped()
point1.point = x
point1.header = rospy.Time.now()
self.pub2.publish(point1)
def main(self):
while not rospy.is_shutdown():
###
self.rate.sleep()
if __name__ == '__main__':
try:
im_class = image_segment()
im_class.main()
finally:
pass

Loading a model once instead of each time I call the image detection script

I have a very simple API that predicts the presence, or not, of unsafe driving behaviors. In addition to identifying the non-regularity of the behavior, the API allows to characterize the infraction committed.
The API also allows to generate a heat map to locate which part of the image allowed the models to make their predictions.
So far, the API loads the ClassificationModel model every time we ask for it on the image passed through the script. How to load the model only once?
main.py
from model import ClassificationModel
import argparse
import os
def predict_img(img_path, with_heatmap=False):
model = ClassificationModel() <--- Aqui descargo el modelo cada vez que llamo a predict_img
row = model.predict(img_path=img_path, row=None, img=None, load=True, original_image_path=img_path,
print_heatmap=with_heatmap)
results = row.to_json()
print(results)
return results
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--path", "-p", help="path of the image", required=True)
parser.add_argument("--heatmap", "-heat", help="generate heatmap or not", required=True)
args = parser.parse_args()
if not(os.path.exists(args.path)):
raise FileNotFoundError("{} does not exist.".format(args.path))
elif args.path.split(".")[-1] not in ["png", "jpg", "jpeg"]:
raise TypeError("The file is not a valid image file.")
if args.heatmap.upper() == "TRUE":
args.heatmap = True
else:
args.heatmap = False
result = predict_img(img_path=args.path, with_heatmap=args.heatmap)
model.py
import numpy as np
np.random.seed(40)
from keras_preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import load_model
import settings as st
from predict import Predict
class ClassificationModel(Predict):
def __init__(self):
self.list_pos = [
"safe driving",
"texting - right",
"talking to the phone - right",
"texting - left",
"talking on the phone - left",
"operating the radio",
"drinking",
"reaching behind",
"hair and makeup",
"talking to passenger"
]
self.image_size = 384
self.generator = ImageDataGenerator(brightness_range=[0.7, 1.5],
rotation_range=35,
fill_mode='nearest',
zoom_range=0.125,
rescale=1. / 255)
self.model = load_model(st.model_path)
self.threshold = 0.5
self.n_step = 5
self.normal_index = 0

must have same datatype, got tf.uint8 != tf.float32 (behavioral cloning simulator)

While trying the project of car behavioral cloning, I successfully trained my model and generated the file model-001.h5.
now in order to test the model, it must successfully drive the vehicle in Udacity self-driving simulator this is done by running the drive.py and the generated model-001.h5 file
python drive.py model-001.h5
however, I receive this error.
x and y must have the same dtype, got tf.uint8 != tf.float32
x and y must have the same dtype, got tf.uint8 != tf.float32
x and y must have the same dtype, got tf.uint8 != tf.float32
Error message
I don't know if this is error is from drive.py or model.py or utils.py
but here is the drive.py code here:
import argparse
import base64
from datetime import datetime
import os
import shutil
import numpy as np
import socketio
import eventlet
import eventlet.wsgi
from PIL import Image
from flask import Flask
from io import BytesIO
from tensorflow.keras.models import load_model
import utils
sio = socketio.Server()
app = Flask(__name__)
model = None
prev_image_array = None
MAX_SPEED = 25.0
MIN_SPEED = 10.0
speed_limit = MAX_SPEED
#sio.on('telemetry')
def telemetry(sid, data):
if data:
# The current steering angle of the car
steering_angle = float(data["steering_angle"])
# The current throttle of the car
throttle = float(data["throttle"])
# The current speed of the car
speed = float(data["speed"])
# The current image from the center camera of the car
image = Image.open(BytesIO(base64.b64decode(data["image"])))
# save frame
if args.image_folder != '':
timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
image_filename = os.path.join(args.image_folder, timestamp)
image.save('{}.jpg'.format(image_filename))
try:
image = np.asarray(image) # from PIL image to numpy array
image = utils.preprocess(image) # apply the preprocessing
image = np.array([image]) # the model expects 4D array
# predict the steering angle for the image
steering_angle = float(model.predict(image, batch_size=1))
# lower the throttle as the speed increases
# if the speed is above the current speed limit, we are on a downhill.
# make sure we slow down first and then go back to the original max speed.
global speed_limit
if speed > speed_limit:
speed_limit = MIN_SPEED # slow down
else:
speed_limit = MAX_SPEED
throttle = 1.0 - steering_angle**2 - (speed/speed_limit)**2
print('{} {} {}'.format(steering_angle, throttle, speed))
send_control(steering_angle, throttle)
except Exception as e:
print(e)
else:
# NOTE: DON'T EDIT THIS.
sio.emit('manual', data={}, skip_sid=True)
#sio.on('connect')
def connect(sid, environ):
print("connect ", sid)
send_control(0, 0)
def send_control(steering_angle, throttle):
sio.emit(
"steer",
data={
'steering_angle': steering_angle.__str__(),
'throttle': throttle.__str__()
},
skip_sid=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Remote Driving')
parser.add_argument(
'model',
type=str,
help='Path to model h5 file. Model should be on the same path.'
)
parser.add_argument(
'image_folder',
type=str,
nargs='?',
default='',
help='Path to image folder. This is where the images from the run will be saved.'
)
args = parser.parse_args()
model = load_model(args.model)
if args.image_folder != '':
print("Creating image folder at {}".format(args.image_folder))
if not os.path.exists(args.image_folder):
os.makedirs(args.image_folder)
else:
shutil.rmtree(args.image_folder)
os.makedirs(args.image_folder)
print("RECORDING THIS RUN ...")
else:
print("NOT RECORDING THIS RUN ...")
# wrap Flask application with engineio's middleware
app = socketio.Middleware(sio, app)
# deploy as an eventlet WSGI server
eventlet.wsgi.server(eventlet.listen(('', 4567)), app)
even though the telemetry(sid, data) has an if statement to change the incoming data to float format

It'd be helpful to see on what line you get an error.
It's not obvious to me where the "if statement to change the incoming data to float format" that you refer to is. I see that the model is called on whatever image is loaded, in whatever datatype it comes from. I predict that the image is stored as uint8, the most efficient storage for a (0,255) colour representation.
So try:
image = np.asarray(image, dtype=np.float32)
To create an array of the correct datatype for your model, which I am presuming is tf.float32.

how do i reduce the loading time of a pre-trained model?

While loading the weights of "Imagenet" using ResNet50 it nearly takes 10-11sec each time while loading the weights.
Is there any way to reduces the loading time ?
Code:
from flask import Flask, render_template, request
from werkzeug import secure_filename
from flask import request,Flask
import json
import os
import time
from keras.preprocessing import image as image_util
from keras.applications.imagenet_utils import preprocess_input
from keras.applications.imagenet_utils import decode_predictions
# from keras.applications import ResNet50
from keras.applications.inception_v3 import InceptionV3
import numpy as np
app = Flask(__name__)
#app.route('/object_rec', methods=['POST'])
def object_rec():
f = request.files['file']
file_path = ("./upload/"+secure_filename(f.filename))
f.save(file_path)
image = image_util.load_img(file_path,target_size=(299,299))
image = image_util.img_to_array(image)
image = np.expand_dims(image,axis=0) #(224,224,3) --> (1,224,224,3)
image = preprocess_input(image)
start_time = time.time()
model = InceptionV3(weights="imagenet")
pred = model.predict(image)
p = decode_predictions(pred)
ans = p[0][0]
acc = ans[2]
acc = str(acc)
if ans[1] == "Granny_Smith":
ans = ans[1]
ans = 'Apple'
else:
ans = ans[1]
print("THE PREDICTED IMAGE IS: "+ans)
print("THE ACCURACY IS: "+acc)
print("--- %s seconds ---" % (time.time() - start_time))
result = {
"status": True,
"object": ans,
"score":acc
}
result = json.dumps(result)
return result
if __name__ == '__main__':
app.run(host='0.0.0.0',port=6000,debug=True)
time taken would differ between 8-11 sec.
I would be good if it loads the model in 3-4sec and does classification.
Thanks in advance

The way you can do it, is to load the model in a specific session and then every time you want to use the model just set that specific session, then just call predict where you need it:
app = Flask(__name__)
sess = tf.Session(config=tf_config)
graph = tf.get_default_graph()
# IMPORTANT: models have to be loaded AFTER SETTING THE SESSION for keras!
# Otherwise, their weights will be unavailable in the threads after the
session there has been set
set_session(sess)
model = InceptionV3(weights="imagenet")
#app.route('/object_rec', methods=['POST'])
def object_rec():
global sess
global graph
with graph.as_default():
set_session(sess)
model.predict(...)
if __name__ == '__main__':
app.run(host='0.0.0.0',port=6000,debug=True)

how to load keras models which are being called within a function outside the URL's endpoint?

i have the following script which currently is working fine (although quite slow 33s/qry). My main problem right now is that i am calling the loadweights() function under the \predict endpoint and so everytime i start a query it starts initializing the the keras models and the weights which makes it really slow. i somehow want to call the function outside the endpoints scope or store the models and weights somewhere. How can i do that? i tried to call the function outside the endpoint just beneath the loadweights() definition but it gives me 504 gateway error related to nginx. i am working with flask,nginx,uwsgi on google cloud server.
import dlib
import requests
import numpy as np
from skimage import io
from skimage.transform import resize
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras import applications
from flask import Flask, jsonify, request, abort, make_response
import tensorflow as tf
app = Flask(__name__)
auth_token = 'WyIxYSDFg467YT'
top_model_weights_ethnicity = '/ethnicity.071217.23-0.28.hdf5'
img_width, img_height = 139, 139
confidence_ethnicity = '0.59'
detector = dlib.get_frontal_face_detector()
g1 = tf.Graph()
class_to_label_ethnicity = {"0": "arabs", "1": "asia", "2": "black", "3": "hispanics-latinos",
"4": "southasia", "5": "white"}
with g1.as_default():
model_ethnicity = Sequential()
model_ethnicity.add(Flatten(input_shape=(3, 3, 1536)))
model_ethnicity.add(Dense(256, activation='relu'))
model_ethnicity.add(Dropout(0.5))
model_ethnicity.add(Dense(6, activation='softmax'))
def get_face(path):
with g1.as_default():
img = io.imread(path)
dets = detector(img, 1)
output = None
for i, d in enumerate(dets):
img = img[d.top():d.bottom(), d.left():d.right()]
img = resize(img, (img_width, img_height))
output = np.expand_dims(img, axis=0)
break
return output
def get_pretrained_model():
with g1.as_default():
global pretrained_model
pretrained_model = applications.InceptionResNetV2(include_top=False, weights='imagenet',
input_shape=(img_width, img_height, 3))
return pretrained_model
def get_features(image, pretrained_model):
with g1.as_default():
features = pretrained_model.predict(image)
return features
def loadweights():
with g1.as_default():
model_ethnicity.load_weights(top_model_weights_ethnicity)
pretrained_model = get_pretrained_model()
return model_ethnicity
#app.route("/predict", methods=['GET', 'POST'])
def predict_ethnicity():
with g1.as_default():
loadweights()
confidence = request.args.get('confidence', confidence_ethnicity)
path_to_img = request.args.get('url')
if get_face(path_to_img) is None:
abort(make_response(jsonify(message="No face found."), 454))
else:
features = get_features(get_face(path_to_img), pretrained_model)
prediction = model_ethnicity.predict_proba(features)
ethnicity = {class_to_label_ethnicity[str(y)]: str(value) for (x, y), value in np.ndenumerate(prediction)}
suggestion = class_to_label_ethnicity[str(np.argmax(prediction[0]))] \
if np.max(prediction[0]) > float(confidence) else ""
return jsonify({'probabilities': ethnicity, 'suggestion': suggestion}), 200
if __name__ == "__main__":
app.run(host='0.0.0.0', debug=False)