diff --git a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py index 858e4bfa..d0efdaeb 100755 --- a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py +++ b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/DummyRealSense.py @@ -85,6 +85,9 @@ def __init__(self): self.camera_info_publisher = self.create_publisher( CameraInfo, "~/camera_info", 1 ) + self.aligned_depth_camera_info_publisher = self.create_publisher( + CameraInfo, "~/aligned_depth/camera_info", 1 + ) if self.video is not None: self.num_frames = 0 self.bridge = CvBridge() @@ -174,6 +177,7 @@ def publish_frames(self): self.compressed_image_publisher.publish(compressed_frame_msg) self.aligned_depth_publisher.publish(depth_frame_msg) self.camera_info_publisher.publish(self.camera_info_msg) + self.aligned_depth_camera_info_publisher.publish(self.camera_info_msg) rate.sleep() diff --git a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py index da9ead4b..cd5da709 100755 --- a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py +++ b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/FaceDetection.py @@ -7,7 +7,7 @@ import numpy as np import rclpy from rclpy.node import Node -from sensor_msgs.msg import Image +from sensor_msgs.msg import CompressedImage from threading import Lock @@ -65,7 +65,10 @@ def __init__( # Subscribe to the camera feed self.subscription = self.create_subscription( - Image, "camera/color/image_raw", self.camera_callback, 1 + CompressedImage, + "camera/color/image_raw/compressed", + self.camera_callback, + 1, ) self.subscription # prevent unused variable warning @@ -73,7 +76,9 @@ def __init__( self.publisher_results = self.create_publisher( FaceDetection, "face_detection", 1 ) - self.publisher_image = self.create_publisher(Image, "face_detection_img", 1) + self.publisher_image = self.create_publisher( + CompressedImage, "face_detection_img/compressed", 1 + ) def toggle_face_detection_callback(self, request, response): """ @@ -150,15 +155,16 @@ def camera_callback(self, msg): face_detection_msg.is_face_detected = is_face_detected if is_face_detected: # Add a dummy face marker to the sensor_msgs/Image - cv_image = self.bridge.imgmsg_to_cv2(msg, "bgr8") + cv_image = self.bridge.compressed_imgmsg_to_cv2(msg, "bgr8") + height, width, _ = cv_image.shape cv2.circle( cv_image, - (msg.width // 2, msg.height // 2), - msg.height // 25, + (width // 2, height // 2), + height // 25, (0, 0, 255), -1, ) - annotated_msg = self.bridge.cv2_to_imgmsg(cv_image, "bgr8") + annotated_msg = self.bridge.cv2_to_compressed_imgmsg(cv_image, "jpeg") annotated_img = annotated_msg # Publish the detected mouth center. The below is a hardcoded # rough position of the mouth from the side staging location, @@ -185,7 +191,6 @@ def camera_callback(self, msg): annotated_img = msg face_detection_msg.is_mouth_open = open_mouth_detected self.publisher_results.publish(face_detection_msg) - self.get_logger().info("Published face detection") self.publisher_image.publish(annotated_img) diff --git a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py index 0bec0a45..c330d7ef 100644 --- a/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py +++ b/feeding_web_app_ros2_test/feeding_web_app_ros2_test/SegmentFromPoint.py @@ -9,7 +9,7 @@ from rclpy.action import ActionServer, CancelResponse, GoalResponse from rclpy.executors import MultiThreadedExecutor from rclpy.node import Node -from sensor_msgs.msg import CompressedImage, Image, RegionOfInterest +from sensor_msgs.msg import CompressedImage, RegionOfInterest from shapely.geometry import MultiPoint import threading import time @@ -34,8 +34,8 @@ def __init__(self, sleep_time=2.0, send_feedback_hz=10): # Subscribe to the image topic, to store the latest image self.image_subscriber = self.create_subscription( - Image, - "/camera/color/image_raw", + CompressedImage, + "/camera/color/image_raw/compressed", self.image_callback, 1, ) @@ -123,8 +123,8 @@ def segment_image(self, seed_point, result, segmentation_success): with self.latest_img_msg_lock: latest_img_msg = self.latest_img_msg result.header = latest_img_msg.header - width = latest_img_msg.width - height = latest_img_msg.height + img = self.bridge.compressed_imgmsg_to_cv2(latest_img_msg, "bgr8") + width, height, _ = img.shape # Sleep (dummy segmentation) time.sleep(self.sleep_time) @@ -175,6 +175,10 @@ def segment_image(self, seed_point, result, segmentation_success): format="jpeg", data=cv2.imencode(".jpg", mask_img)[1].tostring(), ) + mask_msg.rgb_image = CompressedImage( + format="jpeg", + data=cv2.imencode(".jpg", img[y_min:y_max, x_min:x_max])[1].tostring(), + ) mask_msg.item_id = "dummy_food_id_%d" % (i) mask_msg.confidence = np.random.random() result.detected_items.append(mask_msg) diff --git a/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml b/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml index 467a9a47..ec32a668 100644 --- a/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml +++ b/feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml @@ -10,8 +10,6 @@ - - @@ -23,8 +21,9 @@ + - + diff --git a/feedingwebapp/src/App.jsx b/feedingwebapp/src/App.jsx index 7c0290eb..779b7a63 100644 --- a/feedingwebapp/src/App.jsx +++ b/feedingwebapp/src/App.jsx @@ -22,23 +22,22 @@ import BiteSelectionPointMask from './Pages/Home/BiteSelectionUIStates/BiteSelec * @param {APP_PAGE} appPage - The current app page. Must be one of the * states specified in APP_PAGE. * @param {string} rosbridgeURL - The URL of the rosbridge server. - * @param {string} webVideoServerURL - The URL of the web_video_server. * @param {bool} debug - Whether to run it in debug mode or not. */ -function getComponentByAppPage(appPage, rosbridgeURL, webVideoServerURL, debug) { +function getComponentByAppPage(appPage, rosbridgeURL, debug) { switch (appPage) { case APP_PAGE.Home: // Must wrap a component in ROS tags for it to be able to connect to ROS return ( -
- +
+ ) case APP_PAGE.Settings: return ( -
+
) @@ -57,8 +56,6 @@ function App() { // Get the rosbridge URL const rosbridgeURL = 'ws://'.concat(window.location.hostname, ':', process.env.REACT_APP_ROSBRIDGE_PORT) - // Get the web_video_server URL - const webVideoServerURL = 'http://'.concat(window.location.hostname, ':', process.env.REACT_APP_WEB_VIDEO_SERVER_PORT) // Get the debug flag const debug = process.env.REACT_APP_DEBUG === 'true' @@ -68,7 +65,7 @@ function App() { <> - + -
+
} @@ -93,7 +90,7 @@ function App() { path='/test_bite_selection_ui/point_mask_selection' element={ -
+
} @@ -103,7 +100,7 @@ function App() { path='/test_bite_selection_ui/food_name_selection' element={ -
+
} diff --git a/feedingwebapp/src/Pages/Constants.js b/feedingwebapp/src/Pages/Constants.js index ebe256fe..dd6ede91 100644 --- a/feedingwebapp/src/Pages/Constants.js +++ b/feedingwebapp/src/Pages/Constants.js @@ -48,10 +48,10 @@ NON_MOVING_STATES.add(MEAL_STATE.U_PostMeal) export { NON_MOVING_STATES } // The names of the ROS topic(s) -export const CAMERA_FEED_TOPIC = '/local/camera/color/image_raw' +export const CAMERA_FEED_TOPIC = '/local/camera/color/image_raw/compressed' export const FACE_DETECTION_TOPIC = '/face_detection' export const FACE_DETECTION_TOPIC_MSG = 'ada_feeding_msgs/FaceDetection' -export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img' +export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img/compressed' // States from which, if they fail, it is NOT okay for the user to retry the // same action. diff --git a/feedingwebapp/src/Pages/Header/Header.jsx b/feedingwebapp/src/Pages/Header/Header.jsx index 63f47f19..5e32d6d8 100644 --- a/feedingwebapp/src/Pages/Header/Header.jsx +++ b/feedingwebapp/src/Pages/Header/Header.jsx @@ -4,9 +4,6 @@ import React, { useCallback, useEffect, useState } from 'react' import Navbar from 'react-bootstrap/Navbar' import Nav from 'react-bootstrap/Nav' import { useMediaQuery } from 'react-responsive' -// PropTypes is used to validate that the used props are in fact passed to this -// Component -import PropTypes from 'prop-types' // Toast generates a temporary pop-up with a timeout. import { ToastContainer /* , toast */ } from 'react-toastify' import 'react-toastify/dist/ReactToastify.css' @@ -23,7 +20,7 @@ import LiveVideoModal from './LiveVideoModal' * clicking "Video", and the ToastContainer popup that specifies when the user * cannot click Settings. */ -const Header = (props) => { +const Header = () => { // Create a local state variable to toggle on/off the video // TODO: Since this local state variable is in the header, the LiveVideoModal // continues showing even if the state changes. Is this desirable? Perhaps @@ -166,13 +163,9 @@ const Header = (props) => { * The LiveVideoModal toggles on and off with the Video button and shows the * robot's live camera feed. */} - setVideoShow(false)} /> + setVideoShow(false)} /> ) } -Header.propTypes = { - // The URL of the ROS web video server - webVideoServerURL: PropTypes.string.isRequired -} export default Header diff --git a/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx b/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx index a46c95b7..23f92c97 100644 --- a/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx +++ b/feedingwebapp/src/Pages/Header/LiveVideoModal.jsx @@ -49,7 +49,8 @@ function LiveVideoModal(props) {