From 2e8f3d37a8a97258d307b8ac3efaeb765b4716da Mon Sep 17 00:00:00 2001 From: Amal Nanavati Date: Fri, 19 Jan 2024 14:26:09 -0800 Subject: [PATCH] Allow Users to Zoom Into Bite Selection Video (#120) * Allow users to zoom into the bite selection video * Rearrange display on DetectingFace page * Fix bug in scale factor * Proper error return in siganlling server * Lower robot browser's memory footprint by half * Add Segfault monitor --------- Co-authored-by: Amal Nanavati --- feedingwebapp/package-lock.json | 29 ++++ feedingwebapp/package.json | 1 + feedingwebapp/server.js | 5 + feedingwebapp/src/Pages/GlobalState.jsx | 7 + .../Pages/Home/MealStates/BiteSelection.jsx | 6 +- .../Pages/Home/MealStates/DetectingFace.jsx | 27 ++-- .../MealStates/DetectingFaceSubcomponent.jsx | 16 +-- feedingwebapp/src/Pages/Home/VideoFeed.jsx | 134 +++++++++++++++--- feedingwebapp/src/robot/VideoStream.jsx | 28 +++- 9 files changed, 205 insertions(+), 48 deletions(-) diff --git a/feedingwebapp/package-lock.json b/feedingwebapp/package-lock.json index 47fec088..c09eed55 100644 --- a/feedingwebapp/package-lock.json +++ b/feedingwebapp/package-lock.json @@ -37,6 +37,7 @@ "react-toastify": "^9.0.7", "roslib": "github:personalrobotics/roslibjs", "rosreact": "^0.2.0", + "segfault-handler": "^1.3.0", "styled-components": "^5.3.9", "web-vitals": "^2.1.4", "webpack": "^5.82.1", @@ -7269,6 +7270,14 @@ "node": ">=8" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -10472,6 +10481,11 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -15129,6 +15143,11 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nan": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", + "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==" + }, "node_modules/nanoid": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", @@ -18694,6 +18713,16 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/segfault-handler": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/segfault-handler/-/segfault-handler-1.3.0.tgz", + "integrity": "sha512-p7kVHo+4uoYkr0jmIiTBthwV5L2qmWtben/KDunDZ834mbos+tY+iO0//HpAJpOFSQZZ+wxKWuRo4DxV02B7Lg==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.2.1", + "nan": "^2.14.0" + } + }, "node_modules/select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", diff --git a/feedingwebapp/package.json b/feedingwebapp/package.json index a10f3dce..378efbad 100644 --- a/feedingwebapp/package.json +++ b/feedingwebapp/package.json @@ -32,6 +32,7 @@ "react-toastify": "^9.0.7", "roslib": "github:personalrobotics/roslibjs", "rosreact": "^0.2.0", + "segfault-handler": "^1.3.0", "styled-components": "^5.3.9", "web-vitals": "^2.1.4", "webpack": "^5.82.1", diff --git a/feedingwebapp/server.js b/feedingwebapp/server.js index 9aa941bc..0c9f886a 100644 --- a/feedingwebapp/server.js +++ b/feedingwebapp/server.js @@ -6,6 +6,8 @@ * Each subscriber sees the video stream from the publisher. */ +const SegfaultHandler = require('segfault-handler') +SegfaultHandler.registerHandler('crash.log') const express = require('express') const app = express() const bodyParser = require('body-parser') @@ -65,6 +67,7 @@ app.post('/subscribe', async ({ body }, res) => { res.json(payload) } catch (err) { console.error('Failed to process subscriber, exception: ' + err.message) + res.sendStatus(500) } }) @@ -107,10 +110,12 @@ app.post('/publish', async ({ body }, res) => { res.json(payload) } catch (err) { console.error('Failed to process publisher, exception: ' + err.message) + res.sendStatus(500) } }) function handleTrackEvent(e, topic) { + console.log('Handle track for publisher') senderStream[topic] = e.streams[0] } diff --git a/feedingwebapp/src/Pages/GlobalState.jsx b/feedingwebapp/src/Pages/GlobalState.jsx index 702990d2..746f1d58 100644 --- a/feedingwebapp/src/Pages/GlobalState.jsx +++ b/feedingwebapp/src/Pages/GlobalState.jsx @@ -138,6 +138,9 @@ export const useGlobalState = create( // this is the state we transition to after R_MovingFromMouth. In practice, // it is either R_MovingAbovePlate, R_MovingToRestingPosition, or R_DetectingFace. mostRecentBiteDoneResponse: MEAL_STATE.R_DetectingFace, + // How much the video on the Bite Selection page should be zoomed in. + biteSelectionZoom: 1.0, + // Settings values // stagingPosition: SETTINGS.stagingPosition[0], // biteInitiation: SETTINGS.biteInitiation[0], @@ -196,6 +199,10 @@ export const useGlobalState = create( setBiteTransferPageAtFace: (biteTransferPageAtFace) => set(() => ({ biteTransferPageAtFace: biteTransferPageAtFace + })), + setBiteSelectionZoom: (biteSelectionZoom) => + set(() => ({ + biteSelectionZoom: biteSelectionZoom })) // setStagingPosition: (stagingPosition) => // set(() => ({ diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx index 6551d0a6..5a3ec46a 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx @@ -36,6 +36,8 @@ const BiteSelection = (props) => { // Get the relevant global variables const setMealState = useGlobalState((state) => state.setMealState) const setBiteAcquisitionActionGoal = useGlobalState((state) => state.setBiteAcquisitionActionGoal) + const biteSelectionZoom = useGlobalState((state) => state.biteSelectionZoom) + const setBiteSelectionZoom = useGlobalState((state) => state.setBiteSelectionZoom) // Get icon image for move to mouth let moveToStagingConfigurationImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingToStagingConfiguration] @@ -458,7 +460,7 @@ const BiteSelection = (props) => { height: '100%' }} > - + { imageClicked, props.debug, props.webrtcURL, + biteSelectionZoom, + setBiteSelectionZoom, debugButton ]) diff --git a/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx b/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx index 33468408..6cc8dd66 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx @@ -38,10 +38,10 @@ const DetectingFace = (props) => { let otherDimension = isPortrait ? 'row' : 'column' // Font size for text let textFontSize = 3 - let buttonWidth = 30 - let buttonHeight = 18 - let iconWidth = 28 - let iconHeight = 16 + // let buttonWidth = 22 + let buttonHeight = 14 + let iconWidth = 20 + let iconHeight = 12 let sizeSuffix = isPortrait ? 'vh' : 'vw' /** @@ -130,10 +130,10 @@ const DetectingFace = (props) => { width: '100%' }} > - + - +

{mouthDetected ? 'Continue' : 'Continue without detection'}

@@ -144,7 +144,7 @@ const DetectingFace = (props) => { size='lg' onClick={moveToMouthCallback} style={{ - width: buttonWidth.toString() + sizeSuffix, + width: '90%', height: buttonHeight.toString() + sizeSuffix, display: 'flex', justifyContent: 'center', @@ -180,8 +180,8 @@ const DetectingFace = (props) => { size='lg' onClick={moveToRestingCallback} style={{ - width: (buttonWidth / 2).toString() + sizeSuffix, - height: (buttonHeight / 2).toString() + sizeSuffix, + width: '90%', + height: ((buttonHeight * 2) / 3).toString() + sizeSuffix, display: 'flex', justifyContent: 'center', alignContent: 'center' @@ -191,7 +191,7 @@ const DetectingFace = (props) => { src={moveToRestingImage} alt='move_to_resting_image' className='center' - style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: (iconHeight / 2).toString() + sizeSuffix }} + style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: ((iconHeight * 2) / 3).toString() + sizeSuffix }} /> @@ -206,8 +206,8 @@ const DetectingFace = (props) => { size='lg' onClick={moveAbovePlateCallback} style={{ - width: (buttonWidth / 2).toString() + sizeSuffix, - height: (buttonHeight / 2).toString() + sizeSuffix, + width: '90%', + height: ((buttonHeight * 2) / 3).toString() + sizeSuffix, display: 'flex', justifyContent: 'center', alignContent: 'center' @@ -217,7 +217,7 @@ const DetectingFace = (props) => { src={moveAbovePlateImage} alt='move_above_plate_image' className='center' - style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: (iconHeight / 2).toString() + sizeSuffix }} + style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: ((iconHeight * 2) / 3).toString() + sizeSuffix }} /> @@ -239,7 +239,6 @@ const DetectingFace = (props) => { props.webrtcURL, textFontSize, buttonHeight, - buttonWidth, sizeSuffix, iconHeight, iconWidth, diff --git a/feedingwebapp/src/Pages/Home/MealStates/DetectingFaceSubcomponent.jsx b/feedingwebapp/src/Pages/Home/MealStates/DetectingFaceSubcomponent.jsx index 0513c1aa..fe832277 100644 --- a/feedingwebapp/src/Pages/Home/MealStates/DetectingFaceSubcomponent.jsx +++ b/feedingwebapp/src/Pages/Home/MealStates/DetectingFaceSubcomponent.jsx @@ -1,5 +1,5 @@ // React Imports -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import React, { useCallback, useEffect, useRef, useState } from 'react' import PropTypes from 'prop-types' import { useMediaQuery } from 'react-responsive' import { View } from 'react-native' @@ -7,7 +7,6 @@ import { View } from 'react-native' // Local Imports import { useROS, createROSService, createROSServiceRequest, subscribeToROSTopic, unsubscribeFromROSTopic } from '../../../ros/ros_helpers' import '../Home.css' -import { convertRemToPixels } from '../../../helpers' import { MEAL_STATE } from '../../GlobalState' import { FACE_DETECTION_IMG_TOPIC, FACE_DETECTION_TOPIC, FACE_DETECTION_TOPIC_MSG, ROS_SERVICE_NAMES } from '../../Constants' import VideoFeed from '../VideoFeed' @@ -29,10 +28,6 @@ const DetectingFaceSubcomponent = (props) => { // conidered valid. NOTE: This must match the values in the MoveToMouth tree. const min_face_distance = 0.4 const max_face_distance = 1.25 - // Margin for the video feed and between the mask buttons. Note this cannot - // be re-defined per render, otherwise it messes up re-rendering order upon - // resize in VideoFeed. - const margin = useMemo(() => convertRemToPixels(1), []) /** * Connect to ROS, if not already connected. Put this in useRef to avoid @@ -130,14 +125,7 @@ const DetectingFaceSubcomponent = (props) => { height: '100%' }} > - + ) diff --git a/feedingwebapp/src/Pages/Home/VideoFeed.jsx b/feedingwebapp/src/Pages/Home/VideoFeed.jsx index 63426883..34575a8f 100644 --- a/feedingwebapp/src/Pages/Home/VideoFeed.jsx +++ b/feedingwebapp/src/Pages/Home/VideoFeed.jsx @@ -143,10 +143,10 @@ const VideoFeed = (props) => { ) // Set the width and height of the video feed - setImgWidth(childWidth) - setImgHeight(childHeight) - setScaleFactor(childScaleFactor) - }, [parentRef, props.marginTop, props.marginBottom, props.marginLeft, props.marginRight]) + setImgWidth(childWidth * props.zoom) + setImgHeight(childHeight * props.zoom) + setScaleFactor(childScaleFactor * props.zoom) + }, [parentRef, props.marginTop, props.marginBottom, props.marginLeft, props.marginRight, props.zoom]) /** When the resize event is triggered, the elements have not yet been laid out, * and hence the parent width/height might not be accurate yet based on the @@ -183,6 +183,83 @@ const VideoFeed = (props) => { [props.pointClicked, scaleFactor] ) + const renderZoomControls = useCallback( + (zoom, setZoom, zoomMin, zoomMax) => { + // Return three views, containing a button, -, to reduce the zoom, a button, + // +, to increase the zoom, and text in-between that indicates the zoom + // level. + return ( + <> + + + + +

+ {Math.round(zoom * 100)}% +

+ + + + + + ) + }, + [textFontSize] + ) + // Render the component return ( <> @@ -193,7 +270,8 @@ const VideoFeed = (props) => { alignItems: 'center', justifyContent: 'center', width: '100%', - height: '100%' + height: '100%', + overflow: 'hidden' }} > ) @@ -255,14 +343,24 @@ VideoFeed.propTypes = { */ pointClicked: PropTypes.func, // The URL of the webrtc signalling server - webrtcURL: PropTypes.string.isRequired + webrtcURL: PropTypes.string.isRequired, + // How much to zoom the camera feed by + zoom: PropTypes.number.isRequired, + // Min/max zoom + zoomMin: PropTypes.number.isRequired, + zoomMax: PropTypes.number.isRequired, + // If this is set, then VideoFeed renders buttons to allow users to change the zoom + setZoom: PropTypes.func } VideoFeed.defaultProps = { marginTop: 0, marginBottom: 0, marginLeft: 0, marginRight: 0, - topic: CAMERA_FEED_TOPIC + topic: CAMERA_FEED_TOPIC, + zoom: 1.0, + zoomMin: 1.0, + zoomMax: 2.0 } export default VideoFeed diff --git a/feedingwebapp/src/robot/VideoStream.jsx b/feedingwebapp/src/robot/VideoStream.jsx index 3b9c99ed..928271e1 100644 --- a/feedingwebapp/src/robot/VideoStream.jsx +++ b/feedingwebapp/src/robot/VideoStream.jsx @@ -7,6 +7,29 @@ import { useROS, subscribeToROSTopic, unsubscribeFromROSTopic } from '../ros/ros import { WebRTCConnection } from '../webrtc/webrtc_helpers' import { REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../Pages/Constants' +function dataURItoBlob(dataURI) { + // Adapted from https://stackoverflow.com/a/38788279 + + // convert base64 to raw binary data held in a string + // doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this + var splitString = dataURI.split(',') + var byteString = atob(splitString[1]) + + // separate out the mime component + var mimeString = splitString[0].split(':')[1].split(';')[0] + + // write the bytes of the string to an ArrayBuffer + var ab = new ArrayBuffer(byteString.length) + var ia = new Uint8Array(ab) + for (var i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i) + } + + // write the ArrayBuffer to a blob, and you're done + var blob = new Blob([ab], { type: mimeString }) + return blob +} + /** * Renders a video stream from the robot. * @@ -34,7 +57,10 @@ function VideoStream(props) { const imageCallback = useCallback( (message) => { // console.log('Got image message', message) - img.src = 'data:image/jpg;base64,' + message.data + if (img.src) { + URL.revokeObjectURL(img.src) + } + img.src = URL.createObjectURL(dataURItoBlob('data:image/jpg;base64,' + message.data)) }, [img] )