Allow Users to Zoom Into Bite Selection Video (#120)

* Allow users to zoom into the bite selection video * Rearrange display on DetectingFace page * Fix bug in scale factor * Proper error return in siganlling server * Lower robot browser's memory footprint by half * Add Segfault monitor --------- Co-authored-by: Amal Nanavati <amaln@cs.washington.edu>
personalrobotics · Jan 19, 2024 · 2e8f3d3 · 2e8f3d3
1 parent 90c95e2
commit 2e8f3d3
Show file tree

Hide file tree

Showing 9 changed files with 205 additions and 48 deletions.
diff --git a/feedingwebapp/package-lock.json b/feedingwebapp/package-lock.json
diff --git a/feedingwebapp/package.json b/feedingwebapp/package.json
@@ -32,6 +32,7 @@
     "react-toastify": "^9.0.7",
     "roslib": "github:personalrobotics/roslibjs",
     "rosreact": "^0.2.0",
+    "segfault-handler": "^1.3.0",
     "styled-components": "^5.3.9",
     "web-vitals": "^2.1.4",
     "webpack": "^5.82.1",

diff --git a/feedingwebapp/server.js b/feedingwebapp/server.js
@@ -6,6 +6,8 @@
  * Each subscriber sees the video stream from the publisher.
  */
 
+const SegfaultHandler = require('segfault-handler')
+SegfaultHandler.registerHandler('crash.log')
 const express = require('express')
 const app = express()
 const bodyParser = require('body-parser')
@@ -65,6 +67,7 @@ app.post('/subscribe', async ({ body }, res) => {
     res.json(payload)
   } catch (err) {
     console.error('Failed to process subscriber, exception: ' + err.message)
+    res.sendStatus(500)
   }
 })
 
@@ -107,10 +110,12 @@ app.post('/publish', async ({ body }, res) => {
     res.json(payload)
   } catch (err) {
     console.error('Failed to process publisher, exception: ' + err.message)
+    res.sendStatus(500)
   }
 })
 
 function handleTrackEvent(e, topic) {
+  console.log('Handle track for publisher')
   senderStream[topic] = e.streams[0]
 }
 

diff --git a/feedingwebapp/src/Pages/GlobalState.jsx b/feedingwebapp/src/Pages/GlobalState.jsx
@@ -138,6 +138,9 @@ export const useGlobalState = create(
       // this is the state we transition to after R_MovingFromMouth. In practice,
       // it is either R_MovingAbovePlate, R_MovingToRestingPosition, or R_DetectingFace.
       mostRecentBiteDoneResponse: MEAL_STATE.R_DetectingFace,
+      // How much the video on the Bite Selection page should be zoomed in.
+      biteSelectionZoom: 1.0,
+
       // Settings values
       // stagingPosition: SETTINGS.stagingPosition[0],
       // biteInitiation: SETTINGS.biteInitiation[0],
@@ -196,6 +199,10 @@ export const useGlobalState = create(
       setBiteTransferPageAtFace: (biteTransferPageAtFace) =>
         set(() => ({
           biteTransferPageAtFace: biteTransferPageAtFace
+        })),
+      setBiteSelectionZoom: (biteSelectionZoom) =>
+        set(() => ({
+          biteSelectionZoom: biteSelectionZoom
         }))
       // setStagingPosition: (stagingPosition) =>
       //   set(() => ({

diff --git a/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx b/feedingwebapp/src/Pages/Home/MealStates/BiteSelection.jsx
@@ -36,6 +36,8 @@ const BiteSelection = (props) => {
   // Get the relevant global variables
   const setMealState = useGlobalState((state) => state.setMealState)
   const setBiteAcquisitionActionGoal = useGlobalState((state) => state.setBiteAcquisitionActionGoal)
+  const biteSelectionZoom = useGlobalState((state) => state.biteSelectionZoom)
+  const setBiteSelectionZoom = useGlobalState((state) => state.setBiteSelectionZoom)
   // Get icon image for move to mouth
   let moveToStagingConfigurationImage = MOVING_STATE_ICON_DICT[MEAL_STATE.R_MovingToStagingConfiguration]
 
@@ -458,7 +460,7 @@ const BiteSelection = (props) => {
                 height: '100%'
               }}
             >
-              <VideoFeed pointClicked={imageClicked} webrtcURL={props.webrtcURL} />
+              <VideoFeed pointClicked={imageClicked} webrtcURL={props.webrtcURL} zoom={biteSelectionZoom} setZoom={setBiteSelectionZoom} />
             </View>
           </View>
           <View
@@ -592,6 +594,8 @@ const BiteSelection = (props) => {
     imageClicked,
     props.debug,
     props.webrtcURL,
+    biteSelectionZoom,
+    setBiteSelectionZoom,
     debugButton
   ])
 

diff --git a/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx b/feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx
@@ -38,10 +38,10 @@ const DetectingFace = (props) => {
   let otherDimension = isPortrait ? 'row' : 'column'
   // Font size for text
   let textFontSize = 3
-  let buttonWidth = 30
-  let buttonHeight = 18
-  let iconWidth = 28
-  let iconHeight = 16
+  // let buttonWidth = 22
+  let buttonHeight = 14
+  let iconWidth = 20
+  let iconHeight = 12
   let sizeSuffix = isPortrait ? 'vh' : 'vw'
 
   /**
@@ -130,10 +130,10 @@ const DetectingFace = (props) => {
             width: '100%'
           }}
         >
-          <View style={{ flex: 5, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
+          <View style={{ flex: 6, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
             <DetectingFaceSubcomponent faceDetectedCallback={faceDetectedCallback} webrtcURL={props.webrtcURL} />
           </View>
-          <View style={{ flex: 3, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
+          <View style={{ flex: 2, alignItems: 'center', justifyContent: 'center', width: '100%', height: '100%' }}>
             <p className='transitionMessage' style={{ marginBottom: '0px', fontSize: textFontSize.toString() + sizeSuffix }}>
               {mouthDetected ? 'Continue' : 'Continue without detection'}
             </p>
@@ -144,7 +144,7 @@ const DetectingFace = (props) => {
               size='lg'
               onClick={moveToMouthCallback}
               style={{
-                width: buttonWidth.toString() + sizeSuffix,
+                width: '90%',
                 height: buttonHeight.toString() + sizeSuffix,
                 display: 'flex',
                 justifyContent: 'center',
@@ -180,8 +180,8 @@ const DetectingFace = (props) => {
                 size='lg'
                 onClick={moveToRestingCallback}
                 style={{
-                  width: (buttonWidth / 2).toString() + sizeSuffix,
-                  height: (buttonHeight / 2).toString() + sizeSuffix,
+                  width: '90%',
+                  height: ((buttonHeight * 2) / 3).toString() + sizeSuffix,
                   display: 'flex',
                   justifyContent: 'center',
                   alignContent: 'center'
@@ -191,7 +191,7 @@ const DetectingFace = (props) => {
                   src={moveToRestingImage}
                   alt='move_to_resting_image'
                   className='center'
-                  style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: (iconHeight / 2).toString() + sizeSuffix }}
+                  style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: ((iconHeight * 2) / 3).toString() + sizeSuffix }}
                 />
               </Button>
             </View>
@@ -206,8 +206,8 @@ const DetectingFace = (props) => {
                 size='lg'
                 onClick={moveAbovePlateCallback}
                 style={{
-                  width: (buttonWidth / 2).toString() + sizeSuffix,
-                  height: (buttonHeight / 2).toString() + sizeSuffix,
+                  width: '90%',
+                  height: ((buttonHeight * 2) / 3).toString() + sizeSuffix,
                   display: 'flex',
                   justifyContent: 'center',
                   alignContent: 'center'
@@ -217,7 +217,7 @@ const DetectingFace = (props) => {
                   src={moveAbovePlateImage}
                   alt='move_above_plate_image'
                   className='center'
-                  style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: (iconHeight / 2).toString() + sizeSuffix }}
+                  style={{ width: (iconWidth / 2).toString() + sizeSuffix, height: ((iconHeight * 2) / 3).toString() + sizeSuffix }}
                 />
               </Button>
             </View>
@@ -239,7 +239,6 @@ const DetectingFace = (props) => {
     props.webrtcURL,
     textFontSize,
     buttonHeight,
-    buttonWidth,
     sizeSuffix,
     iconHeight,
     iconWidth,

diff --git a/feedingwebapp/src/Pages/Home/MealStates/DetectingFaceSubcomponent.jsx b/feedingwebapp/src/Pages/Home/MealStates/DetectingFaceSubcomponent.jsx
@@ -1,13 +1,12 @@
 // React Imports
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
+import React, { useCallback, useEffect, useRef, useState } from 'react'
 import PropTypes from 'prop-types'
 import { useMediaQuery } from 'react-responsive'
 import { View } from 'react-native'
 
 // Local Imports
 import { useROS, createROSService, createROSServiceRequest, subscribeToROSTopic, unsubscribeFromROSTopic } from '../../../ros/ros_helpers'
 import '../Home.css'
-import { convertRemToPixels } from '../../../helpers'
 import { MEAL_STATE } from '../../GlobalState'
 import { FACE_DETECTION_IMG_TOPIC, FACE_DETECTION_TOPIC, FACE_DETECTION_TOPIC_MSG, ROS_SERVICE_NAMES } from '../../Constants'
 import VideoFeed from '../VideoFeed'
@@ -29,10 +28,6 @@ const DetectingFaceSubcomponent = (props) => {
   // conidered valid. NOTE: This must match the values in the MoveToMouth tree.
   const min_face_distance = 0.4
   const max_face_distance = 1.25
-  // Margin for the video feed and between the mask buttons. Note this cannot
-  // be re-defined per render, otherwise it messes up re-rendering order upon
-  // resize in VideoFeed.
-  const margin = useMemo(() => convertRemToPixels(1), [])
 
   /**
    * Connect to ROS, if not already connected. Put this in useRef to avoid
@@ -130,14 +125,7 @@ const DetectingFaceSubcomponent = (props) => {
           height: '100%'
         }}
       >
-        <VideoFeed
-          marginTop={margin}
-          marginBottom={margin}
-          marginLeft={margin}
-          marginRight={margin}
-          topic={FACE_DETECTION_IMG_TOPIC}
-          webrtcURL={props.webrtcURL}
-        />
+        <VideoFeed topic={FACE_DETECTION_IMG_TOPIC} webrtcURL={props.webrtcURL} />
       </View>
     </>
   )