Skip to content

Commit 8c59c91

Browse files
committed
Added dummy RealSense node, integrated with the app
1 parent e43c464 commit 8c59c91

File tree

10 files changed

+136
-24
lines changed

10 files changed

+136
-24
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env python3
2+
import cv2
3+
from cv_bridge import CvBridge
4+
import pathlib
5+
import rclpy
6+
from rclpy.node import Node
7+
from sensor_msgs.msg import Image
8+
import threading
9+
10+
11+
class DummyRealSense(Node):
12+
"""
13+
Reads in a video file and publishes the frames as ROS2 messages.
14+
"""
15+
16+
def __init__(self, video_path, fps=30, topic="/camera/color/image_raw"):
17+
super().__init__("dummy_real_sense")
18+
19+
# Read in the video file
20+
self.video = cv2.VideoCapture(video_path)
21+
self.fps = fps
22+
23+
# Create the publisher
24+
self.topic = topic
25+
self.publisher_ = self.create_publisher(Image, topic, 1)
26+
self.num_frames = 0
27+
self.bridge = CvBridge()
28+
29+
# Launch the publisher in a separate thread
30+
self.thread = threading.Thread(target=self.publish_frames, daemon=True)
31+
self.thread.start()
32+
33+
def publish_frames(self):
34+
# Maintain the rate
35+
rate = self.create_rate(self.fps)
36+
37+
while True:
38+
# Capture frame-by-frame
39+
ret, frame = self.video.read()
40+
self.num_frames += 1
41+
# If the last frame is reached, reset the capture and the frame_counter
42+
if self.num_frames == self.video.get(cv2.CAP_PROP_FRAME_COUNT):
43+
self.num_frames = 0
44+
self.video.set(cv2.CAP_PROP_POS_FRAMES, self.num_frames)
45+
# Convert to a ROS2 msg
46+
frame_msg = self.bridge.cv2_to_imgmsg(frame, "bgr8")
47+
frame_msg.header.stamp = self.get_clock().now().to_msg()
48+
self.publisher_.publish(frame_msg)
49+
# # Our operations on the frame come here
50+
# gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
51+
# # Display the resulting frame
52+
# cv2.imshow('frame',gray)
53+
# if cv2.waitKey(1) & 0xFF == ord('q'):
54+
# break
55+
rate.sleep()
56+
57+
58+
def main(args=None):
59+
rclpy.init(args=args)
60+
61+
video_name = "2022_11_01_ada_picks_up_carrots_camera_compressed_ft_tf.mp4"
62+
video_path = str(
63+
(
64+
pathlib.Path(__file__).parent.parent.parent.parent.parent
65+
/ "share/data"
66+
/ video_name
67+
).resolve()
68+
)
69+
70+
dummy_real_sense = DummyRealSense(video_path)
71+
72+
rclpy.spin(dummy_real_sense)
73+
74+
# Destroy the node explicitly
75+
# (optional - otherwise it will be done automatically
76+
# when the garbage collector destroys the node object)
77+
dummy_real_sense.destroy_node()
78+
rclpy.shutdown()
79+
80+
81+
if __name__ == "__main__":
82+
main()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<launch>
2+
<!-- The ROSBridge Node -->
3+
<include file="$(find-pkg-share rosbridge_server)/launch/rosbridge_websocket_launch.xml"/>
4+
<!-- The ROS web_video_server -->
5+
<node pkg="web_video_server" exec="web_video_server" name="web_video_server"/>
6+
7+
<!-- The dummy nodes -->
8+
9+
<!-- The RealSense Node -->
10+
<node pkg="feeding_web_app_ros2_test" exec="DummyRealSense" name="DummyRealSense"/>
11+
<!-- The MoveAbovePlate action -->
12+
<node pkg="feeding_web_app_ros2_test" exec="MoveAbovePlate" name="MoveAbovePlate"/>
13+
</launch>

feeding_web_app_ros2_test/package.xml

+2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
<test_depend>ament_pep257</test_depend>
1313
<test_depend>python3-pytest</test_depend>
1414
<exec_depend>feeding_web_app_ros2_msgs</exec_depend>
15+
<exec_depend>cv_bridge</exec_depend>
1516
<exec_depend>rclpy</exec_depend>
17+
<exec_depend>sensor_msgs</exec_depend>
1618
<exec_depend>std_msgs</exec_depend>
1719

1820
<export>

feeding_web_app_ros2_test/setup.py

+13
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from glob import glob
2+
import os
13
from setuptools import setup
24

35
package_name = "feeding_web_app_ros2_test"
@@ -9,6 +11,16 @@
911
data_files=[
1012
("share/ament_index/resource_index/packages", ["resource/" + package_name]),
1113
("share/" + package_name, ["package.xml"]),
14+
# Add the video used by the dummy RealSense publisher
15+
(
16+
"share/data",
17+
["data/2022_11_01_ada_picks_up_carrots_camera_compressed_ft_tf.mp4"],
18+
),
19+
# Include all launch files.
20+
(
21+
os.path.join("share", package_name, "launch"),
22+
glob(os.path.join("launch", "*launch.[pxy][yma]*")),
23+
),
1224
],
1325
install_requires=["setuptools"],
1426
zip_safe=True,
@@ -20,6 +32,7 @@
2032
entry_points={
2133
"console_scripts": [
2234
# Scripts for the main app
35+
"DummyRealSense = feeding_web_app_ros2_test.DummyRealSense:main",
2336
"MoveAbovePlate = feeding_web_app_ros2_test.MoveAbovePlate:main",
2437
# Scripts for the "TestROS" component
2538
"listener = feeding_web_app_ros2_test.subscriber:main",

feedingwebapp/README.md

+5-6
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ The overall user flow for this robot can be seen below.
1515
- [ROS2 Humble](https://docs.ros.org/en/humble/Installation.html)
1616
- [PRL fork of rosbridge_suite](https://github.com/personalrobotics/rosbridge_suite). This fork enables rosbridge_suite to communicate with ROS2 actions.
1717
- [ada_feeding (branch: ros2-devel)](https://github.com/personalrobotics/ada_feeding/tree/ros2-devel).
18+
- [web_video_server (`ros2` branch)](https://github.com/RobotWebTools/web_video_server/tree/ros2)
1819

1920
## Getting Started in Computer
2021

@@ -35,14 +36,12 @@ The overall user flow for this robot can be seen below.
3536
#### Launching Dummy Nodes
3637
This repository includes several dummy nodes that match the interface that the robot nodes will use. By running the dummy nodes alongside the app, we can test the app's communication with the robot even without actual robot code running.
3738

38-
The below instructions are for `MoveAbovePlate`; we will add to the instructions as more dummy nodes get implemented.
3939
1. Navigate to your ROS2 workspace: `cd {path/to/your/ros2/workspace}`
4040
2. Build your workspace: `colcon build`
41-
3. Launch rosbridge: `source install/setup.bash; ros2 launch rosbridge_server rosbridge_websocket_launch.xml`
42-
4. In another terminal, run the MoveAbovePlate action: `source install/setup.bash; ros2 run feeding_web_app_ros2_test MoveAbovePlate`
43-
5. In another terminal, navigate to the web app folder: `cd {path/to/feeding_web_interface}/feedingwebapp`
44-
6. Start the app: `npm start`
45-
7. Use a web browser to navigate to `localhost:3000`.
41+
3. Launch the dummy nodes, rosbridge, and web_video_server: `source install/setup.bash; ros2 launch feeding_web_app_ros2_test feeding_web_app_dummy_nodes_launch.xml`
42+
4. In another terminal, navigate to the web app folder: `cd {path/to/feeding_web_interface}/feedingwebapp`
43+
5. Start the app: `npm start`
44+
6. Use a web browser to navigate to `localhost:3000`.
4645

4746
You should now see that the web browser is connected to ROS. Further, you should see that when the `MoveAbovePlate` page starts, it should call the action (exactly once), and render feedback. "Pause" should cancel the action, and "Resume" should re-call it. Refreshing the page should cancel the action. When the action returns success, the app should automatically transition to the next page.
4847

feedingwebapp/src/Pages/Constants.js

+3
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ FOOTER_STATE_ICON_DICT[MEAL_STATE.R_MovingToMouth] = '/robot_state_imgs/move_to_
2323
FOOTER_STATE_ICON_DICT[MEAL_STATE.R_StowingArm] = '/robot_state_imgs/stowing_arm_position.svg'
2424
export { FOOTER_STATE_ICON_DICT }
2525

26+
// The names of the camera feed ROS topic(s)
27+
export const CAMERA_FEED_TOPIC = '/camera/color/image_raw'
28+
2629
// For states that call ROS actions, this dictionary contains
2730
// the action name and the message type
2831
let ROS_ACTIONS_NAMES = {}

feedingwebapp/src/Pages/Header/Header.jsx

+3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ import LiveVideoModal from './LiveVideoModal'
2121
*/
2222
const Header = () => {
2323
// Create a local state variable to toggle on/off the video
24+
// TODO: Since this local state variable is in the header, the LiveVideoModal
25+
// continues showing even if the state changes. Is this desirable? Perhaps
26+
// it should close if the state changes?
2427
const [videoShow, setVideoShow] = useState(false)
2528

2629
// Get the relevant global state variables

feedingwebapp/src/Pages/Header/LiveVideoModal.jsx

+7-9
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import React, { useRef } from 'react'
55
import Modal from 'react-bootstrap/Modal'
66

77
// Local imports
8-
import { REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../Constants'
8+
import { REALSENSE_WIDTH, REALSENSE_HEIGHT, CAMERA_FEED_TOPIC } from '../Constants'
99
import { convertRemToPixels, scaleWidthHeightToWindow } from '../../helpers'
1010

1111
/**
@@ -23,7 +23,6 @@ function LiveVideoModal(props) {
2323
// marginTop: bs-modal-header-padding, h4 font size & line height, bs-modal-header-padding, bs-modal-padding
2424
const marginTop = convertRemToPixels(1 + 1.5 * 1.5 + 1 + 1)
2525
const marginBottom = convertRemToPixels(1)
26-
console.log('marginBottom', marginBottom)
2726
const marginLeft = convertRemToPixels(1)
2827
const marginRight = convertRemToPixels(1)
2928

@@ -53,13 +52,12 @@ function LiveVideoModal(props) {
5352
</Modal.Header>
5453
<Modal.Body style={{ overflow: 'hidden' }}>
5554
<center>
56-
<iframe
57-
src={`http://localhost:8080/stream?topic=/camera/color/image_raw&default_transport=compressed&width=${Math.round(
58-
width
59-
)}&height=${Math.round(height)}&quality=20`}
60-
allow='autoplay; encrypted-media'
61-
allowfullscreen
62-
title='video'
55+
<img
56+
src={'http://localhost:8080/stream?topic='.concat(
57+
CAMERA_FEED_TOPIC,
58+
`&width=${Math.round(width)}&height=${Math.round(height)}&quality=20`
59+
)}
60+
alt='Live video feed from the robot'
6361
style={{ width: width, height: height, display: 'block' }}
6462
/>
6563
</center>

feedingwebapp/src/Pages/Home/MealStates/PlateLocator.jsx

+8-9
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import Button from 'react-bootstrap/Button'
55
// Local Imports
66
import '../Home.css'
77
import { useGlobalState, MEAL_STATE } from '../../GlobalState'
8-
import { REALSENSE_WIDTH, REALSENSE_HEIGHT } from '../../Constants'
8+
import { REALSENSE_WIDTH, REALSENSE_HEIGHT, CAMERA_FEED_TOPIC } from '../../Constants'
99
import { convertRemToPixels, scaleWidthHeightToWindow } from '../../../helpers'
1010

1111
/**
@@ -51,14 +51,13 @@ const PlateLocator = () => {
5151
* Display the live stream from the robot's camera.
5252
*/}
5353
<center>
54-
<iframe
55-
src={`http://localhost:8080/stream?topic=/camera/color/image_raw&default_transport=compressed&width=${Math.round(
56-
width
57-
)}&height=${Math.round(height)}&quality=20`}
58-
allow='autoplay; encrypted-media'
59-
allowFullScreen
60-
title='video'
61-
style={{ width: width, height: height, display: 'block', margin: '1rem' }}
54+
<img
55+
src={'http://localhost:8080/stream?topic='.concat(
56+
CAMERA_FEED_TOPIC,
57+
`&width=${Math.round(width)}&height=${Math.round(height)}&quality=20`
58+
)}
59+
alt='Live video feed from the robot'
60+
style={{ width: width, height: height, display: 'block' }}
6261
/>
6362
</center>
6463

0 commit comments

Comments
 (0)