feat: enable evaluation script (#5)

Signed-off-by: Takagi, Isamu <[email protected]>

Dockerfile

@@ -1,18 +1,24 @@
 # FROM osrf/ros:humble-desktop AS common
-FROM ghcr.io/automotiveaichallenge/aichallenge2023-racing/autoware-universe-no-cuda:latest AS common
+FROM ghcr.io/automotiveaichallenge/autoware-universe:humble-latest AS common
 
 RUN apt-get update
 RUN apt-get -y install libgl1-mesa-glx libgl1-mesa-dri
 RUN apt-get -y install iproute2
 
+# PATH="$PATH:/root/.local/bin"
+# PATH="/usr/local/cuda/bin:$PATH"
+ENV XDG_RUNTIME_DIR /tmp/xdg
+ENV ROS_LOCALHOST_ONLY 1
+ENV RMW_IMPLEMENTATION rmw_cyclonedds_cpp
+
 FROM common AS dev
 
-ENV ROS_LOCALHOST_ONLY 1
 ENV RCUTILS_COLORIZED_OUTPUT 1
-ENV RMW_IMPLEMENTATION rmw_cyclonedds_cpp
 
 FROM common AS eval
 
+ENV RCUTILS_COLORIZED_OUTPUT 0
+
 RUN mkdir /ws
 RUN git clone --depth 1 https://github.com/AutomotiveAIChallenge/aichallenge-2024 /ws/repository
 RUN mv /ws/repository/aichallenge /aichallenge
@@ -21,8 +27,9 @@ RUN rm -rf /aichallenge/autoware/src/aichallenge_submit
 RUN chmod 757 /aichallenge
 
 COPY aichallenge/simulator/ /aichallenge/simulator/
-COPY output/aichallenge_submit.tar.gz /ws
+COPY submit/aichallenge_submit.tar.gz /ws
 RUN tar zxf /ws/aichallenge_submit.tar.gz -C /aichallenge/autoware/src
+RUN rm -rf /ws
 
 RUN bash -c ' \
   source /autoware/install/setup.bash; \
@@ -31,5 +38,5 @@ RUN bash -c ' \
   rosdep install -y -r -i --from-paths src --ignore-src --rosdistro $ROS_DISTRO; \
   colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=Release'
 
-# ENTRYPOINT []
-# CMD ["bash", "/ws/main.bash"]
+ENTRYPOINT []
+CMD ["bash", "/aichallenge/run_evaluation.bash"]

aichallenge/autoware/src/aichallenge_system/aichallenge_system_launch/config/autoware.rviz

@@ -22,6 +22,8 @@ Panels:
     Name: AutowareDateTimePanel
   - Class: rviz_plugins::AutowareStatePanel
     Name: AutowareStatePanel
+  - Class: AutowareScreenCapturePanel
+    Name: AutowareScreenCapturePanel
 Visualization Manager:
   Class: ""
   Displays:

aichallenge/autoware/src/aichallenge_system/script/result-converter.py

@@ -0,0 +1,58 @@
+import argparse
+import json
+import numpy
+
+
+def lpf_axis(axis, ws):
+    v = numpy.ones(ws) / ws
+    return numpy.convolve(axis, v, mode="valid")
+
+
+def lpf_axes(axes, ws):
+    return numpy.array([lpf_axis(axis, ws) for axis in axes])
+
+
+def create_laps(data):
+    return data["laps"]
+
+
+def create_min_time(data):
+    if len(data["laps"]) == 0:
+        return None
+    return min(data["laps"])
+
+
+def create_max_jerk(data, dt, ws):
+    if len(data["laps"]) == 0:
+        return None
+    original_v = numpy.array([[v["x"], v["y"], v["z"]] for v in data["velocities"]]).T
+    filtered_v = lpf_axes(original_v, ws)
+    original_a = numpy.diff(filtered_v) / dt
+    filtered_a = lpf_axes(original_a, ws)
+    original_j = numpy.diff(filtered_a) / dt
+    filtered_j = lpf_axes(original_j, ws)
+    return max(numpy.linalg.norm(j, ord=2) for j in filtered_j.T)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("hz", type=float)
+parser.add_argument("ws", type=int)
+parser.add_argument("--input", default="result-details.json")
+parser.add_argument("--output", default="result-summary.json")
+
+args = parser.parse_args()
+dt = 1.0 / args.hz
+ws = args.ws
+
+with open(args.input) as fp:
+    details = json.load(fp)
+
+summary = {
+    "laps": create_laps(details),
+    "min_time": create_min_time(details),
+    "max_jerk": create_max_jerk(details, dt, ws),
+}
+
+with open(args.output, "w") as fp:
+    json.dump(summary, fp, indent=4)
+    fp.write("\n")

aichallenge/run_evaluation.bash

@@ -1,71 +1,67 @@
 #!/bin/bash
 
-export PATH="$PATH:/root/.local/bin"
-export PATH="/usr/local/cuda/bin:$PATH"
-export XDG_RUNTIME_DIR=/tmp/xdg
-export RCUTILS_COLORIZED_OUTPUT=0
-export ROS_LOCALHOST_ONLY=1
+# Move working directory
+OUTPUT_DIRECTORY=$(date +%Y%m%d-%H%M%S)
+cd /output || exit
+mkdir "$OUTPUT_DIRECTORY"
+cd "$OUTPUT_DIRECTORY" || exit
 
 # shellcheck disable=SC1091
 source /aichallenge/autoware/install/setup.bash
 sudo ip link set multicast on lo
+sudo sysctl -w net.core.rmem_max=2147483647 >/dev/null
 
-# Move working directory
-cd /output || exit
-
-# Launch the simulator
-echo "Launch AWSIM"
-bash /aichallenge/simulator/simulator.bash &
-
-# Waiting for the simulator to start up
-sleep 3
+# Start AWSIM
+echo "Start AWSIM"
+/aichallenge/simulator/AWSIM.x86_64 >/dev/null &
+PID_AWSIM=$!
+sleep 20
 
-# Launch Autoware
-echo "Launch user Autoware code"
+# Start Autoware
+echo "Start Autoware"
 ros2 launch aichallenge_system_launch aichallenge_system.launch.xml >autoware.log 2>&1 &
-ROSLAUNCH_PID=$!
-
-# Waiting for Autoware to start up
-sleep 3
+PID_AUTOWARE=$!
+sleep 10
 
 # Start recording rosbag
-rm -r rosbag2_autoware
-ros2 bag record -a -o rosbag2_autoware &
-ROSBAG_RECORD_PID=$!
-
-# Waiting for screen capture (TODO: This will not wait if there is no service)
-# echo "Waiting for screen capture"
-# until (ros2 service type /debug/service/capture_screen); do
-#     sleep 5
-# done
-
-# Start recording rviz2
-# ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger
-
-# Waiting for the simulator results
-# echo "Waiting for the simulator results"
-# until [ -f ~/awsim-logs/result.json ]; do
-#     sleep 5
-# done
+echo "Start rosbag"
+ros2 bag record -a -o rosbag2_autoware >/dev/null 2>&1 &
+PID_ROSBAG=$!
+sleep 5
+
+# Start recording rviz2 (TODO: This will not wait if there is no service)
+echo "Start screen capture"
+until (ros2 service type /debug/service/capture_screen >/dev/null); do
+    sleep 5
+done
+ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger >/dev/null
+sleep 5
+
+# Start driving and wait for the simulation to finish
+echo "Waiting for the simulation"
+ros2 service call /localization/trigger_node std_srvs/srv/SetBool '{data: true}' >/dev/null
+wait $PID_AWSIM
 
 # Stop recording rviz2
-# ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger
-
-# Waiting for the screen capture to finish
-sleep 3
-
-## Stop rosbag and Autoware to finish writing logs
-kill $ROSBAG_RECORD_PID
-kill $ROSLAUNCH_PID
-
-# Waiting for the rosbag and logs
-sleep 3
-
-## Compress rosbag
+echo "Stop screen capture"
+ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger >/dev/null
+sleep 10
+
+# Stop recording rosbag
+echo "Stop rosbag"
+kill $PID_ROSBAG
+wait $PID_ROSBAG
+
+# Stop Autoware
+echo "Stop Autoware"
+kill $PID_AUTOWARE
+wait $PID_AUTOWARE
+
+# Convert result
+echo "Convert result"
+python3 /aichallenge/autoware/src/aichallenge_system/script/result-converter.py 60 11
+
+# Compress rosbag
+echo "Compress rosbag"
 tar -czf rosbag2_autoware.tar.gz rosbag2_autoware
-sleep 3
-
-## Copy the logs to output directory
-echo "Generation of result.json is completed."
-cp ~/awsim-logs/result.json /output
-cp ~/awsim-logs/verbose_result.json /output
+rm -rf rosbag2_autoware

aichallenge/run_simulator.bash

@@ -3,4 +3,4 @@
 # shellcheck disable=SC1091
 source /aichallenge/autoware/install/setup.bash
 sudo ip link set multicast on lo
-/aichallenge/simulator/simulator.bash
+/aichallenge/simulator/AWSIM.x86_64

aichallenge/simulator/.gitignore

@@ -1,4 +1,3 @@
 # Download the simulator and place it in this directory.
 *
 !.gitignore
-!simulator.bash

create_submit_file.bash

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-tar zcvf output/aichallenge_submit.tar.gz -C ./aichallenge/autoware/src aichallenge_submit
+tar zcvf submit/aichallenge_submit.tar.gz -C ./aichallenge/autoware/src aichallenge_submit

submit/.gitignore

@@ -0,0 +1,3 @@
+# This directory will store the submit file.
+*
+!.gitignore