From c90b2510cd6cc50124b11b57efc91145c1a93687 Mon Sep 17 00:00:00 2001 From: "Takagi, Isamu" <43976882+isamu-takagi@users.noreply.github.com> Date: Tue, 7 May 2024 15:48:26 +0900 Subject: [PATCH] feat: enable evaluation script (#5) Signed-off-by: Takagi, Isamu --- Dockerfile | 19 ++- .../config/autoware.rviz | 2 + .../script/result-converter.py | 58 +++++++++ aichallenge/run_evaluation.bash | 110 +++++++++--------- aichallenge/run_simulator.bash | 2 +- aichallenge/simulator/.gitignore | 1 - aichallenge/simulator/simulator.bash | 4 - create_submit_file.bash | 2 +- submit/.gitignore | 3 + 9 files changed, 131 insertions(+), 70 deletions(-) create mode 100644 aichallenge/autoware/src/aichallenge_system/script/result-converter.py delete mode 100755 aichallenge/simulator/simulator.bash create mode 100644 submit/.gitignore diff --git a/Dockerfile b/Dockerfile index 45cd7291..d2bc5b92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,24 @@ # FROM osrf/ros:humble-desktop AS common -FROM ghcr.io/automotiveaichallenge/aichallenge2023-racing/autoware-universe-no-cuda:latest AS common +FROM ghcr.io/automotiveaichallenge/autoware-universe:humble-latest AS common RUN apt-get update RUN apt-get -y install libgl1-mesa-glx libgl1-mesa-dri RUN apt-get -y install iproute2 +# PATH="$PATH:/root/.local/bin" +# PATH="/usr/local/cuda/bin:$PATH" +ENV XDG_RUNTIME_DIR /tmp/xdg +ENV ROS_LOCALHOST_ONLY 1 +ENV RMW_IMPLEMENTATION rmw_cyclonedds_cpp + FROM common AS dev -ENV ROS_LOCALHOST_ONLY 1 ENV RCUTILS_COLORIZED_OUTPUT 1 -ENV RMW_IMPLEMENTATION rmw_cyclonedds_cpp FROM common AS eval +ENV RCUTILS_COLORIZED_OUTPUT 0 + RUN mkdir /ws RUN git clone --depth 1 https://github.com/AutomotiveAIChallenge/aichallenge-2024 /ws/repository RUN mv /ws/repository/aichallenge /aichallenge @@ -21,8 +27,9 @@ RUN rm -rf /aichallenge/autoware/src/aichallenge_submit RUN chmod 757 /aichallenge COPY aichallenge/simulator/ /aichallenge/simulator/ -COPY output/aichallenge_submit.tar.gz /ws +COPY submit/aichallenge_submit.tar.gz /ws RUN tar zxf /ws/aichallenge_submit.tar.gz -C /aichallenge/autoware/src +RUN rm -rf /ws RUN bash -c ' \ source /autoware/install/setup.bash; \ @@ -31,5 +38,5 @@ RUN bash -c ' \ rosdep install -y -r -i --from-paths src --ignore-src --rosdistro $ROS_DISTRO; \ colcon build --symlink-install --cmake-args -DCMAKE_BUILD_TYPE=Release' -# ENTRYPOINT [] -# CMD ["bash", "/ws/main.bash"] +ENTRYPOINT [] +CMD ["bash", "/aichallenge/run_evaluation.bash"] diff --git a/aichallenge/autoware/src/aichallenge_system/aichallenge_system_launch/config/autoware.rviz b/aichallenge/autoware/src/aichallenge_system/aichallenge_system_launch/config/autoware.rviz index cac9bb8e..dff8b062 100644 --- a/aichallenge/autoware/src/aichallenge_system/aichallenge_system_launch/config/autoware.rviz +++ b/aichallenge/autoware/src/aichallenge_system/aichallenge_system_launch/config/autoware.rviz @@ -22,6 +22,8 @@ Panels: Name: AutowareDateTimePanel - Class: rviz_plugins::AutowareStatePanel Name: AutowareStatePanel + - Class: AutowareScreenCapturePanel + Name: AutowareScreenCapturePanel Visualization Manager: Class: "" Displays: diff --git a/aichallenge/autoware/src/aichallenge_system/script/result-converter.py b/aichallenge/autoware/src/aichallenge_system/script/result-converter.py new file mode 100644 index 00000000..7073dfff --- /dev/null +++ b/aichallenge/autoware/src/aichallenge_system/script/result-converter.py @@ -0,0 +1,58 @@ +import argparse +import json +import numpy + + +def lpf_axis(axis, ws): + v = numpy.ones(ws) / ws + return numpy.convolve(axis, v, mode="valid") + + +def lpf_axes(axes, ws): + return numpy.array([lpf_axis(axis, ws) for axis in axes]) + + +def create_laps(data): + return data["laps"] + + +def create_min_time(data): + if len(data["laps"]) == 0: + return None + return min(data["laps"]) + + +def create_max_jerk(data, dt, ws): + if len(data["laps"]) == 0: + return None + original_v = numpy.array([[v["x"], v["y"], v["z"]] for v in data["velocities"]]).T + filtered_v = lpf_axes(original_v, ws) + original_a = numpy.diff(filtered_v) / dt + filtered_a = lpf_axes(original_a, ws) + original_j = numpy.diff(filtered_a) / dt + filtered_j = lpf_axes(original_j, ws) + return max(numpy.linalg.norm(j, ord=2) for j in filtered_j.T) + + +parser = argparse.ArgumentParser() +parser.add_argument("hz", type=float) +parser.add_argument("ws", type=int) +parser.add_argument("--input", default="result-details.json") +parser.add_argument("--output", default="result-summary.json") + +args = parser.parse_args() +dt = 1.0 / args.hz +ws = args.ws + +with open(args.input) as fp: + details = json.load(fp) + +summary = { + "laps": create_laps(details), + "min_time": create_min_time(details), + "max_jerk": create_max_jerk(details, dt, ws), +} + +with open(args.output, "w") as fp: + json.dump(summary, fp, indent=4) + fp.write("\n") diff --git a/aichallenge/run_evaluation.bash b/aichallenge/run_evaluation.bash index b98c691a..786bf97d 100755 --- a/aichallenge/run_evaluation.bash +++ b/aichallenge/run_evaluation.bash @@ -1,71 +1,67 @@ #!/bin/bash -export PATH="$PATH:/root/.local/bin" -export PATH="/usr/local/cuda/bin:$PATH" -export XDG_RUNTIME_DIR=/tmp/xdg -export RCUTILS_COLORIZED_OUTPUT=0 -export ROS_LOCALHOST_ONLY=1 +# Move working directory +OUTPUT_DIRECTORY=$(date +%Y%m%d-%H%M%S) +cd /output || exit +mkdir "$OUTPUT_DIRECTORY" +cd "$OUTPUT_DIRECTORY" || exit # shellcheck disable=SC1091 source /aichallenge/autoware/install/setup.bash sudo ip link set multicast on lo +sudo sysctl -w net.core.rmem_max=2147483647 >/dev/null -# Move working directory -cd /output || exit - -# Launch the simulator -echo "Launch AWSIM" -bash /aichallenge/simulator/simulator.bash & - -# Waiting for the simulator to start up -sleep 3 +# Start AWSIM +echo "Start AWSIM" +/aichallenge/simulator/AWSIM.x86_64 >/dev/null & +PID_AWSIM=$! +sleep 20 -# Launch Autoware -echo "Launch user Autoware code" +# Start Autoware +echo "Start Autoware" ros2 launch aichallenge_system_launch aichallenge_system.launch.xml >autoware.log 2>&1 & -ROSLAUNCH_PID=$! - -# Waiting for Autoware to start up -sleep 3 +PID_AUTOWARE=$! +sleep 10 # Start recording rosbag -rm -r rosbag2_autoware -ros2 bag record -a -o rosbag2_autoware & -ROSBAG_RECORD_PID=$! - -# Waiting for screen capture (TODO: This will not wait if there is no service) -# echo "Waiting for screen capture" -# until (ros2 service type /debug/service/capture_screen); do -# sleep 5 -# done - -# Start recording rviz2 -# ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger - -# Waiting for the simulator results -# echo "Waiting for the simulator results" -# until [ -f ~/awsim-logs/result.json ]; do -# sleep 5 -# done +echo "Start rosbag" +ros2 bag record -a -o rosbag2_autoware >/dev/null 2>&1 & +PID_ROSBAG=$! +sleep 5 + +# Start recording rviz2 (TODO: This will not wait if there is no service) +echo "Start screen capture" +until (ros2 service type /debug/service/capture_screen >/dev/null); do + sleep 5 +done +ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger >/dev/null +sleep 5 + +# Start driving and wait for the simulation to finish +echo "Waiting for the simulation" +ros2 service call /localization/trigger_node std_srvs/srv/SetBool '{data: true}' >/dev/null +wait $PID_AWSIM # Stop recording rviz2 -# ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger - -# Waiting for the screen capture to finish -sleep 3 - -## Stop rosbag and Autoware to finish writing logs -kill $ROSBAG_RECORD_PID -kill $ROSLAUNCH_PID - -# Waiting for the rosbag and logs -sleep 3 - -## Compress rosbag +echo "Stop screen capture" +ros2 service call /debug/service/capture_screen std_srvs/srv/Trigger >/dev/null +sleep 10 + +# Stop recording rosbag +echo "Stop rosbag" +kill $PID_ROSBAG +wait $PID_ROSBAG + +# Stop Autoware +echo "Stop Autoware" +kill $PID_AUTOWARE +wait $PID_AUTOWARE + +# Convert result +echo "Convert result" +python3 /aichallenge/autoware/src/aichallenge_system/script/result-converter.py 60 11 + +# Compress rosbag +echo "Compress rosbag" tar -czf rosbag2_autoware.tar.gz rosbag2_autoware -sleep 3 - -## Copy the logs to output directory -echo "Generation of result.json is completed." -cp ~/awsim-logs/result.json /output -cp ~/awsim-logs/verbose_result.json /output +rm -rf rosbag2_autoware diff --git a/aichallenge/run_simulator.bash b/aichallenge/run_simulator.bash index 258cbcde..774ff6d1 100755 --- a/aichallenge/run_simulator.bash +++ b/aichallenge/run_simulator.bash @@ -3,4 +3,4 @@ # shellcheck disable=SC1091 source /aichallenge/autoware/install/setup.bash sudo ip link set multicast on lo -/aichallenge/simulator/simulator.bash +/aichallenge/simulator/AWSIM.x86_64 diff --git a/aichallenge/simulator/.gitignore b/aichallenge/simulator/.gitignore index a253b027..ae4fd4a9 100644 --- a/aichallenge/simulator/.gitignore +++ b/aichallenge/simulator/.gitignore @@ -1,4 +1,3 @@ # Download the simulator and place it in this directory. * !.gitignore -!simulator.bash diff --git a/aichallenge/simulator/simulator.bash b/aichallenge/simulator/simulator.bash deleted file mode 100755 index e1309f26..00000000 --- a/aichallenge/simulator/simulator.bash +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo "Run AWSIM" -/aichallenge/simulator/AWSIM.x86_64 diff --git a/create_submit_file.bash b/create_submit_file.bash index 4e33fdc7..5b0a8890 100755 --- a/create_submit_file.bash +++ b/create_submit_file.bash @@ -1,3 +1,3 @@ #!/bin/bash -tar zcvf output/aichallenge_submit.tar.gz -C ./aichallenge/autoware/src aichallenge_submit +tar zcvf submit/aichallenge_submit.tar.gz -C ./aichallenge/autoware/src aichallenge_submit diff --git a/submit/.gitignore b/submit/.gitignore new file mode 100644 index 00000000..9a02cc5f --- /dev/null +++ b/submit/.gitignore @@ -0,0 +1,3 @@ +# This directory will store the submit file. +* +!.gitignore