From bc00973828efe17cfccfc35b1ecd77fba9c8e98c Mon Sep 17 00:00:00 2001 From: Joao Luiz Godoy Date: Wed, 26 Aug 2026 13:51:05 -0600 Subject: [PATCH] Added troubleshooting section and file to RAI to solve the ROS2 stack is not ready in time error --- packages/robotics/rai/Dockerfile | 2 + packages/robotics/rai/README.md | 20 +++ packages/robotics/rai/ros_preflight.sh | 163 +++++++++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100755 packages/robotics/rai/ros_preflight.sh diff --git a/packages/robotics/rai/Dockerfile b/packages/robotics/rai/Dockerfile index 70720a37..e0ebfe1e 100644 --- a/packages/robotics/rai/Dockerfile +++ b/packages/robotics/rai/Dockerfile @@ -92,6 +92,8 @@ COPY test.sh /ryzers/test_rai.sh RUN chmod +x /ryzers/test_rai.sh COPY manipulation_demo.sh /ryzers/manipulation_demo.sh RUN chmod +x /ryzers/manipulation_demo.sh +COPY ros_preflight.sh /ryzers/ros_preflight.sh +RUN chmod +x /ryzers/ros_preflight.sh COPY lemonade_env.sh /ryzers/ EXPOSE 8501 diff --git a/packages/robotics/rai/README.md b/packages/robotics/rai/README.md index bba1f451..463eb46a 100644 --- a/packages/robotics/rai/README.md +++ b/packages/robotics/rai/README.md @@ -118,6 +118,26 @@ python src/rai_bench/rai_bench/examples/vlm_benchmark.py --model-name Gemma-4-E2 Available models and serving config live in `lemonade_env.sh`. +## Troubleshooting: "ROS2 stack is not ready in time." + +Run the preflight check before starting a benchmark or the demo: + +```bash +bash /ryzers/ros_preflight.sh # clean up, then verify +bash /ryzers/ros_preflight.sh --dry-run # report only, kill nothing +``` + +`rai_sim` raises this error from two different places with the same text, so the +message alone doesn't tell you which half of the stack failed. The useful detail +is the `Still waiting for ROS2 components to initialize. Missing ...` warning +logged just above it in `experiments/o3de_manipulation//benchmark.log`: + +| Missing | Meaning | +| --- | --- | +| `/spawn_entity`, `/delete_entity`, `/color_image*` | the O3DE simulator started but never brought up its ROS2 interfaces | +| `/grounding_dino_classify`, `/grounded_sam_segment`, `/manipulator_move_to` | MoveIt and the perception services never came up | + + ## Documentation - Official docs: https://robotecai.github.io/rai/ diff --git a/packages/robotics/rai/ros_preflight.sh b/packages/robotics/rai/ros_preflight.sh new file mode 100755 index 00000000..9b16da1f --- /dev/null +++ b/packages/robotics/rai/ros_preflight.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# Run me before every benchmark/demo: bash /ryzers/ros_preflight.sh +# Report without killing anything: bash /ryzers/ros_preflight.sh --dry-run +# +# A benchmark run that fails or is interrupted leaves its nodes behind. Because +# the container uses --network=host --ipc=host with a shared ROS_DOMAIN_ID, the +# next run joins the same DDS graph as those orphans and collides with them, +# which surfaces as "ROS2 stack is not ready in time." +# +# This kills the leftovers, clears the daemon's cached graph, and verifies the +# preconditions the O3DE benchmark silently depends on. + +set -uo pipefail + +RAI_DIR="${RAI_DIR:-/ryzers/rai}" +DRY_RUN=0 +[ "${1:-}" = "--dry-run" ] || [ "${1:-}" = "-n" ] && DRY_RUN=1 + +fail=0 +note() { printf ' %s\n' "$*"; } +bad() { printf ' FAIL: %s\n' "$*"; fail=1; } + +# Leftovers from a previous run. lemond/lemonade are deliberately absent: the +# server is expensive to restart and holds the loaded model. +STALE_PATTERNS=( + 'RAIManipulationDemo.GameLauncher' + 'run_perception_services.py' + 'run_perception_agents.py' + 'panda_moveit_config_demo.launch.py' + 'manipulation-demo-streamlit.py' + 'manipulation_o3de.py' + 'robotic_manipulation' + 'move_group' + 'moveit' + 'static_transform_publisher' + 'robot_state_publisher' + 'rclcpp_components/component_container' +) + +# Never signal ourselves or anything we're running under. +protected=" $$ $PPID " +pid=$$ +while read -r ppid; do + [ -z "$ppid" ] || [ "$ppid" = "0" ] && break + protected+="$ppid " + pid=$ppid +done < <(while :; do + p=$(awk '{print $4}' "/proc/$pid/stat" 2>/dev/null) || break + [ -z "$p" ] || [ "$p" = "0" ] && break + echo "$p" + pid=$p +done) + +collect() { + local out=() + for pat in "${STALE_PATTERNS[@]}"; do + while read -r p; do + [ -z "$p" ] && continue + [[ "$protected" == *" $p "* ]] && continue + out+=("$p") + done < <(pgrep -f -- "$pat" 2>/dev/null) + done + printf '%s\n' "${out[@]+"${out[@]}"}" | sort -u | sed '/^$/d' +} + +echo "== 1. stale ROS2 / simulator processes ==" +mapfile -t stale < <(collect) +if [ "${#stale[@]}" -eq 0 ]; then + note "none found" +elif [ "$DRY_RUN" -eq 1 ]; then + for p in "${stale[@]}"; do + note "would kill $p $(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null | cut -c1-80)" + done +else + for p in "${stale[@]}"; do + note "TERM $p $(tr '\0' ' ' < "/proc/$p/cmdline" 2>/dev/null | cut -c1-80)" + done + kill -TERM "${stale[@]}" 2>/dev/null + + # O3DE and move_group can take a few seconds to unwind. + for _ in $(seq 1 10); do + mapfile -t stale < <(collect) + [ "${#stale[@]}" -eq 0 ] && break + sleep 1 + done + + mapfile -t stale < <(collect) + if [ "${#stale[@]}" -gt 0 ]; then + note "still alive after 10s, sending KILL: ${stale[*]}" + kill -KILL "${stale[@]}" 2>/dev/null + sleep 2 + fi + + mapfile -t stale < <(collect) + [ "${#stale[@]}" -eq 0 ] && note "all cleared" || bad "survived KILL: ${stale[*]}" +fi + +echo "== 2. ROS2 environment ==" +if [ ! -d "$RAI_DIR" ]; then + bad "$RAI_DIR does not exist (set RAI_DIR)" +else + cd "$RAI_DIR" || bad "cannot cd to $RAI_DIR" +fi + +# The benchmark resolves the simulator binary, the MoveIt launch file and the +# perception script through paths relative to the workspace root, so an +# unsourced overlay or a stray cwd both break it in non-obvious ways. +# The ROS setup scripts read unset variables, so -u has to come off first. +set +u +[ -n "${ROS_DISTRO:-}" ] || . "/opt/ros/${ROS_DISTRO:-jazzy}/setup.bash" 2>/dev/null +case " ${AMENT_PREFIX_PATH:-} " in + *"$RAI_DIR/install"*) ;; + *) . "$RAI_DIR/install/setup.bash" 2>/dev/null ;; +esac +set -u + +case " ${AMENT_PREFIX_PATH:-} " in + *"$RAI_DIR/install"*) note "overlay sourced (AMENT_PREFIX_PATH ok)" ;; + *) bad "workspace overlay not on AMENT_PREFIX_PATH; openset.launch.py will not resolve" ;; +esac +note "cwd = $PWD" +note "ROS_DOMAIN_ID = ${ROS_DOMAIN_ID:-}" + +echo "== 3. paths the benchmark resolves relatively ==" +for p in \ + "demo_assets/manipulation/RAIManipulationDemo/RAIManipulationDemo.GameLauncher" \ + "src/examples/rai-manipulation-demo/Project/Examples/panda_moveit_config_demo.launch.py" \ + "src/rai_extensions/rai_perception/rai_perception/scripts/run_perception_services.py" +do + [ -e "$p" ] && note "ok $p" || bad "missing $p" +done + +for w in ~/.cache/rai/vision/weights/groundingdino_swint_ogc.pth \ + ~/.cache/rai/vision/weights/sam2_hiera_large.pt +do + [ -s "$w" ] && note "ok $(basename "$w")" || bad "missing perception weights: $w" +done + +echo "== 4. ROS2 graph ==" +# The daemon caches the graph and will happily report nodes that are already +# gone, so drop it and let the checks below repopulate from scratch. +if [ "$DRY_RUN" -eq 0 ]; then + ros2 daemon stop >/dev/null 2>&1 + sleep 1 +fi + +nodes=$(timeout 25 ros2 node list 2>/dev/null | sed '/^$/d') +if [ -z "$nodes" ]; then + note "graph is empty" +else + bad "graph still advertises nodes:" + printf ' %s\n' $nodes + note "another container may share this domain; set a unique ROS_DOMAIN_ID" +fi + +echo +if [ "$fail" -eq 0 ]; then + echo "PREFLIGHT OK - safe to start a run." +else + echo "PREFLIGHT FAILED - fix the items above; a run started now will likely" + echo "die with 'ROS2 stack is not ready in time.'" +fi +exit "$fail"