From a4590e3f48d13f6e5f7dcbe745b63383ca294867 Mon Sep 17 00:00:00 2001 From: Dave Coleman Date: Thu, 21 May 2026 14:24:34 -0600 Subject: [PATCH 1/2] ci: switch to GPU runner via moveit_pro_ci v0.3.1, surface per-objective test progress Two CI-infra changes folded together: 1. Bump the reusable workflow ref to PickNikRobotics/moveit_pro_ci v0.3.1, set enable_gpu: true, and switch the runner label from picknik-16-amd64 to picknik-16-amd64-gpu. v0.3.1 appends the CUDA image suffix when enable_gpu is true (moveit_pro_ci#26) -- without it, v0.3.0 set the label but kept the non-CUDA image, so MuJoCo's EGL rendering still ran through llvmpipe on CPU. 2. Add src/lab_sim/test/conftest.py with two pytest hooks (logstart, logreport) that write directly to fd 2, bypassing pytest's --capture=fd. Without this, a CTest timeout kills pytest before any per-test output is flushed, leaving the CI log silent past 'collected N items'. The hooks emit START / PASSED|FAILED|SKIPPED + duration for every test, so the next timeout (if any) names the offending objective. --- .github/workflows/ci.yaml | 19 ++++++++--- src/lab_sim/test/conftest.py | 62 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 src/lab_sim/test/conftest.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 45d2cddeb..701747193 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,13 +19,22 @@ concurrency: jobs: integration-test-in-studio-container: - uses: PickNikRobotics/moveit_pro_ci/.github/workflows/workspace_integration_test.yaml@13d4cdf34697226ac67e34d93dda8b94fa477336 # v0.2.1 + uses: PickNikRobotics/moveit_pro_ci/.github/workflows/workspace_integration_test.yaml@d490a1dfe91758dd1da3277fbd47b5b1efe5e0c3 # v0.3.1 with: - image_tag: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.ref_name }} + # Pinned to 9.3.0-rc9 while the main-branch CUDA images are not yet + # published; 9.2.1's moveit_pro_test_utils is too old (missing + # reset_simulation_before_test that lab_sim's integration test imports). + # The dynamic expression below would normally resolve to the PR's base + # branch (main) or the pushed ref. Revert once the main-* CUDA tags exist. + # image_tag: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.ref_name }} + image_tag: "9.3.0-rc9" colcon_test_args: "--executor sequential" - runner: "picknik-16-amd64" - # Coarsen MuJoCo timestep on CI (default 0.002s = 500Hz) so the heavier 3.6.0 - # constraint solver stays at-or-under realtime on CI runners. See + runner: "picknik-16-amd64-gpu" + enable_gpu: true + # Re-assert MuJoCo timestep on CI as a backstop. Scene files in this repo + # are standardized to 0.003s, but this override catches any future scene + # whose include chain bypasses that standard, keeping the heavier 3.6.0 + # constraint solver at-or-under realtime on CI runners. See # PickNikRobotics/moveit_pro#18534 for the underlying flake history. mujoco_ci_timestep: "0.003" use_ccache: true diff --git a/src/lab_sim/test/conftest.py b/src/lab_sim/test/conftest.py new file mode 100644 index 000000000..1890fef52 --- /dev/null +++ b/src/lab_sim/test/conftest.py @@ -0,0 +1,62 @@ +# Copyright 2026 PickNik Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of the PickNik Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +"""Live per-objective progress for objectives_integration_test. + +Pytest's default ``--capture=fd`` redirects fds 1 and 2, so when CTest kills +the test on timeout (TIMEOUT 600 in CMakeLists.txt) all per-test output is +lost and the CI log shows nothing past pytest's "collected N items" header. + +These hooks write directly to fd 2, bypassing the capture, so the CI log +always shows which objective was running when the timeout fired and how long +each completed objective took — the information needed to triage flakes, +budget overruns, and runner regressions. +""" + +import os +import time + +_started_at: dict[str, float] = {} + + +def pytest_runtest_logstart(nodeid, location): + _started_at[nodeid] = time.monotonic() + os.write(2, f" START {nodeid}\n".encode()) + + +def pytest_runtest_logreport(report): + if report.when != "call": + return + nodeid = report.nodeid + elapsed = time.monotonic() - _started_at.get(nodeid, time.monotonic()) + outcome = report.outcome.upper() # PASSED / FAILED / SKIPPED + line = f" {outcome:7s} {nodeid} ({elapsed:.1f}s)" + if report.failed and report.longrepr: + reason = str(report.longrepr).splitlines()[-1][:200] + line += f"\n └─ {reason}" + os.write(2, (line + "\n").encode()) From a12d42150334784954c639830a11391b812c41c0 Mon Sep 17 00:00:00 2001 From: Dave Coleman Date: Wed, 20 May 2026 16:48:38 -0600 Subject: [PATCH 2/2] chore(sim): standardize MuJoCo camera resolution to 1280x720 across robot configs Sets `resolution="1280 720"` on every in the workspace's MJCF scene files, and ensures every top-level scene's is at least 1280x720 so the offscreen render buffer fits the largest camera. Previously inconsistent: - 1280x720 (lunar_sim, hangar_sim, picknik_accessories/ur5e wrist_camera) - 640x480 (lab_sim, kitchen_sim, grinding_sim, april_tag_sim, dual_arm_sim, factory_sim, space_satellite_sim, kinova_sim and variants) This was discovered when `lab_sim` failed to start with: [ros2_control_node] [ERROR] The published robot description file (urdf) seems not to be genuine. ... Camera resolution mismatch, set: in the MJCF model. The wrist_camera in picknik_accessories/mujoco_assets/ur5e/ur5e.xml was already at 1280x720, but lab_sim's scene declared the global offscreen buffer at only 640x480, so the hardware plugin rejected the URDF. Standardizing to 1280x720 everywhere keeps the included ur5e wrist_camera working in every UR-based config and avoids the same trap silently waiting in the others. space_satellite_sim_camera_cal kept at 1920x1080 (intentional higher resolution for camera-intrinsics calibration data). phoebe_sim (submodule) is not included; it needs a separate PR in PickNikRobotics/phoebe_ws (and was already at 1280x720 anyway). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/april_tag_sim/description/scene.xml | 6 +++++- src/dual_arm_sim/description/mujoco/scene.xml | 10 ++++++++-- src/factory_sim/description/scene.xml | 6 +++--- src/grinding_sim/description/scene.xml | 4 ++-- src/kitchen_sim/description/mujoco/scene.xml | 6 +++--- src/lab_sim/description/scene.xml | 9 +++++---- .../kinova_sim/description/mujoco/gen3_7dof.xml | 2 +- .../description/mujoco/gen3_7dof_rafti_fingers.xml | 2 +- .../kinova_sim/description/mujoco/landsat_scene.xml | 4 ++-- .../description/mujoco/rafti_fingers_scene.xml | 4 ++-- .../kinova_sim/description/mujoco/scene.xml | 4 ++-- .../description/mujoco/gen3_7dof_body.xml | 2 +- .../space_satellite_sim/description/mujoco/scene.xml | 6 +++--- 13 files changed, 38 insertions(+), 27 deletions(-) diff --git a/src/april_tag_sim/description/scene.xml b/src/april_tag_sim/description/scene.xml index 2b42bc047..87a223eb7 100644 --- a/src/april_tag_sim/description/scene.xml +++ b/src/april_tag_sim/description/scene.xml @@ -8,6 +8,10 @@