Dedup sam2 and sam3 models

.gitmodules

@@ -27,3 +27,9 @@
 	path = src/external_dependencies/phoebe_ws
 	url = https://github.com/PickNikRobotics/phoebe_ws.git
 	branch = for-example-ws-no-dups
+[submodule "src/moveit_pro_sam3"]
+	path = src/moveit_pro_sam3
+	url = https://github.com/PickNikRobotics/moveit_pro_sam3.git
+[submodule "src/moveit_pro_sam2"]
+	path = src/moveit_pro_sam2
+	url = https://github.com/PickNikRobotics/moveit_pro_sam2.git

src/dual_arm_sim/objectives/find_with_prompt.xml

@@ -7,8 +7,6 @@
         ID="Segment Point Cloud from Text Prompt Subtree"
         _collapsed="false"
         camera_topic_name="/scene_camera/camera_info"
-        decoder_model_path="models/decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
         image_topic_name="/scene_camera/color"
         masks_visualization_topic="/masks_visualization"
         model_package="moveit_pro_clipseg"

src/dual_arm_sim/objectives/segment_point_cloud_from_text.xml

@@ -9,8 +9,6 @@
     _description="Captures a point cloud and requests the user to click an object in the image to be segmented. The point cloud is then filtered to only include the selected object."
     _favorite="false"
     camera_topic_name="/wrist_camera/camera_info"
-    decoder_model_path="models/decoder.onnx"
-    encoder_model_path="models/sam2_hiera_large_encoder.onnx"
     image_topic_name="/wrist_camera/color"
     masks_visualization_topic="/masks_visualization"
     model_package="moveit_pro_clipseg"
@@ -87,11 +85,6 @@
       />
       <input_port name="clip_model_path" default="{clip_model_path}" />
       <input_port name="clipseg_model_path" default="{clipseg_model_path}" />
-      <input_port name="decoder_model_path" default="models/decoder.onnx" />
-      <input_port
-        name="encoder_model_path"
-        default="models/sam2_hiera_large_encoder.onnx"
-      />
       <input_port name="erosion_size" default="{erosion_size}" />
       <input_port name="image_topic_name" default="/wrist_camera/color" />
       <input_port

src/hangar_sim/objectives/move_boxes_to_loading_zone_from_waypoint.xml

@@ -130,8 +130,8 @@
                   image="{image}"
                   pixel_coords="{pixel_coords}"
                   masks2d="{masks2d}"
-                  model_package="lab_sim"
-                  decoder_model_path="models/decoder.onnx"
+                  model_package="moveit_pro_sam2"
+                  decoder_model_path="models/sam2_decoder.onnx"
                   encoder_model_path="models/sam2_hiera_large_encoder.onnx"
                 />
                 <Action

src/hangar_sim/objectives/segment_image_from_point.xml

@@ -10,11 +10,11 @@
       <SubTree
         ID="Segment Image from Point Subtree"
         _collapsed="true"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         encoder_model_path="models/sam2_hiera_large_encoder.onnx"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
-        model_package="lab_sim"
+        model_package="moveit_pro_sam2"
         pixel_coords="{pixel_coords}"
         masks2d="{masks2d}"
       />

src/hangar_sim/objectives/segment_point_cloud_from_clicked_point.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="UTF-8" ?>
 <root
   BTCPP_format="4"
   main_tree_to_execute="Segment Point Cloud from Clicked Point"
@@ -12,13 +12,12 @@
     <Control ID="Sequence">
       <SubTree
         ID="Segment Point Cloud from Clicked Point Subtree"
-        _collapsed="true"
         camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         encoder_model_path="models/sam2_hiera_large_encoder.onnx"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
-        model_package="lab_sim"
+        model_package="moveit_pro_sam2"
         points_topic_name="/wrist_camera/points"
       />
     </Control>

src/hangar_sim/package.xml

@@ -27,6 +27,7 @@
   <exec_depend>realsense2_description</exec_depend>
   <exec_depend>ur_description</exec_depend>
   <exec_depend>velocity_force_controller</exec_depend>
+  <exec_depend>moveit_pro_sam2</exec_depend>
 
   <test_depend>ament_lint_auto</test_depend>
 

src/kitchen_sim/objectives/generate_graspable_object.xml

@@ -27,9 +27,9 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-        model_package="kitchen_sim"
+        model_package="moveit_pro_sam2"
       />
       <Action
         ID="PublishMask2D"

src/kitchen_sim/objectives/grasp_object_from_point.xml

@@ -26,9 +26,9 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-        model_package="kitchen_sim"
+        model_package="moveit_pro_sam2"
       />
       <Action
         ID="PublishMask2D"

src/kitchen_sim/objectives/segment_image_from_point.xml

@@ -24,9 +24,9 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-        model_package="kitchen_sim"
+        model_package="moveit_pro_sam2"
       />
       <Action
         ID="PublishMask2D"

src/kitchen_sim/objectives/segment_point_cloud_from_point.xml

@@ -25,9 +25,9 @@
         image="{image}"
         masks2d="{masks2d}"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-        model_package="kitchen_sim"
+        model_package="moveit_pro_sam2"
       />
       <Action
         ID="PublishMask2D"

src/kitchen_sim/package.xml

@@ -15,6 +15,8 @@
   <exec_depend>franka_description</exec_depend>
   <exec_depend>moveit_pro_behavior</exec_depend>
   <exec_depend>moveit_studio_agent</exec_depend>
+  <exec_depend>moveit_pro_clipseg</exec_depend>
+  <exec_depend>moveit_pro_sam2</exec_depend>
 
   <test_depend>ament_clang_format</test_depend>
   <test_depend>ament_clang_tidy</test_depend>

src/lab_sim/objectives/get_candidate_grasps_subtree.xml

@@ -67,9 +67,9 @@
             image="{image}"
             masks2d="{refined_masks}"
             pixel_coords="{center2d}"
-            decoder_model_path="models/decoder.onnx"
+            decoder_model_path="models/sam2_decoder.onnx"
             encoder_model_path="models/sam2_hiera_large_encoder.onnx"
-            model_package="lab_sim"
+            model_package="moveit_pro_sam2"
           />
         </Control>
       </Decorator>

src/lab_sim/objectives/ml_auto_grasp_object_from_clicked_point.xml

@@ -34,9 +34,9 @@
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
         masks2d="{masks2d}"
-        model_package="lab_sim"
+        model_package="moveit_pro_sam2"
         pixel_coords="{pixel_coords}"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         name="Ask the user to pick an object"
       />
       <Action

src/lab_sim/objectives/ml_find_bottles_on_table_from_image_exemplar.xml

@@ -30,7 +30,7 @@
       <SubTree ID="Take Wrist Camera Image" _collapsed="true" />
       <Action
         ID="GetMasks2DFromExemplar"
-        model_package="lab_sim"
+        model_package="moveit_pro_sam3"
         encoder_model_path="models/sam3_vision_encoder.onnx"
         decoder_model_path="models/sam3_decoder.onnx"
         geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"

src/lab_sim/objectives/ml_find_objects_on_table.xml

@@ -10,7 +10,7 @@
       <SubTree ID="Take Wrist Camera Image" _collapsed="false" />
       <Action
         ID="GetMasks2DFromExemplar"
-        model_package="lab_sim"
+        model_package="moveit_pro_sam3"
         encoder_model_path="models/sam3_vision_encoder.onnx"
         decoder_model_path="models/sam3_decoder.onnx"
         geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"

src/lab_sim/objectives/ml_segment_bottles_from_file.xml

@@ -12,7 +12,7 @@
       />
       <Action
         ID="GetMasks2DFromExemplar"
-        model_package="lab_sim"
+        model_package="moveit_pro_sam3"
         encoder_model_path="models/sam3_vision_encoder.onnx"
         decoder_model_path="models/sam3_decoder.onnx"
         geometry_encoder_model_path="models/sam3_geometry_encoder.onnx"

src/lab_sim/objectives/ml_segment_point_cloud.xml

@@ -11,8 +11,6 @@
         ID="Segment Point Cloud from Text Prompt Subtree"
         _collapsed="true"
         camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
         model_package="moveit_pro_clipseg"

src/lab_sim/objectives/ml_segment_point_cloud_from_clicked_point.xml

@@ -14,11 +14,11 @@
         ID="Segment Point Cloud from Clicked Point Subtree"
         _collapsed="true"
         camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/decoder.onnx"
+        decoder_model_path="models/sam2_decoder.onnx"
         encoder_model_path="models/sam2_hiera_large_encoder.onnx"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
-        model_package="lab_sim"
+        model_package="moveit_pro_sam2"
         points_topic_name="/wrist_camera/points"
       />
     </Control>

src/lab_sim/objectives/pick_1_pill_bottle.xml

@@ -19,8 +19,6 @@
         ID="Segment Point Cloud from Text Prompt Subtree"
         _collapsed="false"
         camera_topic_name="/wrist_camera/camera_info"
-        decoder_model_path="models/decoder.onnx"
-        encoder_model_path="models/sam2_hiera_large_encoder.onnx"
         image_topic_name="/wrist_camera/color"
         masks_visualization_topic="/masks_visualization"
         model_package="moveit_pro_clipseg"

src/lab_sim/package.xml

@@ -29,6 +29,9 @@
   <exec_depend>ur_description</exec_depend>
   <exec_depend>velocity_force_controller</exec_depend>
   <exec_depend>lab_sim_behaviors</exec_depend>
+  <exec_depend>moveit_pro_clipseg</exec_depend>
+  <exec_depend>moveit_pro_sam2</exec_depend>
+  <exec_depend>moveit_pro_sam3</exec_depend>
 
   <test_depend>ament_lint_auto</test_depend>
 

src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_point_subtree.xml

@@ -1,15 +1,15 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="UTF-8" ?>
 <root BTCPP_format="4" main_tree_to_execute="Segment Image from Point Subtree">
   <!--//////////-->
   <BehaviorTree
     ID="Segment Image from Point Subtree"
     _description="Captures an image and segments it using three user provided prompt points."
     _favorite="false"
-    decoder_model_path="models/decoder.onnx"
+    decoder_model_path="models/sam2_decoder.onnx"
     encoder_model_path="models/sam2_hiera_large_encoder.onnx"
     image_topic_name="/wrist_camera/color"
     masks_visualization_topic="/masks_visualization"
-    model_package="lab_sim"
+    model_package="moveit_pro_sam2"
     masks2d="{masks2d}"
     pixel_coords="{pixel_coords}"
   >
@@ -47,20 +47,26 @@
   </BehaviorTree>
   <TreeNodesModel>
     <SubTree ID="Segment Image from Point Subtree">
-      <input_port name="decoder_model_path" default="{decoder_model_path}" />
-      <input_port name="encoder_model_path" default="{encoder_model_path}" />
-      <input_port name="image_topic_name" default="{image_topic_name}" />
+      <MetadataFields>
+        <Metadata runnable="false" />
+        <Metadata subcategory="Perception - ML" />
+      </MetadataFields>
+      <input_port
+        name="decoder_model_path"
+        default="models/sam2_decoder.onnx"
+      />
+      <input_port
+        name="encoder_model_path"
+        default="models/sam2_hiera_large_encoder.onnx"
+      />
+      <input_port name="image_topic_name" default="/wrist_camera/color" />
       <input_port
         name="masks_visualization_topic"
-        default="{masks_visualization_topic}"
+        default="/masks_visualization"
       />
       <output_port name="masks2d" default="{masks2d}" />
-      <input_port name="model_package" default="{model_package}" />
+      <input_port name="model_package" default="moveit_pro_sam2" />
       <output_port name="pixel_coords" default="{pixel_coords}" />
-      <MetadataFields>
-        <Metadata runnable="false" />
-        <Metadata subcategory="Perception - ML" />
-      </MetadataFields>
     </SubTree>
   </TreeNodesModel>
 </root>

src/moveit_pro_ur_configs/picknik_ur_base_config/objectives/segment_image_from_text_prompt_subtree.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="UTF-8" ?>
 <root
   BTCPP_format="4"
   main_tree_to_execute="Segment Image from Text Prompt Subtree"
@@ -38,23 +38,23 @@
   </BehaviorTree>
   <TreeNodesModel>
     <SubTree ID="Segment Image from Text Prompt Subtree">
+      <MetadataFields>
+        <Metadata runnable="false" />
+        <Metadata subcategory="Perception - ML" />
+      </MetadataFields>
       <input_port name="clip_model_path" default="{clip_model_path}" />
       <input_port name="clipseg_model_path" default="{clipseg_model_path}" />
       <input_port name="erosion_size" default="{erosion_size}" />
+      <input_port name="image_topic_name" default="{image_topic_name}" />
       <input_port
         name="masks_visualization_topic"
         default="{masks_visualization_topic}"
       />
+      <output_port name="masks2d" default="{masks2d}" />
       <input_port name="model_package" default="{model_package}" />
       <input_port name="negative_prompts" default="{negative_prompts}" />
       <input_port name="prompts" default="{prompts}" />
       <input_port name="threshold" default="{threshold}" />
-      <input_port name="image_topic_name" default="{image_topic_name}" />
-      <output_port name="masks2d" default="{masks2d}" />
-      <MetadataFields>
-        <Metadata runnable="false" />
-        <Metadata subcategory="Perception - ML" />
-      </MetadataFields>
     </SubTree>
   </TreeNodesModel>
 </root>