Merge pull request #5 from logivations/zero_copy_sub

update rolling

.github/workflows/lint.yml

@@ -8,14 +8,14 @@ jobs:
     name: ament_${{ matrix.linter }}
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-focal-ros-rolling-ros-base-latest
+      image: rostooling/setup-ros-docker:ubuntu-noble-ros-rolling-ros-base-latest
     strategy:
       fail-fast: false
       matrix:
           linter: [copyright, xmllint]
     steps:
-    - uses: actions/checkout@v2
-    - uses: ros-tooling/action-ros-lint@v0.1
+    - uses: actions/checkout@v4
+    - uses: ros-tooling/action-ros-lint@0.1.4
       with:
         linter: ${{ matrix.linter }}
         distribution: rolling
@@ -39,14 +39,15 @@ jobs:
     name: ament_${{ matrix.linter }}
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-focal-ros-rolling-ros-base-latest
+      image: rostooling/setup-ros-docker:ubuntu-noble-ros-rolling-ros-base-latest
     strategy:
       fail-fast: false
       matrix:
-          linter: [cppcheck, cpplint, uncrustify]
+          # We exclude cppcheck due to https://github.com/ament/ament_lint/pull/345
+          linter: [cpplint, uncrustify]
     steps:
-    - uses: actions/checkout@v2
-    - uses: ros-tooling/action-ros-lint@v0.1
+    - uses: actions/checkout@v4
+    - uses: ros-tooling/action-ros-lint@0.1.4
       with:
         linter: ${{ matrix.linter }}
         distribution: rolling
@@ -67,17 +68,18 @@ jobs:
     name: ament_${{ matrix.linter }}
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-focal-ros-rolling-ros-base-latest
+      image: rostooling/setup-ros-docker:ubuntu-noble-ros-rolling-ros-base-latest
     strategy:
       fail-fast: false
       matrix:
-          linter: [cppcheck, cpplint, clang_format]
+          # We exclude cppcheck due to https://github.com/ament/ament_lint/pull/345
+          linter: [cpplint, clang_format]
           include:
           - linter: clang_format
             arguments: "--config rosbag2_storage_mcap/.clang-format"
     steps:
-    - uses: actions/checkout@v2
-    - uses: ros-tooling/action-ros-lint@v0.1
+    - uses: actions/checkout@v4
+    - uses: ros-tooling/action-ros-lint@0.1.4
       with:
         linter: ${{ matrix.linter }}
         arguments: ${{ matrix.arguments }}
@@ -88,14 +90,14 @@ jobs:
     name: ament_${{ matrix.linter }}
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-focal-ros-rolling-ros-base-latest
+      image: rostooling/setup-ros-docker:ubuntu-noble-ros-rolling-ros-base-latest
     strategy:
       fail-fast: false
       matrix:
           linter: [pep257, flake8]
     steps:
-    - uses: actions/checkout@v2
-    - uses: ros-tooling/action-ros-lint@v0.1
+    - uses: actions/checkout@v4
+    - uses: ros-tooling/action-ros-lint@0.1.4
       with:
         linter: ${{ matrix.linter }}
         distribution: rolling

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     container:
-      image: rostooling/setup-ros-docker:ubuntu-jammy-latest
+      image: rostooling/setup-ros-docker:ubuntu-noble-latest
     steps:
     - name: Build and run tests
       id: action-ros-ci
@@ -88,7 +88,17 @@ jobs:
         source /opt/ros/rolling/setup.sh && colcon test --mixin linters-skip --packages-select ${rosbag2_packages} --packages-skip rosbag2_performance_benchmarking --event-handlers console_cohesion+ --return-code-on-test-failure --ctest-args "-L xfail" --pytest-args "-m xfail"
       working-directory: ${{ steps.action-ros-ci.outputs.ros-workspace-directory-name }}
       shell: bash
-    - uses: actions/upload-artifact@v1
+    - name: Is regeneration of Python stubs required?
+      run: |
+        rosbag2_path=$(colcon list -p --packages-select rosbag2)/..
+        sudo apt update && sudo apt -y install mypy
+        source install/setup.sh
+        stubgen -p rosbag2_py -o ${rosbag2_path}/rosbag2_py
+        cd ${rosbag2_path}
+        git diff --exit-code
+      working-directory: ${{ steps.action-ros-ci.outputs.ros-workspace-directory-name }}
+      shell: bash
+    - uses: actions/upload-artifact@v4
       with:
         name: colcon-logs
         path: ros_ws/log

.gitignore

@@ -6,4 +6,4 @@ venv/
 **/.pytest_cache/
 __pycache__/
 *.db3-*
-
+**/.vscode/

DEVELOPING.md

@@ -11,6 +11,7 @@ colcon build --packages-up-to rosbag2
 
 Note: building Rosbag2 from source, overlaid on a debian installation of `ros-$ROS_DISTRO-rosbag2` has undefined behavior (but should work in most cases, just beware the build may find headers from the binaries instead of the local workspace.)
 
+Note: make sure to [regenerate stub files](rosbag2_py/README.md), when making changes to pybind11-related files in `rosbag2_py`.
 
 ## Executing tests
 

README.md

@@ -38,6 +38,9 @@ These verbs are available for `ros2 bag`:
 
 For up-to-date information on the available options for each, use `ros2 bag <verb> --help`.
 
+Moreover, `rosbag2_transport::Player` and `rosbag2_transport::Recorder` components can be instantiated in `rclcpp` component containers, which makes possible to use intra-process communication for greater efficiency.
+See [composition](#using-with-composition) section for details.
+
 ### Recording data
 
 In order to record all topics currently available in the system:
@@ -91,6 +94,8 @@ Currently, the only `compression-format` available is `zstd`. Both the mode and
 
 It is recommended to use this feature with the splitting options.
 
+**Note**: Some storage plugins may have their own compression methods, which are separate from the rosbag2 compression specified by the CLI options `--compression-mode` and `--compression-format`. Notably, the MCAP file format offered by the `rosbag2_storage_mcap` storage plugin supports compression in a way that produces files that are still indexable (whereas using the rosbag2 compression will not). To utilize storage plugin specific compression or other options, see [Recording with a storage configuration](#Recording-with-a-storage-configuration).
+
 #### Recording with a storage configuration
 
 Storage configuration can be specified in a YAML file passed through the `--storage-config-file` option.
@@ -221,11 +226,17 @@ output_bags:
   max_bagfile_duration: 0
   storage_preset_profile: ""
   storage_config_uri: ""
-  all: false
+  all_topics: false
   topics: []
+  topic_types: []
+  all_services: false
+  services: []
   rmw_serialization_format: ""  # defaults to using the format of the input topic
   regex: ""
-  exclude: ""
+  exclude_regex: ""
+  exclude_topics: []
+  exclude_topic_types: []
+  exclude_services: []
   compression_mode: ""
   compression_format: ""
   compression_queue_size: 1
@@ -242,7 +253,8 @@ $ ros2 bag convert -i bag1 -i bag2 -o out.yaml
 # out.yaml
 output_bags:
 - uri: merged_bag
-  all: true
+  all_topics: true
+  all_services: true
 ```
 
 Example split:
@@ -266,7 +278,8 @@ $ ros2 bag convert -i bag1 -o out.yaml
 # out.yaml
 output_bags:
 - uri: compressed
-  all: true
+  all_topics: true
+  all_services: true
   compression_mode: file
   compression_format: zstd
 ```
@@ -343,6 +356,122 @@ For example, if we named the above XML launch script, `record_all.launch.xml`:
 $ ros2 launch record_all.launch.xml
 ```
 
+You can also invoke the `play` and `record` functionalities provided by `rosbag2_transport` package as nodes.
+The advantage to use this invocation strategy is that the Python layer handling the `ros2 bag` CLI is completely skipped.
+
+```python
+import launch
+
+def generate_launch_description():
+    return launch.LaunchDescription([
+        launch.actions.Node(
+            package='rosbag2_transport',
+            executable='player',
+            name='player',
+            output="screen",
+            parameters=["/path/to/params.yaml"],
+        )
+    ])
+```
+
+## Using with composition
+
+Play and record are fundamental tasks of `rosbag2`. However, playing or recording data at high rates may have limitations (e.g. spurious packet drops) due to one of the following:
+- low network bandwith
+- high CPU load
+- slow mass memory
+- ROS 2 middleware serialization/deserialization delays & overhead
+
+ROS 2 C++ nodes can benefit from intra-process communication to partially or completely bypass network transport of messages between two nodes.
+
+Multiple _components_ can be _composed_, either [statically](https://docs.ros.org/en/rolling/Tutorials/Intermediate/Composition.html#compile-time-composition-using-ros-services) or [dynamically](https://docs.ros.org/en/rolling/Tutorials/Intermediate/Composition.html#run-time-composition-using-ros-services-with-a-publisher-and-subscriber): all the composed component will share the same address space because they will be loaded in a single process.
+
+A prerequirement is for each C++ node to be [_composable_](https://docs.ros.org/en/rolling/Concepts/Intermediate/About-Composition.html?highlight=composition) and to follow the [guidelines](https://docs.ros.org/en/rolling/Tutorials/Demos/Intra-Process-Communication.html?highlight=intra) for efficient publishing & subscription.
+
+With the above requirements met, the user can:
+- compose multiple nodes together
+- explicitly enable intra-process communication
+
+Whenever a publisher and a subscriber on the same topic belong to the same _composed_ process, and intra-process is enabled for both, `rclcpp` completely bypasses RMW layer and below transport layer (i.e. DDS). Instead, messages are shared via process memory and *potentially* never copied. Some exception hold, so please have a look to the [IPC guidelines](https://docs.ros.org/en/rolling/Tutorials/Demos/Intra-Process-Communication.html?highlight=intra).
+
+Here is an example of Python launchfile composition. Notice that composable container components do not expect YAML files to be directly passed to them: parameters have to be "dumped" out from the YAML file (if you have one). A suggestion of possible implementation is offered as a starting point.
+
+```python
+import launch
+import launch_ros
+import yaml
+
+'''
+Used to load parameters for composable nodes from a standard param file
+'''
+def dump_params(param_file_path, node_name):
+    with open(param_file_path, 'r') as file:
+        return [yaml.safe_load(file)[node_name]['ros__parameters']]
+
+def generate_launch_description():
+    return launch.LaunchDescription([
+        launch.actions.ComposableNodeContainer(
+            name='composable_container',
+            package='rclcpp_components',
+            executable='component_container',
+            composable_node_descriptions=[
+                launch_ros.descriptions.ComposableNode(
+                    package='rosbag2_transport',
+                    plugin='rosbag2_transport::Player',
+                    name='player',
+                    parameters=dump_params("/path/to/params.yaml", "player"),
+                    extra_arguments=[{'use_intra_process_comms': True}]
+                ),
+                # your other components here
+            ]
+        )
+    ])
+}
+```
+
+Here's an example YAML configuration for both composable player and recorder:
+```yaml
+recorder:
+  ros__parameters:
+    use_sim_time: false
+    record:
+      all: true
+      is_discovery_disabled: false
+      topic_polling_interval:
+        sec: 0
+        nsec: 10000000
+      include_hidden_topics: true
+      ignore_leaf_topics: false
+      start_paused: false
+
+    storage:
+      uri: "/path/to/destination/folder"
+      storage_id: "sqlite3"
+      max_cache_size: 20000000
+```
+and
+```yaml
+player:
+  ros__parameters:
+    play:
+      read_ahead_queue_size: 1000
+      node_prefix: ""
+      rate: 1.0
+      loop: false
+      # Negative timestamps will make the playback to not stop.  
+      playback_duration:
+        sec: -1
+        nsec: 00000000
+      start_paused: false
+
+    storage:
+      uri: "path/to/rosbag/file"
+      storage_id: "mcap"
+      storage_config_uri: ""
+```
+
+For a full list of available parameters, you can refer to [`player`](rosbag2_transport/test/resources/player_node_params.yaml) and [`recorder`](rosbag2_transport/test/resources/recorder_node_params.yaml) configurations from the `test` folder of `rosbag2_transport`.
+
 ## Storage format plugin architecture
 
 Looking at the output of the `ros2 bag info` command, we can see a field `Storage id:`.

docs/design/rosbag2_record_replay_service.md

@@ -78,7 +78,7 @@ A new parameter is added.
 
 - `-v` or `--verbose`
 
-    This parameter only affects the service event topic. if `-v` or `--verbose` is set, the info command shows the number of requests and responses for each service based on the service event. Otherwise, only show the number of service events. Note that parsing the number of request and response need spent time. The duration of the parsing is related to the number of recorded service events.
+    If `-v` or `--verbose` is set, the info command shows two additional pieces of information: the size contribution of topics and services, and the number of requests and responses for each service based on the service event. Otherwise, only show the message count of the topics and the general number of service events. Note that parsing the messages need spent time. The duration of the parsing is related to the number of recorderd topic messages and service events.
 
 
 Without `-v` or `--verbose` parameter, info command shows as below example.
@@ -111,31 +111,44 @@ Duration:          15.59s
 Start:             May 19 2023 13:22:25.340 (1684473745.340)
 End:               May 19 2023 13:22:40.400 (1684473760.400)
 Messages:          60
-Topic information: Topic: /chatter | Type: std_msgs/msg/String | Count: 16 | Serialization Format: cdr
-                   Topic: /events/write_split | Type: rosbag2_interfaces/msg/WriteSplitEvent | Count: 0 | Serialization Format: cdr
-                   Topic: /parameter_events | Type: rcl_interfaces/msg/ParameterEvent | Count: 0 | Serialization Format: cdr
-                   Topic: /rosout | Type: rcl_interfaces/msg/Log | Count: 44 | Serialization Format: cdr
+Topic information: Topic: /chatter | Type: std_msgs/msg/String | Count: 16 | Size Contribution XX xxB | Serialization Format: cdr
+                   Topic: /events/write_split | Type: rosbag2_interfaces/msg/WriteSplitEvent | Count: 0 | Size Contribution 0 B | Serialization Format: cdr
+                   Topic: /parameter_events | Type: rcl_interfaces/msg/ParameterEvent | Count: 0 | Size Contribution 0 B | Serialization Format: cdr
+                   Topic: /rosout | Type: rcl_interfaces/msg/Log | Count: 44 | Size Contribution XX xxB | Serialization Format: cdr
 
 >>>> The below is new added items for service <<<<<<<
 Service : XX  <== The number of service
-Service information: Service: /xxx/xxx | Type: xxx/xxx/xxx | Request Count: XX | Response Count: XX | Serialization Format: XX
-                     Service: /xxx/xxx | Type: xxx/xxx/xxx | Request Count: XX | Response Count: XX | Serialization Format: XX
+Service information: Service: /xxx/xxx | Type: xxx/xxx/xxx | Request Count: XX | Response Count: XX | Size Contribution XX xxB | Serialization Format: XX
+                     Service: /xxx/xxx | Type: xxx/xxx/xxx | Request Count: XX | Response Count: XX | Size Contribution XX xxB | Serialization Format: XX
 ```
 
 ### Expand the 'play' command
 
-Add 3 parameters.
-- `--services ServiceName1 [[ServiceName2 ...]`
+Added or changed parameters:
+- `--publish-service-requests`
+
+    Send service requests based on recorded service introspection messages. By default, recorded service introspection messages will be published by service event topic.
+
+- `--service-requests-source {service_introspection, client_introspection}`
+
+    Determine the source of the service request to be replayed. By default, the service request is from recorded service introspection message.
+    This option only makes sense if the `--publish-service-requests` option is set.
+
+- `--services ServiceName1 [ServiceName2 ...]`
 
     Determine which service's requests are played.
 
-- `--exclude-topics`
+- `--exclude-topics topic1 [topic2 ...]`
 
-    Rename from `--exclude`. Exclude topics containing provided regular expression.
+    List of topics not being played.
 
-- `--exclude-services`
+- `--exclude-services ServiceName1 [ServiceName2 ...]`
+
+    List of services not being played.
+
+- `--exclude-regex REGEX`
 
-    Exclude services containing provided regular expression.
+    Rename from `--exclude`. Exclude topics and services containing provided regular expression.
 
 `-e REGEX, --regex REGEX` affects both topics and services.