diff --git a/AGENTS.md b/AGENTS.md index 7d7dc68171..b7fcc80411 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,7 @@ pre-commit install --hook-type commit-msg ``` - **REUSE:** files covered by the REUSE hook need **`SPDX-FileCopyrightText`** and **`SPDX-License-Identifier`** in the form the repo already uses (for example the HTML comment block at the top of `README.md` also applies to **`AGENTS.md`** and similar docs). +- In a filesystem-restricted agent sandbox where `~/.cache/pre-commit` is not writable, set `PRE_COMMIT_HOME` to a writable temporary directory and still run the complete hook set. - **C++ formatting is enforced by CI, not pre-commit.** The hook set runs `ruff` for Python but does **not** run `clang-format`; CI (`build-ubuntu.yml`) installs **`clang-format-14`** and rejects unformatted C++ as `-Wclang-format-violations`. Before pushing, format touched C++ with the system `clang-format` (match CI's version 14) and verify: ```bash diff --git a/examples/schemaio/CMakeLists.txt b/examples/schemaio/CMakeLists.txt index 938f8e441f..5d44d82eb4 100644 --- a/examples/schemaio/CMakeLists.txt +++ b/examples/schemaio/CMakeLists.txt @@ -6,14 +6,13 @@ cmake_minimum_required(VERSION 3.20) find_package(Threads REQUIRED) # Create pusher executable -# Note: Examples link to oxr_core for OpenXR session creation, but pusherio itself doesn't add_executable(pedal_pusher pedal_pusher.cpp ) target_link_libraries(pedal_pusher PRIVATE + Teleop::plugin_utils pusherio::pusherio - oxr::oxr_core isaacteleop_schema ${CMAKE_DL_LIBS} Threads::Threads diff --git a/examples/schemaio/pedal_pusher.cpp b/examples/schemaio/pedal_pusher.cpp index b129dae2ea..277ffb798a 100644 --- a/examples/schemaio/pedal_pusher.cpp +++ b/examples/schemaio/pedal_pusher.cpp @@ -2,11 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 /*! - * @brief Demo application that pushes serialized FlatBuffer Generic3AxisPedalOutput data into the OpenXR runtime. + * @brief Demo application that pushes serialized FlatBuffer Generic3AxisPedalOutput data. * * This application demonstrates using the SchemaPusher class to push Generic3AxisPedalOutput FlatBuffer - * messages. The application creates the OpenXR session with required extensions and passes - * the handles to the pusherio library. + * messages. Plugin-facing examples obtain channels from IPluginSession; only the concrete session adapter + * owns transport-specific handles. * * Note: Both pusher and reader agree on the schema (Generic3AxisPedalOutput from pedals.fbs), so the schema * does not need to be sent over the wire. @@ -15,7 +15,8 @@ #include "common_utils.hpp" #include -#include +#include +#include #include #include @@ -25,24 +26,19 @@ #include #include #include +#include using namespace schemaio_example; /*! * @brief Generic3AxisPedalOutput-specific pusher that serializes and pushes foot pedal messages. * - * Uses composition with SchemaPusher to handle the OpenXR tensor pushing. + * Uses composition with the transport-independent SchemaPusher. */ class Generic3AxisPedalPusher { public: - Generic3AxisPedalPusher(const core::OpenXRSessionHandles& handles, const std::string& collection_id) - : m_pusher(handles, - core::SchemaPusherConfig{ .collection_id = collection_id, - .max_flatbuffer_size = MAX_FLATBUFFER_SIZE, - .tensor_identifier = "generic_3axis_pedal", - .localized_name = "Generic 3-Axis Pedal Pusher Demo", - .app_name = "Generic3AxisPedalPusher" }) + explicit Generic3AxisPedalPusher(std::unique_ptr channel) : m_pusher(std::move(channel)) { } @@ -72,21 +68,21 @@ try { std::cout << "Schema Pusher (collection: " << COLLECTION_ID << ")" << std::endl; - // Step 1: Create OpenXR session with required extensions for pushing tensor data - std::cout << "[Step 1] Creating OpenXR session with tensor push extensions..." << std::endl; + // Step 1: Select the concrete transport at the composition root. + std::cout << "[Step 1] Creating OpenXR plugin session..." << std::endl; - auto required_extensions = core::SchemaPusher::get_required_extensions(); + core::PluginSessionHandle session = std::make_shared( + "SchemaPusher", core::PluginSessionRequirements{ .schema_push = true }); - auto oxr_session = std::make_shared("SchemaPusher", required_extensions); - - std::cout << " OpenXR session created" << std::endl; - - // Step 2: Create the pusher with the session handles + // Step 2: Ask the abstract session for a channel, then give it to the typed pusher. std::cout << "[Step 2] Creating Generic3AxisPedalPusher..." << std::endl; - std::unique_ptr pusher; - auto handles = oxr_session->get_handles(); - pusher = std::make_unique(handles, COLLECTION_ID); + auto pusher = std::make_unique(session->create_schema_push_channel( + core::SchemaPusherConfig{ .collection_id = COLLECTION_ID, + .max_flatbuffer_size = MAX_FLATBUFFER_SIZE, + .tensor_identifier = "generic_3axis_pedal", + .localized_name = "Generic 3-Axis Pedal Pusher Demo", + .app_name = "Generic3AxisPedalPusher" })); // Step 3: Push samples std::cout << "[Step 3] Pushing samples..." << std::endl; diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index 36514ba9ca..a7b58ec44a 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -12,6 +12,10 @@ To see **all** `AGENTS.md` files in the IsaacTeleop repo, use the **`find` comma If work under **`src/core/`** went wrong—**user** correction, **pre-commit/CI** failure, or **repeated** same-class mistakes—you **must** follow the repo root **[`AGENTS.md`](../../AGENTS.md)** **Mandatory learning loop**: distill a short rule and **update** the **nearest** relevant `AGENTS.md` (this file or a package file) or **source comments** in the same session (including **delta vs `main`** scope). - Async retargeting pacing behavior belongs on the pacing config objects; keep the worker focused on scheduling mechanics and avoid adding concrete pacing-mode or subclass branches there. +- When transport-neutralizing an existing data path, preserve its operation-specific facade and abstract the session-created channel beneath it; add named-port discovery only when a real dynamic-routing requirement needs it. +- Keep session capability declarations transport-neutral in `PluginSessionRequirements`; concrete sessions translate them into backend prerequisites and reject undeclared channel creation. +- Plugin implementations depend on `IPluginSession`, the multiplexed `IPluginPullChannel`, and operation-specific push channels/sources; only composition roots instantiate concrete session adapters, and runtime handles stay inside adapter-owned implementations. +- Reusing OpenXR value structs and enums in a channel contract is acceptable when minimizing migration; keep runtime handles, function pointers, calls, and runtime-clock timestamps inside the OpenXR implementation. - Prefer coarse-grained async boundaries around an existing synchronous step before splitting DeviceIO/source polling away from graph execution; split internals only when a measured correctness or performance need justifies the extra thread-safety surface. - In pipelined `TeleopSession`, `last_context` follows the returned completed frame; reset/control-transition events travel with that frame and must not force exact-current-frame waits. Use sync mode for exact current-frame behavior. - Keep async retargeting comments short and local to invariants; user-facing pacing tuning guidance belongs in docs rather than long code docstrings. diff --git a/src/core/codegen/templates/fragments/live_factory_push.template b/src/core/codegen/templates/fragments/live_factory_push.template index 4c19c6c9a7..adb703130b 100644 --- a/src/core/codegen/templates/fragments/live_factory_push.template +++ b/src/core/codegen/templates/fragments/live_factory_push.template @@ -1,4 +1,5 @@ std::unique_ptr<@IFACE@> LiveDeviceIOFactory::create_@NAME@_tracker_impl(const @CLASS@* tracker) { - return std::make_unique<@LIVE_IMPL@>(handles_, tracker); + return std::make_unique<@LIVE_IMPL@>( + make_openxr_schema_push_channel(handles_, make_schema_push_config(tracker))); } diff --git a/src/core/codegen/templates/push/live.cpp.template b/src/core/codegen/templates/push/live.cpp.template index 0f43d54d07..50348b35ea 100644 --- a/src/core/codegen/templates/push/live.cpp.template +++ b/src/core/codegen/templates/push/live.cpp.template @@ -7,23 +7,12 @@ #include #include +#include namespace core { -namespace -{ -SchemaPusherConfig make_@NAME@_push_config(const @CLASS@* tracker) -{ - SchemaPusherConfig cfg; - cfg.collection_id = tracker->collection_id(); - cfg.max_flatbuffer_size = tracker->max_payload_size(); - cfg.tensor_identifier = tracker->tensor_identifier(); - cfg.localized_name = tracker->tensor_identifier(); - return cfg; -} -} // namespace -@LIVE_IMPL@::@LIVE_IMPL@(const OpenXRSessionHandles& handles, const @CLASS@* tracker) - : pusher_(handles, make_@NAME@_push_config(tracker)) +@LIVE_IMPL@::@LIVE_IMPL@(std::unique_ptr channel) + : pusher_(std::move(channel)) { // No-op: members set in the initializer list. } diff --git a/src/core/codegen/templates/push/live.hpp.template b/src/core/codegen/templates/push/live.hpp.template index 5a2fa0053a..54f927eae1 100644 --- a/src/core/codegen/templates/push/live.hpp.template +++ b/src/core/codegen/templates/push/live.hpp.template @@ -5,11 +5,12 @@ #include #include -#include +#include #include #include #include +#include #include namespace core @@ -19,9 +20,9 @@ class @LIVE_IMPL@ : public @IFACE@ public: static std::vector required_extensions() { - return SchemaPusher::get_required_extensions(); + return OpenXRSchemaPushChannel::get_required_extensions(); } - @LIVE_IMPL@(const OpenXRSessionHandles& handles, const @CLASS@* tracker); + explicit @LIVE_IMPL@(std::unique_ptr channel); void update(int64_t monotonic_time_ns) override; void push(const Serialized<@FB_TABLE@>& data) const override; diff --git a/src/core/live_trackers/AGENTS.md b/src/core/live_trackers/AGENTS.md index 8e6b1f646a..2be5d7a988 100644 --- a/src/core/live_trackers/AGENTS.md +++ b/src/core/live_trackers/AGENTS.md @@ -9,6 +9,8 @@ SPDX-License-Identifier: Apache-2.0 ## Time and OpenXR +- Keep push-channel selection in `LiveDeviceIOFactory`; push tracker impls receive an + `ISchemaPushChannel` and do not construct OpenXR channels from session handles. - Store **`last_update_time_` as `int64_t`** (monotonic ns), not **`XrTime`**. - **Once per `update` call:** `const XrTime xr_time = time_converter_.convert_monotonic_ns_to_xrtime(monotonic_time_ns);` then use **`xr_time`** for every **`xrLocate*`** / hand / body call **and** for MCAP (see below). **Do not** call **`convert_monotonic_ns_to_xrtime`** again in the MCAP block. - **Full-body limp mode:** if the body tracker handle is null and you **return early**, **do not** compute **`xr_time`** first—only convert after you know you will call OpenXR. diff --git a/src/core/live_trackers/cpp/live_deviceio_factory.cpp b/src/core/live_trackers/cpp/live_deviceio_factory.cpp index c11104e395..52a71fd1a2 100644 --- a/src/core/live_trackers/cpp/live_deviceio_factory.cpp +++ b/src/core/live_trackers/cpp/live_deviceio_factory.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -56,6 +57,15 @@ bool try_add_extensions(const ITracker& tracker, std::set& out) return true; } +template +SchemaPusherConfig make_schema_push_config(const TrackerT* tracker) +{ + return SchemaPusherConfig{ .collection_id = tracker->collection_id(), + .max_flatbuffer_size = tracker->max_payload_size(), + .tensor_identifier = tracker->tensor_identifier(), + .localized_name = tracker->tensor_identifier() }; +} + std::unique_ptr try_create_head_impl(LiveDeviceIOFactory& factory, const ITracker& tracker) { auto* typed = dynamic_cast(&tracker); @@ -455,7 +465,8 @@ std::unique_ptr LiveDeviceIOFactory::create_full_body_trac std::unique_ptr LiveDeviceIOFactory::create_tensor_push_tracker_impl(const TensorPushTracker* tracker) { - return std::make_unique(handles_, tracker); + return std::make_unique( + make_openxr_schema_push_channel(handles_, make_schema_push_config(tracker))); } std::unique_ptr LiveDeviceIOFactory::create_haptic_command_reader_tracker_impl( diff --git a/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.cpp b/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.cpp index 642454881b..afdc14da2f 100644 --- a/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.cpp +++ b/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.cpp @@ -5,26 +5,13 @@ #include -namespace core -{ - -namespace -{ +#include -SchemaPusherConfig make_tensor_push_config(const TensorPushTracker* tracker) +namespace core { - SchemaPusherConfig cfg; - cfg.collection_id = tracker->collection_id(); - cfg.max_flatbuffer_size = tracker->max_payload_size(); - cfg.tensor_identifier = tracker->tensor_identifier(); - cfg.localized_name = tracker->tensor_identifier(); - return cfg; -} - -} // namespace -LiveTensorPushTrackerImpl::LiveTensorPushTrackerImpl(const OpenXRSessionHandles& handles, const TensorPushTracker* tracker) - : pusher_(handles, make_tensor_push_config(tracker)) +LiveTensorPushTrackerImpl::LiveTensorPushTrackerImpl(std::unique_ptr channel) + : pusher_(std::move(channel)) { } diff --git a/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.hpp b/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.hpp index 368678d30d..c9a408b234 100644 --- a/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.hpp +++ b/src/core/live_trackers/cpp/live_tensor_push_tracker_impl.hpp @@ -4,26 +4,26 @@ #pragma once #include -#include -#include +#include #include #include +#include #include namespace core { -// Wraps core::SchemaPusher; owns the XR_NVX1_push_tensor handle. +// Wraps core::SchemaPusher and owns its transport channel. class LiveTensorPushTrackerImpl : public ITensorPushTrackerImpl { public: static std::vector required_extensions() { - return SchemaPusher::get_required_extensions(); + return OpenXRSchemaPushChannel::get_required_extensions(); } - LiveTensorPushTrackerImpl(const OpenXRSessionHandles& handles, const TensorPushTracker* tracker); + explicit LiveTensorPushTrackerImpl(std::unique_ptr channel); LiveTensorPushTrackerImpl(const LiveTensorPushTrackerImpl&) = delete; LiveTensorPushTrackerImpl& operator=(const LiveTensorPushTrackerImpl&) = delete; diff --git a/src/core/pusherio/cpp/CMakeLists.txt b/src/core/pusherio/cpp/CMakeLists.txt index 021f83be5c..a049987903 100644 --- a/src/core/pusherio/cpp/CMakeLists.txt +++ b/src/core/pusherio/cpp/CMakeLists.txt @@ -5,6 +5,13 @@ cmake_minimum_required(VERSION 3.20) # PusherIO library (static; consumed in-tree via pusherio::pusherio) add_library(pusherio STATIC + hand_tracking_pusher.cpp + openxr_schema_push_channel.cpp + inc/pusherio/hand_tracking_push_channel.hpp + inc/pusherio/hand_tracking_pusher.hpp + inc/pusherio/openxr_schema_push_channel.hpp + inc/pusherio/plugin_session.hpp + inc/pusherio/wrist_tracking_source.hpp schema_pusher.cpp inc/pusherio/schema_pusher.hpp ) @@ -16,6 +23,8 @@ target_include_directories(pusherio target_link_libraries(pusherio PUBLIC + deviceio::deviceio_base + # oxr_utils is header-only and provides OpenXRSessionHandles struct # It also brings in OpenXR headers for base types without linking to OpenXR loader oxr::oxr_utils diff --git a/src/core/pusherio/cpp/hand_tracking_pusher.cpp b/src/core/pusherio/cpp/hand_tracking_pusher.cpp new file mode 100644 index 0000000000..550612328a --- /dev/null +++ b/src/core/pusherio/cpp/hand_tracking_pusher.cpp @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/pusherio/hand_tracking_pusher.hpp" + +#include +#include + +namespace core +{ + +HandTrackingPusher::HandTrackingPusher(std::unique_ptr channel) : channel_(std::move(channel)) +{ + if (!channel_) + { + throw std::invalid_argument("HandTrackingPusher requires a channel"); + } +} + +HandTrackingPusher::~HandTrackingPusher() = default; + +void HandTrackingPusher::push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns) +{ + if (!joint_locations) + { + throw std::invalid_argument("HandTrackingPusher requires joint locations"); + } + + channel_->push(joint_locations, sample_time_local_common_clock_ns); +} + +} // namespace core diff --git a/src/core/pusherio/cpp/inc/pusherio/hand_tracking_push_channel.hpp b/src/core/pusherio/cpp/inc/pusherio/hand_tracking_push_channel.hpp new file mode 100644 index 0000000000..a12cee8498 --- /dev/null +++ b/src/core/pusherio/cpp/inc/pusherio/hand_tracking_push_channel.hpp @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include + +namespace core +{ + +/*! + * @brief Transport-owned channel for publishing one hand's tracking data. + * + * OpenXR value types define the established hand data shape. Runtime handles, + * function pointers, and calls remain private to concrete channel implementations. + */ +class IHandTrackingPushChannel +{ +public: + // Orderly destruction closes the logical hand stream and makes it inactive + // at the receiver. Remote implementations must also expire active state + // after an unexpected transport disconnect. + virtual ~IHandTrackingPushChannel() = default; + + // joint_locations contains XR_HAND_JOINT_COUNT_EXT entries and is borrowed + // for this call only. Asynchronous transports must copy it before returning. + virtual void push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns) = 0; +}; + +} // namespace core diff --git a/src/core/pusherio/cpp/inc/pusherio/hand_tracking_pusher.hpp b/src/core/pusherio/cpp/inc/pusherio/hand_tracking_pusher.hpp new file mode 100644 index 0000000000..4c91384e74 --- /dev/null +++ b/src/core/pusherio/cpp/inc/pusherio/hand_tracking_pusher.hpp @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "hand_tracking_push_channel.hpp" + +#include +#include + +namespace core +{ + +/*! + * @brief Publishes one hand's joint locations through a transport-owned channel. + * + * This is the plugin-facing facade. OpenXR and remote transports provide the + * channel implementation while the pusher preserves the existing hand data + * contract used by plugins. + */ +class HandTrackingPusher +{ +public: + explicit HandTrackingPusher(std::unique_ptr channel); + ~HandTrackingPusher(); + + HandTrackingPusher(const HandTrackingPusher&) = delete; + HandTrackingPusher& operator=(const HandTrackingPusher&) = delete; + HandTrackingPusher(HandTrackingPusher&&) = delete; + HandTrackingPusher& operator=(HandTrackingPusher&&) = delete; + + // joint_locations contains XR_HAND_JOINT_COUNT_EXT entries and is borrowed + // for this call only. + void push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns); + +private: + std::unique_ptr channel_; +}; + +} // namespace core diff --git a/src/core/pusherio/cpp/inc/pusherio/openxr_schema_push_channel.hpp b/src/core/pusherio/cpp/inc/pusherio/openxr_schema_push_channel.hpp new file mode 100644 index 0000000000..ab78f174c6 --- /dev/null +++ b/src/core/pusherio/cpp/inc/pusherio/openxr_schema_push_channel.hpp @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "schema_pusher.hpp" + +#include + +#include +#include +#include + +namespace core +{ + +/*! + * @brief OpenXR implementation of one schema-push channel. + * + * The caller owns the parent OpenXR session and must keep it alive until this + * channel has been destroyed. + */ +class OpenXRSchemaPushChannel final : public ISchemaPushChannel +{ +public: + OpenXRSchemaPushChannel(const OpenXRSessionHandles& handles, SchemaPusherConfig config); + ~OpenXRSchemaPushChannel() override; + + OpenXRSchemaPushChannel(const OpenXRSchemaPushChannel&) = delete; + OpenXRSchemaPushChannel& operator=(const OpenXRSchemaPushChannel&) = delete; + OpenXRSchemaPushChannel(OpenXRSchemaPushChannel&&) = delete; + OpenXRSchemaPushChannel& operator=(OpenXRSchemaPushChannel&&) = delete; + + static std::vector get_required_extensions(); + + const SchemaPusherConfig& config() const override; + void push_buffer(const uint8_t* buffer, + size_t size, + int64_t sample_time_local_common_clock_ns, + int64_t sample_time_raw_device_clock_ns) override; + +private: + class Impl; + std::unique_ptr impl_; +}; + +std::unique_ptr make_openxr_schema_push_channel(const OpenXRSessionHandles& handles, + SchemaPusherConfig config); + +} // namespace core diff --git a/src/core/pusherio/cpp/inc/pusherio/plugin_session.hpp b/src/core/pusherio/cpp/inc/pusherio/plugin_session.hpp new file mode 100644 index 0000000000..2f110c6439 --- /dev/null +++ b/src/core/pusherio/cpp/inc/pusherio/plugin_session.hpp @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "hand_tracking_push_channel.hpp" +#include "schema_pusher.hpp" +#include "wrist_tracking_source.hpp" + +#include + +#include + +namespace core +{ + +/*! + * @brief Transport-neutral capabilities a plugin may use during a session. + * + * Concrete sessions translate these capabilities into transport prerequisites + * during construction and reject channel types that were not declared. + */ +struct PluginSessionRequirements +{ + bool schema_push = false; + bool hand_tracking_push = false; + bool wrist_tracking_pull = false; +}; + +/*! + * @brief Multiplexed transport-neutral pull channel for typed DeviceIO trackers and optional sources. + * + * One update establishes the snapshot boundary for every tracker on the channel. + * The pull channel must outlive every source created from it. + */ +class IPluginPullChannel : public ITrackerSession +{ +public: + virtual ~IPluginPullChannel() = default; + + virtual void update() = 0; + + //! Returns null when the requested source mode is unavailable after capability negotiation. + virtual std::unique_ptr create_wrist_tracking_source(const WristTrackingSourceConfig& config) = 0; +}; + +/*! + * @brief Session abstraction that creates operation-specific plugin channels. + * + * The caller must keep the session alive until all pull and push channels + * created from it have been destroyed. + */ +class IPluginSession +{ +public: + virtual ~IPluginSession() = default; + + virtual std::unique_ptr create_pull_channel() = 0; + virtual std::unique_ptr create_schema_push_channel(const SchemaPusherConfig& config) = 0; + virtual std::unique_ptr create_hand_tracking_push_channel(XrHandEXT hand) = 0; +}; + +using PluginSessionHandle = std::shared_ptr; + +} // namespace core diff --git a/src/core/pusherio/cpp/inc/pusherio/schema_pusher.hpp b/src/core/pusherio/cpp/inc/pusherio/schema_pusher.hpp index e18be0e056..518005f9ae 100644 --- a/src/core/pusherio/cpp/inc/pusherio/schema_pusher.hpp +++ b/src/core/pusherio/cpp/inc/pusherio/schema_pusher.hpp @@ -3,15 +3,10 @@ #pragma once -#include -#include - -#include -#include #include #include +#include #include -#include namespace core { @@ -19,8 +14,8 @@ namespace core /*! * @brief Configuration for SchemaPusher. * - * This struct contains all parameters needed to set up a tensor collection - * for pushing FlatBuffer schema data via OpenXR extensions. + * This struct contains all parameters needed to set up a channel for pushing + * FlatBuffer schema data. Local OpenXR sessions map it to a tensor collection. */ struct SchemaPusherConfig { @@ -40,32 +35,43 @@ struct SchemaPusherConfig //! Human-readable description for debugging and runtime display. std::string localized_name; - //! OpenXR application name. If empty, defaults to "Pusher" or "Reader". + //! Optional producer label for backend diagnostics. std::string app_name = ""; }; - /*! - * @brief Pushes FlatBuffer schema data via OpenXR tensor extensions. + * @brief Transport-owned channel for one configured schema stream. * - * This class uses externally-provided OpenXR session handles and handles tensor collection - * creation and sample pushing logic. Use composition to wrap this class with typed push methods. + * OpenXR and remote transports implement this operation-specific interface. + * Implementations own any child resources needed to keep the channel valid. + */ +class ISchemaPushChannel +{ +public: + virtual ~ISchemaPushChannel() = default; + + virtual const SchemaPusherConfig& config() const = 0; + + // buffer is borrowed for this call only; asynchronous transports must copy it before returning. + virtual void push_buffer(const uint8_t* buffer, + size_t size, + int64_t sample_time_local_common_clock_ns, + int64_t sample_time_raw_device_clock_ns) = 0; +}; + +/*! + * @brief Pushes FlatBuffer schema data through a transport-owned channel. * - * The caller is responsible for creating the OpenXR session with the required extensions - * (XR_NVX1_PUSH_TENSOR_EXTENSION_NAME, XR_NVX1_TENSOR_DATA_EXTENSION_NAME) and passing - * the handles to this class. + * This is the plugin-facing facade. It preserves the existing opaque FlatBuffer + * contract while allowing the owning session to select the local OpenXR or remote + * transport implementation. Use composition to add typed push methods where useful. * * Example usage with composition: * @code * class HeadPosePusher { * public: - * HeadPosePusher(const OpenXRSessionHandles& handles, const std::string& collection_id) - * : m_pusher(handles, { - * .collection_id = collection_id, - * .max_flatbuffer_size = 256, - * .tensor_identifier = "head_pose", - * .localized_name = "HeadPose Data" - * }) {} + * explicit HeadPosePusher(std::unique_ptr channel) + * : m_pusher(std::move(channel)) {} * * void push(const HeadPoseT& data, * int64_t sample_time_local_common_clock_ns, @@ -87,30 +93,15 @@ class SchemaPusher { public: /*! - * @brief Get required OpenXR extensions for pushing tensor data. + * @brief Constructs a pusher from an already-created transport channel. * - * Includes platform-specific time conversion extension. - */ - static std::vector get_required_extensions() - { - std::vector required_extensions = { "XR_NVX1_push_tensor", "XR_NVX1_tensor_data" }; - for (const auto& ext : XrTimeConverter::get_required_extensions()) - { - required_extensions.push_back(ext); - } - return required_extensions; - } - - /*! - * @brief Constructs the pusher and initializes the OpenXR tensor collection. - * @param handles OpenXR session handles (caller must create session with required extensions). - * @param config Configuration for the tensor collection. - * @throws std::runtime_error if initialization fails. + * The owning plugin session selects and creates the concrete channel before + * constructing this transport-independent facade. */ - SchemaPusher(const OpenXRSessionHandles& handles, SchemaPusherConfig config); + explicit SchemaPusher(std::unique_ptr channel); /*! - * @brief Destroys the pusher and cleans up OpenXR resources. + * @brief Destroys the pusher and its transport-owned channel. */ ~SchemaPusher(); @@ -123,15 +114,17 @@ class SchemaPusher /*! * @brief Push raw serialized FlatBuffer data with timestamps. * - * The buffer will be padded to max_flatbuffer_size if smaller. + * The channel receives the logical serialized size. The local OpenXR channel + * pads it to max_flatbuffer_size to satisfy the fixed-size tensor contract. * * Both timestamp parameters must be in nanoseconds. The local common clock is * system monotonic time (CLOCK_MONOTONIC on Linux, QueryPerformanceCounter on * Windows) — values are comparable across all sources on the same machine. - * This class converts the local common clock value to XrTime internally before - * storing; the reader side (SchemaTracker) converts it back to monotonic ns so - * that DeviceDataTimestamp always carries monotonic nanoseconds in both - * _local_common_clock fields. + * The local OpenXR channel converts the local common clock value to XrTime + * before storing; the reader side (SchemaTracker) converts it back so that + * DeviceDataTimestamp carries monotonic nanoseconds in both local-common-clock + * fields. A remote channel owns any client-to-server monotonic clock mapping + * needed before its server-side OpenXR injection. * * If the raw device clock is not available, pass the local common clock value * as a best-effort substitute. @@ -153,19 +146,8 @@ class SchemaPusher const SchemaPusherConfig& config() const; private: - void initialize_push_tensor_functions(const OpenXRSessionHandles& handles); - void create_tensor_collection(const OpenXRSessionHandles& handles); - SchemaPusherConfig m_config; - XrTimeConverter m_time_converter; - - // Push tensor collection handle - XrPushTensorCollectionNV m_push_tensor{ XR_NULL_HANDLE }; - - // Extension function pointers - PFN_xrCreatePushTensorCollectionNV m_create_fn{ nullptr }; - PFN_xrPushTensorCollectionDataNV m_push_fn{ nullptr }; - PFN_xrDestroyPushTensorCollectionNV m_destroy_fn{ nullptr }; + std::unique_ptr m_channel; }; } // namespace core diff --git a/src/core/pusherio/cpp/inc/pusherio/wrist_tracking_source.hpp b/src/core/pusherio/cpp/inc/pusherio/wrist_tracking_source.hpp new file mode 100644 index 0000000000..8a05778102 --- /dev/null +++ b/src/core/pusherio/cpp/inc/pusherio/wrist_tracking_source.hpp @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include + +namespace core +{ + +enum class WristTrackingSourceMode +{ + Auto, + HandTracking, + Controller, +}; + +struct WristTrackingSourceConfig +{ + WristTrackingSourceMode mode = WristTrackingSourceMode::Auto; + XrPosef left_aim_to_wrist{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; + XrPosef right_aim_to_wrist{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; +}; + +struct WristTrackingSample +{ + XrPosef pose{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; + bool valid = false; //!< Pose is usable, including a cached last-good pose. + bool tracked = false; //!< Source is actively tracked for this sample. +}; + +/*! @brief Transport-neutral source for a hand's wrist pose in the session base space. */ +class IWristTrackingSource +{ +public: + virtual ~IWristTrackingSource() = default; + + //! Queries the pose corresponding to a timestamp in the local common-clock domain. + virtual WristTrackingSample query(bool is_left, int64_t sample_time_local_common_clock_ns) = 0; +}; + +} // namespace core diff --git a/src/core/pusherio/cpp/openxr_schema_push_channel.cpp b/src/core/pusherio/cpp/openxr_schema_push_channel.cpp new file mode 100644 index 0000000000..a6f90fdfca --- /dev/null +++ b/src/core/pusherio/cpp/openxr_schema_push_channel.cpp @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/pusherio/openxr_schema_push_channel.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core +{ + +namespace +{ + +// DLPack dtype code for uint8: code=1 (unsigned int), bits=8 +// Formula: (code << 8) | bits +constexpr uint32_t DLPACK_DTYPE_UINT8 = (1 << 8) | 8; + +} // namespace + +class OpenXRSchemaPushChannel::Impl +{ +public: + Impl(const OpenXRSessionHandles& handles, SchemaPusherConfig config) + : config_(std::move(config)), time_converter_(handles) + { + assert(handles.instance != XR_NULL_HANDLE && "OpenXR instance handle cannot be null"); + assert(handles.session != XR_NULL_HANDLE && "OpenXR session handle cannot be null"); + assert(handles.xrGetInstanceProcAddr && "xrGetInstanceProcAddr cannot be null"); + + initialize_push_tensor_functions(handles); + create_tensor_collection(handles); + + std::cout << "SchemaPusher initialized for collection: " << config_.collection_id << std::endl; + } + + ~Impl() + { + // push_tensor_ is guaranteed non-null after construction succeeds. + assert(push_tensor_ != XR_NULL_HANDLE && destroy_fn_ != nullptr); + + const XrResult result = destroy_fn_(push_tensor_); + if (result != XR_SUCCESS) + { + std::cerr << "Warning: Failed to destroy push tensor collection, result=" << result << std::endl; + } + } + + const SchemaPusherConfig& config() const + { + return config_; + } + + void push_buffer(const uint8_t* buffer, + size_t size, + int64_t sample_time_local_common_clock_ns, + int64_t sample_time_raw_device_clock_ns) + { + std::vector padded_buffer(config_.max_flatbuffer_size, 0); + if (size != 0) + { + std::memcpy(padded_buffer.data(), buffer, size); + } + + const XrTime xr_time = time_converter_.convert_monotonic_ns_to_xrtime(sample_time_local_common_clock_ns); + + XrPushTensorCollectionDataNV tensor_data{}; + tensor_data.type = XR_TYPE_PUSH_TENSOR_COLLECTION_DATA_NV; + tensor_data.next = nullptr; + tensor_data.timestamp = xr_time; + tensor_data.rawDeviceTimestamp = static_cast(sample_time_raw_device_clock_ns); + tensor_data.buffer = padded_buffer.data(); + tensor_data.bufferSize = static_cast(config_.max_flatbuffer_size); + + const XrResult result = push_fn_(push_tensor_, &tensor_data); + if (result != XR_SUCCESS) + { + throw std::runtime_error("Failed to push tensor data, result=" + std::to_string(result)); + } + } + +private: + void initialize_push_tensor_functions(const OpenXRSessionHandles& handles) + { + loadExtensionFunction(handles.instance, handles.xrGetInstanceProcAddr, "xrCreatePushTensorCollectionNV", + reinterpret_cast(&create_fn_)); + loadExtensionFunction(handles.instance, handles.xrGetInstanceProcAddr, "xrPushTensorCollectionDataNV", + reinterpret_cast(&push_fn_)); + loadExtensionFunction(handles.instance, handles.xrGetInstanceProcAddr, "xrDestroyPushTensorCollectionNV", + reinterpret_cast(&destroy_fn_)); + } + + void create_tensor_collection(const OpenXRSessionHandles& handles) + { + XrPushTensorDlpackCreateInfoNV dlpack_info{}; + dlpack_info.type = XR_TYPE_PUSH_TENSOR_DLPACK_CREATE_INFO_NV; + dlpack_info.next = nullptr; + dlpack_info.data.versionMajor = 1; + dlpack_info.data.versionMinor = 0; + dlpack_info.data.dtype = DLPACK_DTYPE_UINT8; + dlpack_info.data.ndim = 1; + dlpack_info.data.shape[0] = static_cast(config_.max_flatbuffer_size); + dlpack_info.data.strides[0] = sizeof(uint8_t); + dlpack_info.data.byte_offset = 0; + + XrPushTensorCreateInfoNV tensor_info{}; + tensor_info.type = XR_TYPE_PUSH_TENSOR_CREATE_INFO_NV; + tensor_info.next = &dlpack_info; + tensor_info.properties.dataType = XR_TENSOR_DATA_TYPE_DLPACK_NV; + tensor_info.properties.dataTypeSize = config_.max_flatbuffer_size; + tensor_info.properties.offset = 0; + std::strncpy( + tensor_info.properties.identifier, config_.tensor_identifier.c_str(), XR_MAX_TENSOR_IDENTIFIER_SIZE - 1); + tensor_info.properties.identifier[XR_MAX_TENSOR_IDENTIFIER_SIZE - 1] = '\0'; + + XrPushTensorCollectionCreateInfoNV create_info{}; + create_info.type = XR_TYPE_PUSH_TENSOR_COLLECTION_CREATE_INFO_NV; + create_info.next = nullptr; + create_info.tensors = &tensor_info; + create_info.data.tensorCount = 1; + create_info.data.totalSampleSize = config_.max_flatbuffer_size; + std::strncpy(create_info.data.identifier, config_.collection_id.c_str(), XR_MAX_TENSOR_IDENTIFIER_SIZE - 1); + create_info.data.identifier[XR_MAX_TENSOR_IDENTIFIER_SIZE - 1] = '\0'; + std::strncpy( + create_info.data.localizedName, config_.localized_name.c_str(), XR_MAX_TENSOR_LOCALIZED_NAME_SIZE - 1); + create_info.data.localizedName[XR_MAX_TENSOR_LOCALIZED_NAME_SIZE - 1] = '\0'; + std::memset(&create_info.data.uuid, 0, sizeof(create_info.data.uuid)); + + XrPushTensorCollectionCreateResultNV create_result{}; + create_result.type = XR_TYPE_PUSH_TENSOR_COLLECTION_CREATE_RESULT_NV; + create_result.next = nullptr; + + const XrResult result = create_fn_(handles.session, &create_info, &create_result, &push_tensor_); + if (result != XR_SUCCESS) + { + throw std::runtime_error("Failed to create push tensor collection, result=" + std::to_string(result)); + } + } + + SchemaPusherConfig config_; + XrTimeConverter time_converter_; + XrPushTensorCollectionNV push_tensor_{ XR_NULL_HANDLE }; + PFN_xrCreatePushTensorCollectionNV create_fn_{ nullptr }; + PFN_xrPushTensorCollectionDataNV push_fn_{ nullptr }; + PFN_xrDestroyPushTensorCollectionNV destroy_fn_{ nullptr }; +}; + +OpenXRSchemaPushChannel::OpenXRSchemaPushChannel(const OpenXRSessionHandles& handles, SchemaPusherConfig config) + : impl_(std::make_unique(handles, std::move(config))) +{ +} + +OpenXRSchemaPushChannel::~OpenXRSchemaPushChannel() = default; + +std::vector OpenXRSchemaPushChannel::get_required_extensions() +{ + std::vector required_extensions = { "XR_NVX1_push_tensor", "XR_NVX1_tensor_data" }; + for (const auto& ext : XrTimeConverter::get_required_extensions()) + { + required_extensions.push_back(ext); + } + return required_extensions; +} + +const SchemaPusherConfig& OpenXRSchemaPushChannel::config() const +{ + return impl_->config(); +} + +void OpenXRSchemaPushChannel::push_buffer(const uint8_t* buffer, + size_t size, + int64_t sample_time_local_common_clock_ns, + int64_t sample_time_raw_device_clock_ns) +{ + impl_->push_buffer(buffer, size, sample_time_local_common_clock_ns, sample_time_raw_device_clock_ns); +} + +std::unique_ptr make_openxr_schema_push_channel(const OpenXRSessionHandles& handles, + SchemaPusherConfig config) +{ + return std::make_unique(handles, std::move(config)); +} + +} // namespace core diff --git a/src/core/pusherio/cpp/schema_pusher.cpp b/src/core/pusherio/cpp/schema_pusher.cpp index 6aa71a7fbc..29bd8f5558 100644 --- a/src/core/pusherio/cpp/schema_pusher.cpp +++ b/src/core/pusherio/cpp/schema_pusher.cpp @@ -1,94 +1,57 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 #include "inc/pusherio/schema_pusher.hpp" -#include - -#include -#include -#include #include -#include +#include namespace core { -// DLPack dtype code for uint8: code=1 (unsigned int), bits=8 -// Formula: (code << 8) | bits -static constexpr uint32_t DLPACK_DTYPE_UINT8 = (1 << 8) | 8; - -SchemaPusher::SchemaPusher(const OpenXRSessionHandles& handles, SchemaPusherConfig config) - : m_config(std::move(config)), m_time_converter(handles) +namespace { - // Validate handles - assert(handles.instance != XR_NULL_HANDLE && "OpenXR instance handle cannot be null"); - assert(handles.session != XR_NULL_HANDLE && "OpenXR session handle cannot be null"); - assert(handles.xrGetInstanceProcAddr && "xrGetInstanceProcAddr cannot be null"); - - // Initialize extension functions using the provided xrGetInstanceProcAddr - initialize_push_tensor_functions(handles); - - // Create the tensor collection - create_tensor_collection(handles); - std::cout << "SchemaPusher initialized for collection: " << m_config.collection_id << std::endl; -} - -SchemaPusher::~SchemaPusher() +const SchemaPusherConfig& require_channel_config(const std::unique_ptr& channel) { - // m_push_tensor is guaranteed to be non-null by create_tensor_collection(), or the constructor would have thrown. - assert(m_push_tensor != XR_NULL_HANDLE && m_destroy_fn != nullptr); - - // Destroy the push tensor collection - XrResult result = m_destroy_fn(m_push_tensor); - if (result != XR_SUCCESS) + if (!channel) { - std::cerr << "Warning: Failed to destroy push tensor collection, result=" << result << std::endl; + throw std::invalid_argument("SchemaPusher requires a schema push channel"); } + return channel->config(); } +} // namespace + +SchemaPusher::SchemaPusher(std::unique_ptr channel) + : m_config(require_channel_config(channel)), m_channel(std::move(channel)) +{ +} + +SchemaPusher::~SchemaPusher() = default; + void SchemaPusher::push_buffer(const uint8_t* buffer, size_t size, int64_t sample_time_local_common_clock_ns, int64_t sample_time_raw_device_clock_ns) { - // Validate that the serialized size fits within our declared buffer if (size > m_config.max_flatbuffer_size) { throw std::runtime_error("Serialized data size (" + std::to_string(size) + " bytes) exceeds max_flatbuffer_size (" + std::to_string(m_config.max_flatbuffer_size) + " bytes)"); } - - // Create padded buffer to match declared tensor size - // The DLPack tensor is declared as uint8[max_flatbuffer_size], so we need to pad - std::vector padded_buffer(m_config.max_flatbuffer_size, 0); - std::memcpy(padded_buffer.data(), buffer, size); - - // Convert monotonic nanoseconds to XrTime for the tensor header - XrTime xr_time = m_time_converter.convert_monotonic_ns_to_xrtime(sample_time_local_common_clock_ns); - - // Prepare push data structure - XrPushTensorCollectionDataNV tensorData{}; - tensorData.type = XR_TYPE_PUSH_TENSOR_COLLECTION_DATA_NV; - tensorData.next = nullptr; - tensorData.timestamp = xr_time; if (sample_time_raw_device_clock_ns < 0) { throw std::runtime_error("push_buffer: sample_time_raw_device_clock_ns is negative (" + std::to_string(sample_time_raw_device_clock_ns) + ")"); } - tensorData.rawDeviceTimestamp = static_cast(sample_time_raw_device_clock_ns); - tensorData.buffer = padded_buffer.data(); - tensorData.bufferSize = static_cast(m_config.max_flatbuffer_size); - - // Push the data - XrResult result = m_push_fn(m_push_tensor, &tensorData); - if (result != XR_SUCCESS) + if (buffer == nullptr && size != 0) { - throw std::runtime_error("Failed to push tensor data, result=" + std::to_string(result)); + throw std::invalid_argument("push_buffer: buffer is null for a non-empty sample"); } + + m_channel->push_buffer(buffer, size, sample_time_local_common_clock_ns, sample_time_raw_device_clock_ns); } const SchemaPusherConfig& SchemaPusher::config() const @@ -96,64 +59,4 @@ const SchemaPusherConfig& SchemaPusher::config() const return m_config; } -void SchemaPusher::initialize_push_tensor_functions(const OpenXRSessionHandles& handles) -{ - loadExtensionFunction(handles.instance, handles.xrGetInstanceProcAddr, "xrCreatePushTensorCollectionNV", - reinterpret_cast(&m_create_fn)); - loadExtensionFunction(handles.instance, handles.xrGetInstanceProcAddr, "xrPushTensorCollectionDataNV", - reinterpret_cast(&m_push_fn)); - loadExtensionFunction(handles.instance, handles.xrGetInstanceProcAddr, "xrDestroyPushTensorCollectionNV", - reinterpret_cast(&m_destroy_fn)); -} - -void SchemaPusher::create_tensor_collection(const OpenXRSessionHandles& handles) -{ - // Set up DLPack tensor properties for a 1D uint8 array (byte buffer for FlatBuffer) - XrPushTensorDlpackCreateInfoNV dlpackInfo{}; - dlpackInfo.type = XR_TYPE_PUSH_TENSOR_DLPACK_CREATE_INFO_NV; - dlpackInfo.next = nullptr; - dlpackInfo.data.versionMajor = 1; - dlpackInfo.data.versionMinor = 0; - dlpackInfo.data.dtype = DLPACK_DTYPE_UINT8; - dlpackInfo.data.ndim = 1; - dlpackInfo.data.shape[0] = static_cast(m_config.max_flatbuffer_size); - dlpackInfo.data.strides[0] = sizeof(uint8_t); - dlpackInfo.data.byte_offset = 0; - - // Create tensor info with DLPack properties chained - XrPushTensorCreateInfoNV tensorInfo{}; - tensorInfo.type = XR_TYPE_PUSH_TENSOR_CREATE_INFO_NV; - tensorInfo.next = &dlpackInfo; - tensorInfo.properties.dataType = XR_TENSOR_DATA_TYPE_DLPACK_NV; - tensorInfo.properties.dataTypeSize = m_config.max_flatbuffer_size; - tensorInfo.properties.offset = 0; - std::strncpy(tensorInfo.properties.identifier, m_config.tensor_identifier.c_str(), XR_MAX_TENSOR_IDENTIFIER_SIZE - 1); - tensorInfo.properties.identifier[XR_MAX_TENSOR_IDENTIFIER_SIZE - 1] = '\0'; - - // Create tensor collection with one tensor - XrPushTensorCollectionCreateInfoNV createInfo{}; - createInfo.type = XR_TYPE_PUSH_TENSOR_COLLECTION_CREATE_INFO_NV; - createInfo.next = nullptr; - createInfo.tensors = &tensorInfo; - createInfo.data.tensorCount = 1; - createInfo.data.totalSampleSize = m_config.max_flatbuffer_size; - std::strncpy(createInfo.data.identifier, m_config.collection_id.c_str(), XR_MAX_TENSOR_IDENTIFIER_SIZE - 1); - createInfo.data.identifier[XR_MAX_TENSOR_IDENTIFIER_SIZE - 1] = '\0'; - std::strncpy(createInfo.data.localizedName, m_config.localized_name.c_str(), XR_MAX_TENSOR_LOCALIZED_NAME_SIZE - 1); - createInfo.data.localizedName[XR_MAX_TENSOR_LOCALIZED_NAME_SIZE - 1] = '\0'; - // Zero out UUID (optional, runtime may assign) - std::memset(&createInfo.data.uuid, 0, sizeof(createInfo.data.uuid)); - - // Create the tensor collection - XrPushTensorCollectionCreateResultNV createResult{}; - createResult.type = XR_TYPE_PUSH_TENSOR_COLLECTION_CREATE_RESULT_NV; - createResult.next = nullptr; - - XrResult result = m_create_fn(handles.session, &createInfo, &createResult, &m_push_tensor); - if (result != XR_SUCCESS) - { - throw std::runtime_error("Failed to create push tensor collection, result=" + std::to_string(result)); - } -} - } // namespace core diff --git a/src/plugins/controller_se3_tracker/CMakeLists.txt b/src/plugins/controller_se3_tracker/CMakeLists.txt index 64b9811dd2..1d9d4ddb90 100644 --- a/src/plugins/controller_se3_tracker/CMakeLists.txt +++ b/src/plugins/controller_se3_tracker/CMakeLists.txt @@ -10,10 +10,10 @@ add_executable(controller_se3_tracker_plugin ) target_link_libraries(controller_se3_tracker_plugin PRIVATE + Teleop::plugin_utils deviceio::deviceio_session deviceio::deviceio_trackers isaacteleop_schema - oxr::oxr_core pusherio::pusherio ) diff --git a/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.cpp b/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.cpp index 41761b00d5..c384e8a419 100644 --- a/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.cpp +++ b/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.cpp @@ -9,10 +9,10 @@ #include #include -#include #include #include -#include +#include +#include namespace plugins { @@ -22,19 +22,6 @@ namespace controller_se3_tracker namespace { -std::vector make_required_extensions(const std::vector>& trackers) -{ - auto extensions = core::DeviceIOSession::get_required_extensions(trackers); - for (const auto& ext : core::SchemaPusher::get_required_extensions()) - { - if (std::find(extensions.begin(), extensions.end(), ext) == extensions.end()) - { - extensions.push_back(ext); - } - } - return extensions; -} - core::SchemaPusherConfig make_pusher_config(const std::string& collection_id) { // Wire rendezvous (tensor identifier + buffer size) comes from the Se3Tracker facade — @@ -48,17 +35,25 @@ core::SchemaPusherConfig make_pusher_config(const std::string& collection_id) } // namespace -ControllerSe3TrackerPlugin::ControllerSe3TrackerPlugin(bool use_left_hand, const std::string& collection_id) - : m_use_left_hand(use_left_hand) +ControllerSe3TrackerPlugin::ControllerSe3TrackerPlugin(bool use_left_hand, + const std::string& collection_id, + std::shared_ptr controller_tracker, + core::PluginSessionHandle plugin_session) + : m_use_left_hand(use_left_hand), + m_controller_tracker(std::move(controller_tracker)), + m_plugin_session(std::move(plugin_session)) { - m_controller_tracker = std::make_shared(); - std::vector> trackers = { m_controller_tracker }; - - m_session = std::make_shared("ControllerSe3TrackerPlugin", make_required_extensions(trackers)); - const auto handles = m_session->get_handles(); - - m_deviceio_session = core::DeviceIOSession::run(trackers, handles); - m_pusher = std::make_unique(handles, make_pusher_config(collection_id)); + if (!m_controller_tracker || !m_plugin_session) + { + throw std::invalid_argument("ControllerSe3TrackerPlugin requires a controller tracker and plugin session"); + } + m_pull_channel = m_plugin_session->create_pull_channel(); + if (!m_pull_channel) + { + throw std::runtime_error("The plugin session could not create a pull channel"); + } + m_pusher = std::make_unique( + m_plugin_session->create_schema_push_channel(make_pusher_config(collection_id))); std::cout << "ControllerSe3TrackerPlugin: republishing " << (m_use_left_hand ? "left" : "right") << " controller grip pose on collection '" << collection_id << "'" << std::endl; @@ -66,18 +61,15 @@ ControllerSe3TrackerPlugin::ControllerSe3TrackerPlugin(bool use_left_hand, const void ControllerSe3TrackerPlugin::update() { - // Capture the sample time BEFORE update(): DeviceIOSession::update() samples - // os_monotonic_now_ns() internally and locates the controller pose at that tick - // (deviceio_session.cpp). The pre-update capture approximates that tick time to - // microseconds; do not move this to post-update/push time — that would add loop - // processing bias to cross-device synchronization. + // Capture before update so the output timestamp approximates the pull + // channel's polling tick instead of including this loop's processing time. const int64_t sample_time_ns = core::os_monotonic_now_ns(); - m_deviceio_session->update(); + m_pull_channel->update(); const core::Serialized& tracked = - m_use_left_hand ? m_controller_tracker->get_left_controller(*m_deviceio_session) : - m_controller_tracker->get_right_controller(*m_deviceio_session); + m_use_left_hand ? m_controller_tracker->get_left_controller(*m_pull_channel) : + m_controller_tracker->get_right_controller(*m_pull_channel); const core::ControllerSnapshot* snapshot = tracked.get(); diff --git a/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.hpp b/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.hpp index c250de8c85..769df1949c 100644 --- a/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.hpp +++ b/src/plugins/controller_se3_tracker/controller_se3_tracker_plugin.hpp @@ -3,9 +3,8 @@ #pragma once -#include #include -#include +#include #include #include @@ -20,8 +19,8 @@ namespace controller_se3_tracker * @brief Logical SE3 tracker driven by an XR controller. * * Reads the configured controller's grip pose (the OpenXR rigid-attachment frame for the - * physical device) each tick and republishes it as an ``Se3TrackerPose`` via OpenXR - * SchemaPusher, in the same OpenXR session base reference space. Pair with an + * physical device) each tick and republishes it as an ``Se3TrackerPose`` through + * the plugin session, in the same session base reference space. Pair with an * ``Se3Tracker`` on the same ``collection_id``. * * Producer-only: this plugin never registers an ``Se3Tracker`` in its own session (it @@ -30,14 +29,17 @@ namespace controller_se3_tracker class ControllerSe3TrackerPlugin { public: - ControllerSe3TrackerPlugin(bool use_left_hand, const std::string& collection_id); + ControllerSe3TrackerPlugin(bool use_left_hand, + const std::string& collection_id, + std::shared_ptr controller_tracker, + core::PluginSessionHandle plugin_session); ControllerSe3TrackerPlugin(const ControllerSe3TrackerPlugin&) = delete; ControllerSe3TrackerPlugin& operator=(const ControllerSe3TrackerPlugin&) = delete; ControllerSe3TrackerPlugin(ControllerSe3TrackerPlugin&&) = delete; ControllerSe3TrackerPlugin& operator=(ControllerSe3TrackerPlugin&&) = delete; - //! One tick: update the device session, read the controller, push one Se3TrackerPose. + //! One tick: update the pull channel, read the controller, push one Se3TrackerPose. //! Pushes EVERY tick — is_valid=false (identity filler pose) when the controller is //! absent or its grip pose is invalid. void update(); @@ -46,8 +48,8 @@ class ControllerSe3TrackerPlugin bool m_use_left_hand; std::shared_ptr m_controller_tracker; - std::shared_ptr m_session; - std::unique_ptr m_deviceio_session; + core::PluginSessionHandle m_plugin_session; + std::unique_ptr m_pull_channel; std::unique_ptr m_pusher; }; diff --git a/src/plugins/controller_se3_tracker/main.cpp b/src/plugins/controller_se3_tracker/main.cpp index 679ce9da72..3602dff584 100644 --- a/src/plugins/controller_se3_tracker/main.cpp +++ b/src/plugins/controller_se3_tracker/main.cpp @@ -4,12 +4,15 @@ #include "controller_se3_tracker_plugin.hpp" #include +#include #include #include #include #include #include +#include +#include using namespace plugins::controller_se3_tracker; @@ -29,7 +32,11 @@ try std::cout << "Controller SE3 Tracker (hand: " << hand << ", collection: " << collection_id << ")" << std::endl; - ControllerSe3TrackerPlugin plugin(hand == "left", collection_id); + auto controller_tracker = std::make_shared(); + std::vector> trackers = { controller_tracker }; + core::PluginSessionHandle session = std::make_shared( + "ControllerSe3TrackerPlugin", core::PluginSessionRequirements{ .schema_push = true }, std::move(trackers)); + ControllerSe3TrackerPlugin plugin(hand == "left", collection_id, std::move(controller_tracker), std::move(session)); // Push data at 90 Hz const auto frame_duration = std::chrono::nanoseconds(1000000000 / 90); diff --git a/src/plugins/controller_synthetic_hands/README.md b/src/plugins/controller_synthetic_hands/README.md index 61fa98f702..6ab3ca309f 100644 --- a/src/plugins/controller_synthetic_hands/README.md +++ b/src/plugins/controller_synthetic_hands/README.md @@ -5,225 +5,88 @@ SPDX-License-Identifier: Apache-2.0 # Controller Synthetic Hands -Generates hand tracking data from controller poses and injects it into the OpenXR runtime. +Generates hand tracking data from controller poses and publishes it through an +Isaac Teleop plugin session. ## Overview -Reads controller grip and aim poses, generates realistic hand joint configurations, and pushes the data to the runtime via push devices. +The plugin reads controller grip and aim poses through an +`IPluginPullChannel`, generates a 26-joint hand pose, and publishes each active +hand through a `HandTrackingPusher`. + +The executable is the composition root. It currently constructs an +`OpenXRPluginSession`, but the `SyntheticHandsPlugin` implementation depends +only on `IPluginSession` and can use another session adapter. + +```text +ControllerTracker + | +IPluginPullChannel --update()--> controller snapshot + | +HandGenerator + | +HandTrackingPusher --> IHandTrackingPushChannel --> session backend +``` + +The per-hand pusher is created when its controller becomes active and destroyed +when the controller disappears. Closing the channel marks that hand inactive +instead of leaving a frozen pose. ## Quick Start ### Build ```bash -cd build -cmake .. -make controller_synthetic_hands +cmake -S . -B build +cmake --build build --target controller_synthetic_hands ``` ### Run ```bash -./controller_synthetic_hands +./build/src/plugins/controller_synthetic_hands/controller_synthetic_hands ``` Press Ctrl+C to exit. -## Architecture - -### Components - -Four focused components: - -**session** (`oxr/oxr_session.hpp`) - OpenXR initialization and session management (from core) -```cpp -class OpenXRSession { - OpenXRSession(const std::string& app_name, const std::vector& extensions); - OpenXRSessionHandles get_handles() const; -}; -``` - -**controllers** (`controllers.hpp/cpp`) - Controller input tracking -```cpp -class Controllers { - Controllers(XrInstance, XrSession, XrSpace); - void update(XrTime time); - const ControllerPose& left() const; - const ControllerPose& right() const; -}; -``` - -**hand_generator** (`hand_generator.hpp/cpp`) - Hand joint generation -```cpp -class HandGenerator { - void generate(XrHandJointLocationEXT* joints, - const XrPosef& wrist_pose, - bool is_left_hand, - float curl = 0.0f); -}; -``` - -**hand_injector** (`hand_injector.hpp/cpp`) - Push device data injection -```cpp -class HandInjector { - HandInjector(XrInstance, XrSession, XrHandEXT hand, XrSpace base_space); - void push(const XrHandJointLocationEXT*, XrTime); - // Destructor automatically signals isActive=false before releasing the push device. -}; -``` - -### Data Flow - -``` -Controllers → Wrist Pose → Hand Generator → Hand Injector → OpenXR Runtime -``` - -## Implementation - -### Main Loop +## Plugin-facing setup ```cpp -#include -#include -#include - -// Create controller tracker and get required extensions auto controller_tracker = std::make_shared(); -std::vector> trackers = { controller_tracker }; -auto extensions = core::DeviceIOSession::get_required_extensions(trackers); -extensions.push_back(XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME); - -// Create session with required extensions -auto session = std::make_shared("MyApp", extensions); -auto h = session->get_handles(); - -// Create DeviceIOSession to manage trackers -auto deviceio_session = core::DeviceIOSession::run(trackers, h); - -HandGenerator hands; -HandInjector left_injector(h.instance, h.session, XR_HAND_LEFT_EXT, h.space); -HandInjector right_injector(h.instance, h.session, XR_HAND_RIGHT_EXT, h.space); - -while (running) { - deviceio_session->update(); - - const auto& left_tracked = controller_tracker->get_left_controller(*deviceio_session); - if (left_tracked) { - bool grip_valid, aim_valid; - oxr_utils::get_grip_pose(*left_tracked, grip_valid); - XrPosef wrist = oxr_utils::get_aim_pose(*left_tracked, aim_valid); - - if (grip_valid && aim_valid) { - float trigger = left_tracked->inputs().trigger_value(); - hands.generate(joints, wrist, true, trigger); - left_injector.push(joints, time); - } - } -} -// left_injector and right_injector signal isActive=false on destruction. -``` - -### Using Individual Components +std::vector> trackers = {controller_tracker}; -#### Session Initialization +core::PluginSessionHandle session = + std::make_shared( + "ControllerSyntheticHands", + core::PluginSessionRequirements{.hand_tracking_push = true}, + std::move(trackers)); -```cpp -#include - -auto session = std::make_shared("MyApp", {"XR_EXT_hand_tracking"}); -auto handles = session->get_handles(); -// handles.instance, handles.session, handles.space are available +SyntheticHandsPlugin plugin( + plugin_root_id, std::move(controller_tracker), std::move(session)); ``` -#### Controller Tracking +Only the composition root names the concrete adapter. Inside the plugin, one +tick is: ```cpp -#include -#include -#include - -auto controller_tracker = std::make_shared(); -std::vector> trackers = { controller_tracker }; -auto deviceio_session = core::DeviceIOSession::run(trackers, handles); +pull_channel->update(); +const auto& controller = controller_tracker->get_left_controller(*pull_channel); -deviceio_session->update(); -const auto& left_tracked = controller_tracker->get_left_controller(*deviceio_session); -const auto& right_tracked = controller_tracker->get_right_controller(*deviceio_session); -// Dereference left_tracked / right_tracked (empty handles when inactive) -``` - -#### Hand Generation - -```cpp -#include "hand_generator.hpp" - -HandGenerator generator; -XrHandJointLocationEXT joints[XR_HAND_JOINT_COUNT_EXT]; -generator.generate(joints, wrist_pose, true, curl_value); -``` - -#### Hand Injection - -```cpp -#include - -HandInjector left_injector(instance, session, XR_HAND_LEFT_EXT, space); -HandInjector right_injector(instance, session, XR_HAND_RIGHT_EXT, space); -left_injector.push(joints, timestamp); -// Destruction automatically signals isActive=false. -``` - -## Technical Details - -### Hand Joint Generation - -Generates 26 joints per hand with anatomically correct positions and orientations. Joint offsets are defined in meters relative to the wrist, then rotated by the wrist orientation. - -Coordinate system for left hand: -- X-axis: positive = thumb side, negative = pinky side -- Y-axis: positive = back of hand, negative = palm side -- Z-axis: positive = forward (fingers pointing), negative = toward wrist - -Right hand is mirrored on the X-axis. - -### Resource Management - -All components use RAII - resources acquired in constructor, released in destructor. - -### Dependencies - -``` -controller_synthetic_hands.cpp - ├── oxr_session (from core, standalone) - ├── controllers (requires OpenXR handles) - ├── hand_generator (standalone) - └── hand_injector (requires OpenXR handles) -``` - -## Extension Points - -### New Input Sources - -```cpp -class HandTrackingInput { - HandTrackingInput(XrInstance, XrSession); - void update(XrTime); - const HandData& left() const; -}; +if (controller) { + hand_generator.generate(joints, wrist_pose, true, trigger_value); + left_pusher->push(joints, sample_time_local_common_clock_ns); +} ``` -### Data Recording +## Hand generation -```cpp -class HandDataRecorder { - void record(const XrHandJointLocationEXT*, XrTime); -}; -``` +Joint offsets are defined in meters relative to the wrist and transformed by +the wrist pose. The left-hand coordinate conventions are: -### Gesture Recognition +- X: thumb side to pinky side +- Y: back of hand to palm +- Z: fingers to wrist -```cpp -class GestureRecognizer { - Gesture recognize(const XrHandJointLocationEXT*); -}; -``` +The right hand is mirrored on the X axis. OpenXR value structs describe the +established joint and pose layout; runtime handles and calls remain in the +concrete session adapter. diff --git a/src/plugins/controller_synthetic_hands/controller_synthetic_hands.cpp b/src/plugins/controller_synthetic_hands/controller_synthetic_hands.cpp index bb74d5684f..8c125caa56 100644 --- a/src/plugins/controller_synthetic_hands/controller_synthetic_hands.cpp +++ b/src/plugins/controller_synthetic_hands/controller_synthetic_hands.cpp @@ -3,10 +3,15 @@ #include "synthetic_hands_plugin.hpp" +#include +#include + #include #include #include #include +#include +#include using namespace plugins::controller_synthetic_hands; @@ -45,7 +50,12 @@ try std::cout << "Controller Synthetic Hands Plugin" << std::endl; std::cout << "Plugin Root ID: " << plugin_root_id << std::endl; - auto plugin = std::make_unique(plugin_root_id); + auto controller_tracker = std::make_shared(); + std::vector> trackers = { controller_tracker }; + core::PluginSessionHandle session = std::make_shared( + "ControllerSyntheticHands", core::PluginSessionRequirements{ .hand_tracking_push = true }, std::move(trackers)); + auto plugin = + std::make_unique(plugin_root_id, std::move(controller_tracker), std::move(session)); std::cout << "Plugin running. Press Ctrl+C to stop." << std::endl; while (!g_stop_requested.load(std::memory_order_relaxed)) diff --git a/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.cpp b/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.cpp index ebdaa6a231..88f857d2fe 100644 --- a/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.cpp +++ b/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.cpp @@ -3,6 +3,7 @@ #include "synthetic_hands_plugin.hpp" +#include #include #include @@ -10,36 +11,36 @@ #include #include #include +#include +#include namespace plugins { namespace controller_synthetic_hands { -SyntheticHandsPlugin::SyntheticHandsPlugin(const std::string& plugin_root_id) noexcept(false) - : m_root_id(plugin_root_id) +SyntheticHandsPlugin::SyntheticHandsPlugin(const std::string& plugin_root_id, + std::shared_ptr controller_tracker, + core::PluginSessionHandle plugin_session) noexcept(false) + : m_controller_tracker(std::move(controller_tracker)), + m_plugin_session(std::move(plugin_session)), + m_root_id(plugin_root_id) { std::cout << "Initializing SyntheticHandsPlugin with root: " << m_root_id << std::endl; - // Create ControllerTracker first to get required extensions - m_controller_tracker = std::make_shared(); - std::vector> trackers = { m_controller_tracker }; - - // Get required extensions from trackers - auto extensions = core::DeviceIOSession::get_required_extensions(trackers); - extensions.push_back(XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME); - - // Initialize session - constructor automatically begins the session - m_session = std::make_shared("ControllerSyntheticHands", extensions); - const auto handles = m_session->get_handles(); - - // Create DeviceIOSession with trackers - m_deviceio_session = core::DeviceIOSession::run(trackers, handles); + if (!m_controller_tracker || !m_plugin_session) + { + throw std::invalid_argument("SyntheticHandsPlugin requires a controller tracker and plugin session"); + } + m_pull_channel = m_plugin_session->create_pull_channel(); + if (!m_pull_channel) + { + throw std::runtime_error("The plugin session could not create a pull channel"); + } - // Injectors are created lazily in worker_thread once a controller is first seen, + // Pushers are created lazily in worker_thread once a controller is first seen, // and destroyed when the controller disappears. This ensures isActive reflects // whether a controller is actually present. - m_time_converter.emplace(handles); // Start worker thread m_running = true; @@ -73,31 +74,28 @@ void SyntheticHandsPlugin::worker_thread() core::Serialized right_tracked; try { - // Update DeviceIOSession (handles time and tracker updates) - m_deviceio_session->update(); + m_pull_channel->update(); // Read tracker data in the same exception boundary as update. - left_tracked = m_controller_tracker->get_left_controller(*m_deviceio_session); - right_tracked = m_controller_tracker->get_right_controller(*m_deviceio_session); + left_tracked = m_controller_tracker->get_left_controller(*m_pull_channel); + right_tracked = m_controller_tracker->get_right_controller(*m_pull_channel); } catch (const std::exception& e) { std::cerr << "SyntheticHandsPlugin update error: " << e.what() << std::endl; - m_left_injector.reset(); - m_right_injector.reset(); + m_left_pusher.reset(); + m_right_pusher.reset(); std::exit(1); } catch (...) { std::cerr << "SyntheticHandsPlugin update error: unknown exception" << std::endl; - m_left_injector.reset(); - m_right_injector.reset(); + m_left_pusher.reset(); + m_right_pusher.reset(); std::exit(1); } - // Use the OpenXR runtime clock for injection time so it aligns with the - // runtime's own time domain (XrTime), rather than a raw steady_clock cast. - XrTime time = m_time_converter->os_monotonic_now(); + const int64_t sample_time_ns = core::os_monotonic_now_ns(); // Get target curl values from trigger inputs float left_target = 0.0f; @@ -130,7 +128,7 @@ void SyntheticHandsPlugin::worker_thread() // This plugin treats controller presence as a prerequisite for hand injection: // if the controller is gone, the synthetic hand is deactivated by resetting the - // injector. A different plugin could choose a different policy — for example, a + // pusher. A different plugin could choose a different policy — for example, a // plugin with independent joint data (e.g. a glove) could keep pushing joints // even when no controller pose is available. if (m_left_enabled && left_tracked) @@ -142,21 +140,20 @@ void SyntheticHandsPlugin::worker_thread() if (grip_valid && aim_valid) { - if (!m_left_injector) + if (!m_left_pusher) { - const auto handles = m_session->get_handles(); - m_left_injector = std::make_unique( - handles.instance, handles.session, XR_HAND_LEFT_EXT, handles.space); + m_left_pusher = std::make_unique( + m_plugin_session->create_hand_tracking_push_channel(XR_HAND_LEFT_EXT)); } m_hand_gen.generate(left_joints, wrist, true, left_curl_current); - m_left_injector->push(left_joints, time); + m_left_pusher->push(left_joints, sample_time_ns); } } else { - // Controller not present — destroy the injector so the runtime sees + // Controller not present — destroy the pusher so the receiver sees // isActive=false rather than a frozen hand pose. - m_left_injector.reset(); + m_left_pusher.reset(); } if (m_right_enabled && right_tracked) @@ -168,21 +165,20 @@ void SyntheticHandsPlugin::worker_thread() if (grip_valid && aim_valid) { - if (!m_right_injector) + if (!m_right_pusher) { - const auto handles = m_session->get_handles(); - m_right_injector = std::make_unique( - handles.instance, handles.session, XR_HAND_RIGHT_EXT, handles.space); + m_right_pusher = std::make_unique( + m_plugin_session->create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT)); } m_hand_gen.generate(right_joints, wrist, false, right_curl_current); - m_right_injector->push(right_joints, time); + m_right_pusher->push(right_joints, sample_time_ns); } } else { - // Controller not present — destroy the injector so the runtime sees + // Controller not present — destroy the pusher so the receiver sees // isActive=false rather than a frozen hand pose. - m_right_injector.reset(); + m_right_pusher.reset(); } std::this_thread::sleep_for(std::chrono::milliseconds(16)); diff --git a/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.hpp b/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.hpp index e398e6b1fb..eecfb5ad8d 100644 --- a/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.hpp +++ b/src/plugins/controller_synthetic_hands/synthetic_hands_plugin.hpp @@ -4,16 +4,13 @@ #pragma once #include "hand_generator.hpp" -#include #include -#include -#include -#include +#include +#include #include #include #include -#include #include #include @@ -25,7 +22,9 @@ namespace controller_synthetic_hands class SyntheticHandsPlugin { public: - explicit SyntheticHandsPlugin(const std::string& plugin_root_id) noexcept(false); + SyntheticHandsPlugin(const std::string& plugin_root_id, + std::shared_ptr controller_tracker, + core::PluginSessionHandle plugin_session) noexcept(false); ~SyntheticHandsPlugin(); SyntheticHandsPlugin(const SyntheticHandsPlugin&) = delete; @@ -36,12 +35,11 @@ class SyntheticHandsPlugin private: void worker_thread(); - std::shared_ptr m_session; std::shared_ptr m_controller_tracker; - std::unique_ptr m_deviceio_session; - std::unique_ptr m_left_injector; - std::unique_ptr m_right_injector; - std::optional m_time_converter; + core::PluginSessionHandle m_plugin_session; + std::unique_ptr m_pull_channel; + std::unique_ptr m_left_pusher; + std::unique_ptr m_right_pusher; HandGenerator m_hand_gen; std::thread m_thread; diff --git a/src/plugins/generic_3axis_pedal/CMakeLists.txt b/src/plugins/generic_3axis_pedal/CMakeLists.txt index 061c40d071..cf0ebfc56a 100644 --- a/src/plugins/generic_3axis_pedal/CMakeLists.txt +++ b/src/plugins/generic_3axis_pedal/CMakeLists.txt @@ -14,8 +14,8 @@ add_executable(generic_3axis_pedal_plugin ) target_link_libraries(generic_3axis_pedal_plugin PRIVATE + Teleop::plugin_utils pusherio::pusherio - oxr::oxr_core isaacteleop_schema ) diff --git a/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.cpp b/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.cpp index 97347e998e..57d1773b81 100644 --- a/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.cpp +++ b/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include #include @@ -14,7 +13,9 @@ #include #include #include +#include #include +#include namespace plugins { @@ -28,6 +29,21 @@ constexpr size_t kJsEventSize = sizeof(js_event); constexpr double kMaxAxisValue = 32767.0; constexpr size_t kMaxFlatbufferSize = 256; +std::unique_ptr make_push_channel(const core::PluginSessionHandle& session, + const std::string& collection_id) +{ + if (!session) + { + throw std::invalid_argument("Generic3AxisPedalPlugin requires a plugin session"); + } + + return session->create_schema_push_channel(core::SchemaPusherConfig{ .collection_id = collection_id, + .max_flatbuffer_size = kMaxFlatbufferSize, + .tensor_identifier = "generic_3axis_pedal", + .localized_name = "Generic 3-Axis Pedal", + .app_name = "Generic3AxisPedalPlugin" }); +} + double normalize_axis(int16_t raw_value) { return std::max(-1.0, std::min(1.0, static_cast(raw_value) / kMaxAxisValue)); @@ -35,16 +51,10 @@ double normalize_axis(int16_t raw_value) } // namespace -Generic3AxisPedalPlugin::Generic3AxisPedalPlugin(const std::string& device_path, const std::string& collection_id) - : device_path_(device_path), - session_(std::make_shared( - "Generic3AxisPedalPlugin", core::SchemaPusher::get_required_extensions())), - pusher_(session_->get_handles(), - core::SchemaPusherConfig{ .collection_id = collection_id, - .max_flatbuffer_size = kMaxFlatbufferSize, - .tensor_identifier = "generic_3axis_pedal", - .localized_name = "Generic 3-Axis Pedal", - .app_name = "Generic3AxisPedalPlugin" }) +Generic3AxisPedalPlugin::Generic3AxisPedalPlugin(const std::string& device_path, + const std::string& collection_id, + core::PluginSessionHandle session) + : device_path_(device_path), session_(std::move(session)), pusher_(make_push_channel(session_, collection_id)) { if (!open_device()) throw std::runtime_error("Generic3AxisPedalPlugin: Failed to open " + device_path + " (" + strerror(errno) + ")"); diff --git a/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.hpp b/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.hpp index 46bedf5a47..b9f4ee059c 100644 --- a/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.hpp +++ b/src/plugins/generic_3axis_pedal/generic_3axis_pedal_plugin.hpp @@ -3,16 +3,12 @@ #pragma once +#include #include #include #include -namespace core -{ -class OpenXRSession; -} - namespace plugins { namespace generic_3axis_pedal @@ -21,12 +17,14 @@ namespace generic_3axis_pedal /*! * @brief Reads a Linux joystick (e.g. /dev/input/js0), maps axes to * left_pedal, right_pedal, rudder, and pushes Generic3AxisPedalOutput - * via OpenXR SchemaPusher. + * through a session-backed SchemaPusher. */ class Generic3AxisPedalPlugin { public: - Generic3AxisPedalPlugin(const std::string& device_path, const std::string& collection_id); + Generic3AxisPedalPlugin(const std::string& device_path, + const std::string& collection_id, + core::PluginSessionHandle session); ~Generic3AxisPedalPlugin(); void update(); @@ -40,7 +38,8 @@ class Generic3AxisPedalPlugin int device_fd_ = -1; double axes_[3] = { 0.0, 0.0, 0.0 }; - std::shared_ptr session_; + // Keep the session owner before pusher_ so its channel is destroyed first. + core::PluginSessionHandle session_; core::SchemaPusher pusher_; }; diff --git a/src/plugins/generic_3axis_pedal/main.cpp b/src/plugins/generic_3axis_pedal/main.cpp index 432af44899..99b53c4f1c 100644 --- a/src/plugins/generic_3axis_pedal/main.cpp +++ b/src/plugins/generic_3axis_pedal/main.cpp @@ -3,11 +3,15 @@ #include "generic_3axis_pedal_plugin.hpp" +#include + #include #include #include +#include #include #include +#include using namespace plugins::generic_3axis_pedal; @@ -25,7 +29,9 @@ try std::cout << "Generic 3-Axis Pedal (device: " << device_path << ", collection: " << collection_id << ")" << std::endl; - Generic3AxisPedalPlugin plugin(device_path, collection_id); + core::PluginSessionHandle session = std::make_shared( + "Generic3AxisPedalPlugin", core::PluginSessionRequirements{ .schema_push = true }); + Generic3AxisPedalPlugin plugin(device_path, collection_id, std::move(session)); // Push data at 90 Hz // TODO: Make the device push rate configurable diff --git a/src/plugins/haptikos/README.md b/src/plugins/haptikos/README.md index c67ec3250b..fc12f8e7cc 100644 --- a/src/plugins/haptikos/README.md +++ b/src/plugins/haptikos/README.md @@ -7,7 +7,11 @@ SPDX-License-Identifier: Apache-2.0 Use the Haptikos Exoskeletons with the Isaac Teleop framework. Currently only `Linux` is supported. Tested on `Meta Quest` headsets. Other headsets with controller may work as well. ## Overview -Reads the controllers' position and the hand tracking data from the Haptikos Core App, combines them and pushes them into the OpenXR runtime. To inject the hand tracking data, the controllers, the Haptikos Core App and the exoskeletons need to be active. +Reads controller poses through an `IPluginPullChannel`, combines them with hand +data from the Haptikos Core App, and publishes the result through +`HandTrackingPusher`. The executable currently selects `OpenXRPluginSession`; +the plugin implementation depends only on `IPluginSession`. The controllers, +Haptikos Core App, and exoskeletons must all be active. ## Quick Start diff --git a/src/plugins/haptikos/haptikos_hands_plugin.cpp b/src/plugins/haptikos/haptikos_hands_plugin.cpp index 37dd4db437..746242c58d 100644 --- a/src/plugins/haptikos/haptikos_hands_plugin.cpp +++ b/src/plugins/haptikos/haptikos_hands_plugin.cpp @@ -3,40 +3,43 @@ #include "haptikos_hands_plugin.hpp" +#include #include #include +#include +#include +#include +#include namespace plugins { namespace haptikos { -HaptikosHandsPlugin::HaptikosHandsPlugin(const std::string& plugin_root_id) noexcept(false) : m_root_id(plugin_root_id) +HaptikosHandsPlugin::HaptikosHandsPlugin(const std::string& plugin_root_id, + std::shared_ptr controller_tracker, + core::PluginSessionHandle plugin_session) noexcept(false) + : m_controller_tracker(std::move(controller_tracker)), + m_plugin_session(std::move(plugin_session)), + m_root_id(plugin_root_id) { static_assert(XR_HAND_JOINT_COUNT_EXT == HAPTIKOS_NUM_OF_JOINTS, "Unexpected XR Hand Joint number"); std::cout << "Initializing HaptikosHandsPlugin with root: " << m_root_id << std::endl; - // Create ControllerTracker first to get required extensions - m_controller_tracker = std::make_shared(); - m_hand_tracker = std::make_shared(); - std::vector> trackers = { m_controller_tracker, m_hand_tracker }; - - // Get required extensions from trackers - auto extensions = core::DeviceIOSession::get_required_extensions(trackers); - extensions.push_back(XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME); - - // Initialize session - constructor automatically begins the session - m_session = std::make_shared("HaptikosHands", extensions); - const auto handles = m_session->get_handles(); - - // Create DeviceIOSession with trackers - m_deviceio_session = core::DeviceIOSession::run(trackers, handles); + if (!m_controller_tracker || !m_plugin_session) + { + throw std::invalid_argument("HaptikosHandsPlugin requires a controller tracker and plugin session"); + } + m_pull_channel = m_plugin_session->create_pull_channel(); + if (!m_pull_channel) + { + throw std::runtime_error("The plugin session could not create a pull channel"); + } - // Injectors are created lazily in worker_thread once a controller is first seen, + // Pushers are created lazily in worker_thread once a controller is first seen, // and destroyed when the controller disappears. This ensures isActive reflects // whether a controller is actually present. - m_time_converter.emplace(handles); // Start worker thread m_running = true; @@ -65,95 +68,84 @@ void HaptikosHandsPlugin::worker_thread() { auto frame_start = std::chrono::steady_clock::now(); - core::Serialized left_tracked; - core::Serialized right_tracked; - - core::Serialized left_hand; - core::Serialized rigth_hand; - try { - // Update DeviceIOSession (handles time and tracker updates) - m_deviceio_session->update(); - - // Read tracker data in the same exception boundary as update. - left_tracked = m_controller_tracker->get_left_controller(*m_deviceio_session); - right_tracked = m_controller_tracker->get_right_controller(*m_deviceio_session); - } - catch (const std::exception& e) - { - std::cerr << "HaptikosHandsPlugin update error: " << e.what() << std::endl; - m_left_injector.reset(); - m_right_injector.reset(); - std::exit(1); - } - catch (...) - { - std::cerr << "HaptikosHandsPlugin update error: unknown exception" << std::endl; - m_left_injector.reset(); - m_right_injector.reset(); - std::exit(1); - } + core::Serialized left_tracked; + core::Serialized right_tracked; - // Use the OpenXR runtime clock for injection time so it aligns with the - // runtime's own time domain (XrTime), rather than a raw steady_clock cast. - XrTime time = m_time_converter->os_monotonic_now(); + m_pull_channel->update(); + // Read tracker data in the same exception boundary as update. + left_tracked = m_controller_tracker->get_left_controller(*m_pull_channel); + right_tracked = m_controller_tracker->get_right_controller(*m_pull_channel); - bool rigth_published = false; - if (right_tracked) - { - Haptikos::HandData right_data = m_client.GetData(true, Haptikos::GlobalToWrist, true, true, false); - bool valid_wrist = false; - XrPosef rigth_controller = oxr_utils::get_aim_pose(*right_tracked, valid_wrist); + const int64_t sample_time_ns = core::os_monotonic_now_ns(); - if (right_data.IsValid() == 1 && valid_wrist) + bool rigth_published = false; + if (right_tracked) { - calculate_hand_pose(right_joints, right_data, rigth_controller); - if (!m_right_injector) + Haptikos::HandData right_data = m_client.GetData(true, Haptikos::GlobalToWrist, true, true, false); + bool valid_wrist = false; + XrPosef rigth_controller = oxr_utils::get_aim_pose(*right_tracked, valid_wrist); + + if (right_data.IsValid() == 1 && valid_wrist) { - const auto handles = m_session->get_handles(); - m_right_injector = std::make_unique( - handles.instance, handles.session, XR_HAND_RIGHT_EXT, handles.space); + calculate_hand_pose(right_joints, right_data, rigth_controller); + if (!m_right_pusher) + { + m_right_pusher = std::make_unique( + m_plugin_session->create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT)); + } + + m_right_pusher->push(right_joints, sample_time_ns); + rigth_published = true; } - - m_right_injector->push(right_joints, time); - rigth_published = true; } - } + if (!rigth_published && m_right_pusher) + { + m_right_pusher.reset(); + } - if (!rigth_published && m_right_injector) - { - m_right_injector.reset(); - } - - - bool left_published = false; - if (left_tracked) - { - Haptikos::HandData left_data = m_client.GetData(false, Haptikos::GlobalToWrist, true, true, false); - bool valid_wrist = false; - XrPosef left_controller = oxr_utils::get_aim_pose(*left_tracked, valid_wrist); - - if (left_data.IsValid() == 1 && valid_wrist) + bool left_published = false; + if (left_tracked) { - calculate_hand_pose(left_joints, left_data, left_controller); + Haptikos::HandData left_data = m_client.GetData(false, Haptikos::GlobalToWrist, true, true, false); + bool valid_wrist = false; + XrPosef left_controller = oxr_utils::get_aim_pose(*left_tracked, valid_wrist); - if (!m_left_injector) + if (left_data.IsValid() == 1 && valid_wrist) { - const auto handles = m_session->get_handles(); - m_left_injector = std::make_unique( - handles.instance, handles.session, XR_HAND_LEFT_EXT, handles.space); + calculate_hand_pose(left_joints, left_data, left_controller); + + if (!m_left_pusher) + { + m_left_pusher = std::make_unique( + m_plugin_session->create_hand_tracking_push_channel(XR_HAND_LEFT_EXT)); + } + m_left_pusher->push(left_joints, sample_time_ns); + left_published = true; } - m_left_injector->push(left_joints, time); - left_published = true; } - } - if (!left_published && m_left_injector) + if (!left_published && m_left_pusher) + { + m_left_pusher.reset(); + } + } + catch (const std::exception& e) { - m_left_injector.reset(); + std::cerr << "HaptikosHandsPlugin worker error: " << e.what() << std::endl; + m_left_pusher.reset(); + m_right_pusher.reset(); + std::exit(1); + } + catch (...) + { + std::cerr << "HaptikosHandsPlugin worker error: unknown exception" << std::endl; + m_left_pusher.reset(); + m_right_pusher.reset(); + std::exit(1); } std::this_thread::sleep_until(frame_start + target_frame_duration); diff --git a/src/plugins/haptikos/haptikos_hands_plugin.hpp b/src/plugins/haptikos/haptikos_hands_plugin.hpp index 8bf96a0756..b6723245e7 100644 --- a/src/plugins/haptikos/haptikos_hands_plugin.hpp +++ b/src/plugins/haptikos/haptikos_hands_plugin.hpp @@ -3,16 +3,15 @@ #pragma once -#include #include -#include #include -#include -#include -#include +#include +#include #include #include +#include +#include #include @@ -24,7 +23,9 @@ class HaptikosHandsPlugin { public: - HaptikosHandsPlugin(const std::string& plugin_root_id) noexcept(false); + HaptikosHandsPlugin(const std::string& plugin_root_id, + std::shared_ptr controller_tracker, + core::PluginSessionHandle plugin_session) noexcept(false); ~HaptikosHandsPlugin(); HaptikosHandsPlugin(const HaptikosHandsPlugin&) = delete; @@ -35,15 +36,11 @@ class HaptikosHandsPlugin private: void worker_thread(); - // OpenXR State - std::shared_ptr m_session; - std::unique_ptr m_left_injector; - std::unique_ptr m_right_injector; - std::optional m_time_converter; std::shared_ptr m_controller_tracker; - std::shared_ptr m_hand_tracker; - std::unique_ptr m_deviceio_session; - + core::PluginSessionHandle m_plugin_session; + std::unique_ptr m_pull_channel; + std::unique_ptr m_left_pusher; + std::unique_ptr m_right_pusher; std::string m_root_id; Haptikos::Client m_client; diff --git a/src/plugins/haptikos/main.cpp b/src/plugins/haptikos/main.cpp index 4d011d9f09..f1066c5263 100644 --- a/src/plugins/haptikos/main.cpp +++ b/src/plugins/haptikos/main.cpp @@ -3,12 +3,18 @@ #include "haptikos_hands_plugin.hpp" +#include +#include +#include + #include #include #include #include #include #include +#include +#include using namespace plugins::haptikos; @@ -47,7 +53,13 @@ try std::cout << "Haptikos Hands Plugin" << std::endl; std::cout << "Plugin Root ID: " << plugin_root_id << std::endl; - auto plugin = std::make_unique(plugin_root_id); + auto controller_tracker = std::make_shared(); + auto hand_tracker = std::make_shared(); + std::vector> trackers = { controller_tracker, std::move(hand_tracker) }; + core::PluginSessionHandle session = std::make_shared( + "HaptikosHands", core::PluginSessionRequirements{ .hand_tracking_push = true }, std::move(trackers)); + auto plugin = + std::make_unique(plugin_root_id, std::move(controller_tracker), std::move(session)); std::cout << "Plugin running. Press Ctrl+C to stop." << std::endl; while (!g_stop_requested.load(std::memory_order_relaxed)) diff --git a/src/plugins/manus/app/CMakeLists.txt b/src/plugins/manus/app/CMakeLists.txt index 7aa5d6fea3..c6c5dcdb01 100644 --- a/src/plugins/manus/app/CMakeLists.txt +++ b/src/plugins/manus/app/CMakeLists.txt @@ -8,6 +8,7 @@ add_executable(manus_hand_plugin target_link_libraries(manus_hand_plugin PRIVATE manus_plugin_core + Teleop::plugin_utils ) set_target_properties(manus_hand_plugin PROPERTIES diff --git a/src/plugins/manus/app/main.cpp b/src/plugins/manus/app/main.cpp index 3d542eca83..cdb8f8aeb2 100644 --- a/src/plugins/manus/app/main.cpp +++ b/src/plugins/manus/app/main.cpp @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#include +#include #include +#include #include #include @@ -11,6 +14,7 @@ #include #include #include +#include #include using namespace plugins::manus; @@ -127,7 +131,27 @@ try std::signal(SIGINT, signal_handler); std::signal(SIGTERM, signal_handler); - auto& tracker = ManusTracker::instance(config); + std::shared_ptr haptic_reader; + if (config.haptic) + { + haptic_reader = std::make_shared(MANUS_GLOVE_COLLECTION_ID); + } + + ManusPluginSessionFactory session_factory = [config, haptic_reader] + { + std::vector> trackers; + if (haptic_reader) + { + trackers.push_back(haptic_reader); + } + return std::make_shared( + config.app_name, + core::PluginSessionRequirements{ + .schema_push = config.sensors, .hand_tracking_push = config.human, .wrist_tracking_pull = config.human }, + std::move(trackers)); + }; + + auto& tracker = ManusTracker::instance(config, std::move(session_factory), std::move(haptic_reader)); std::cout << "Plugin running. Press Ctrl+C to stop." << std::endl; diff --git a/src/plugins/manus/core/CMakeLists.txt b/src/plugins/manus/core/CMakeLists.txt index bd0401e881..ea8cf4a05f 100644 --- a/src/plugins/manus/core/CMakeLists.txt +++ b/src/plugins/manus/core/CMakeLists.txt @@ -14,15 +14,10 @@ target_include_directories(manus_plugin_core target_link_libraries(manus_plugin_core PUBLIC ManusSDK::ManusSDK - Teleop::plugin_utils - deviceio::deviceio_session - deviceio::deviceio_trackers - OpenXR::openxr_loader - oxr::oxr_core + deviceio::deviceio_trackers pusherio::pusherio PRIVATE oxr::oxr_utils - Teleop::openxr_extensions isaacteleop_schema ) diff --git a/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp b/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp index 1b2fa8e60b..e717db19ac 100644 --- a/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp +++ b/src/plugins/manus/core/inc/manus/manus_hand_tracking_plugin.hpp @@ -5,31 +5,22 @@ #include "manus_glove_collection.hpp" -#include -#include -#include #include #include -#include -#include -#include +#include +#include #include #include -#include #include #include +#include #include #include #include #include #include -namespace core -{ -class OpenXRSession; -} - namespace plugins { namespace manus @@ -46,17 +37,21 @@ struct ManusPluginConfig std::string app_name = "ManusHandPlugin"; std::string left_calibration_file; std::string right_calibration_file; - bool human = true; // OpenXR HandInjector + bool human = true; // hand tracking push bool sensors = true; // RawDeviceData -> SchemaPusher bool haptic = true; // inbound HapticCommandReaderTracker }; +using ManusPluginSessionFactory = std::function; + class __attribute__((visibility("default"))) ManusTracker { public: /// Get the singleton instance. The first call constructs with ``config``; - /// later calls (e.g. from Manus SDK callbacks) ignore ``config``. - static ManusTracker& instance(const ManusPluginConfig& config = ManusPluginConfig{}) noexcept(false); + /// later calls (e.g. from Manus SDK callbacks) ignore all arguments. + static ManusTracker& instance(const ManusPluginConfig& config = ManusPluginConfig{}, + ManusPluginSessionFactory plugin_session_factory = {}, + std::shared_ptr haptic_reader = {}) noexcept(false); void update(); std::vector get_left_hand_nodes() const; @@ -79,7 +74,9 @@ class __attribute__((visibility("default"))) ManusTracker private: // Lifecycle - explicit ManusTracker(const ManusPluginConfig& config) noexcept(false); + ManusTracker(const ManusPluginConfig& config, + ManusPluginSessionFactory plugin_session_factory, + std::shared_ptr haptic_reader) noexcept(false); ~ManusTracker(); ManusTracker(const ManusTracker&) = delete; @@ -101,16 +98,8 @@ class __attribute__((visibility("default"))) ManusTracker void push_sensor_states(); void push_sensor_side(bool is_left, core::SchemaPusher& pusher); - // OpenXR specific methods + // Hand-publishing methods void inject_hand_data(); - void initialize_xdev_hand_trackers(); - void cleanup_xdev_hand_trackers(); - // Returns true if a valid (POSITION_VALID | ORIENTATION_VALID) wrist pose was - // obtained. out_is_tracked is set to true only when the runtime also reports - // POSITION_TRACKED | ORIENTATION_TRACKED, meaning the pose is actively tracked - // rather than predicted/stale. - bool update_xdev_hand(XrHandTrackerEXT tracker, XrTime time, XrPosef& out_wrist_pose, bool& out_is_tracked); - bool get_controller_wrist_pose(bool is_left, XrPosef& out_wrist_pose); // -- Member Variables -- @@ -140,35 +129,19 @@ class __attribute__((visibility("default"))) ManusTracker std::array, 2> m_sensor_transforms{}; std::array m_sensors_logged_on{ { false, false } }; - // OpenXR State - std::shared_ptr m_session; - core::OpenXRSessionHandles m_handles; - std::unique_ptr m_left_injector; - std::unique_ptr m_right_injector; - std::shared_ptr m_controller_tracker; - std::shared_ptr m_hand_tracker; + // Plugin session state + ManusPluginSessionFactory m_plugin_session_factory; + core::PluginSessionHandle m_plugin_session; + std::unique_ptr m_left_hand_pusher; + std::unique_ptr m_right_hand_pusher; // Inbound HapticCommand tensor; collection identity in // inc/manus/manus_glove_collection.hpp. Read each frame in update(). std::shared_ptr m_haptic_reader; - std::unique_ptr m_deviceio_session; + std::unique_ptr m_pull_channel; + std::unique_ptr m_wrist_tracking_source; std::unique_ptr m_left_sensor_pusher; std::unique_ptr m_right_sensor_pusher; - // XDev native hand trackers (Quest 3 hand tracking via XR_MNDX_xdev_space) - XrXDevListMNDX m_xdev_list = XR_NULL_HANDLE; - XrHandTrackerEXT m_native_left_hand_tracker = XR_NULL_HANDLE; - XrHandTrackerEXT m_native_right_hand_tracker = XR_NULL_HANDLE; - bool m_xdev_available = false; - - // XDev function pointers - PFN_xrCreateXDevListMNDX m_pfn_create_xdev_list = nullptr; - PFN_xrDestroyXDevListMNDX m_pfn_destroy_xdev_list = nullptr; - PFN_xrEnumerateXDevsMNDX m_pfn_enumerate_xdevs = nullptr; - PFN_xrGetXDevPropertiesMNDX m_pfn_get_xdev_properties = nullptr; - PFN_xrCreateHandTrackerEXT m_pfn_create_hand_tracker = nullptr; - PFN_xrDestroyHandTrackerEXT m_pfn_destroy_hand_tracker = nullptr; - PFN_xrLocateHandJointsEXT m_pfn_locate_hand_joints = nullptr; - // Persistent root poses (initialized to identity) XrPosef m_left_root_pose = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; XrPosef m_right_root_pose = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; @@ -180,9 +153,6 @@ class __attribute__((visibility("default"))) ManusTracker // Node topology (parent IDs) — populated once per glove connect std::vector m_left_node_info; std::vector m_right_node_info; - - // Time converter for XR timestamps (initialized after handles are ready) - std::optional m_time_converter; }; } // namespace manus diff --git a/src/plugins/manus/core/manus_hand_tracking_plugin.cpp b/src/plugins/manus/core/manus_hand_tracking_plugin.cpp index 2c72205836..92c84056e7 100644 --- a/src/plugins/manus/core/manus_hand_tracking_plugin.cpp +++ b/src/plugins/manus/core/manus_hand_tracking_plugin.cpp @@ -6,11 +6,9 @@ #include "inc/manus/manus_glove_collection.hpp" #include -#include #include #include #include -#include #include #include #include @@ -29,9 +27,9 @@ #include #include #include +#include #include - namespace plugins { namespace manus @@ -40,25 +38,6 @@ namespace manus namespace { -// Returns true if the OpenXR loader/runtime advertises the given extension. -// xrEnumerateInstanceExtensionProperties is a loader-level function that can be -// called before any XrInstance exists, so this is safe to use at init time. -bool is_openxr_extension_supported(const char* ext_name) -{ - uint32_t count = 0; - if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, 0, &count, nullptr))) - { - return false; - } - std::vector props(count, XrExtensionProperties{ XR_TYPE_EXTENSION_PROPERTIES }); - if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, count, &count, props.data()))) - { - return false; - } - return std::any_of(props.begin(), props.end(), - [ext_name](const XrExtensionProperties& p) { return std::string(p.extensionName) == ext_name; }); -} - SDKReturnCode get_raw_skeleton_node_count(uint32_t glove_id, uint32_t& node_count) { #if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) || defined(_M_ARM) @@ -100,22 +79,24 @@ std::vector read_calibration_file(const std::string& path) static constexpr XrPosef kLeftHandOffset = { { -0.70710678f, -0.5f, 0.0f, 0.5f }, { -0.1f, 0.02f, -0.02f } }; static constexpr XrPosef kRightHandOffset = { { -0.70710678f, 0.5f, 0.0f, 0.5f }, { 0.1f, 0.02f, -0.02f } }; -ManusTracker& ManusTracker::instance(const ManusPluginConfig& config) noexcept(false) +ManusTracker& ManusTracker::instance(const ManusPluginConfig& config, + ManusPluginSessionFactory plugin_session_factory, + std::shared_ptr haptic_reader) noexcept(false) { - static ManusTracker s(config); + static ManusTracker s(config, std::move(plugin_session_factory), std::move(haptic_reader)); return s; } void ManusTracker::update() { - if (!m_deviceio_session) + if (!m_pull_channel) { - // OpenXR unavailable — nothing to update for positioning/injection/push + // The configured plugin session is unavailable, so no session-backed + // pull, positioning, or output can be updated. return; } - // Update DeviceIOSession which handles time conversion and tracker updates internally - m_deviceio_session->update(); + m_pull_channel->update(); // Latest-wins per endpoint: the hardware only retains the most recent // vibration call, so dropping intermediate samples on a slow tick is fine. @@ -127,7 +108,7 @@ void ManusTracker::update() { for (const std::string_view endpoint : { std::string_view("left"), std::string_view("right") }) { - const auto& tracked = m_haptic_reader->get_data(*m_deviceio_session, endpoint); + const auto& tracked = m_haptic_reader->get_data(*m_pull_channel, endpoint); const core::HapticCommand* command = tracked.get(); if (command != nullptr && command->values() != nullptr && command->values()->size() == kManusFingerCount) { @@ -218,7 +199,12 @@ void ManusTracker::apply_haptic_command(bool is_left, const std::array haptic_reader) noexcept(false) + : m_config(config), + m_plugin_session_factory(std::move(plugin_session_factory)), + m_haptic_reader(std::move(haptic_reader)) { initialize(); } @@ -281,10 +267,10 @@ void ManusTracker::initialize() noexcept(false) ConnectToGloves(); - const bool needs_openxr = m_config.human || m_config.sensors || m_config.haptic; - if (!needs_openxr) + const bool needs_plugin_session = m_config.human || m_config.sensors || m_config.haptic; + if (!needs_plugin_session) { - std::cout << "[Manus] No OpenXR datasets enabled; running Manus-only (skeleton callbacks only)." << std::endl; + std::cout << "[Manus] No session datasets enabled; running Manus-only (skeleton callbacks only)." << std::endl; std::lock_guard lock(m_lifecycle_mutex); m_initialized = true; return; @@ -295,140 +281,66 @@ void ManusTracker::initialize() noexcept(false) try { - std::vector> trackers; - - if (m_config.human) + if (!m_plugin_session_factory) { - // Create ControllerTracker unconditionally; HandTracker requires - // XR_EXT_hand_tracking which is optional — only add it when the runtime - // advertises support so xrCreateInstance does not fail with - // XR_ERROR_EXTENSION_NOT_PRESENT on runtimes that lack the extension. - m_controller_tracker = std::make_shared(); - trackers.push_back(m_controller_tracker); - - const bool hand_tracking_supported = is_openxr_extension_supported(XR_EXT_HAND_TRACKING_EXTENSION_NAME); - if (hand_tracking_supported) - { - m_hand_tracker = std::make_shared(); - trackers.push_back(m_hand_tracker); - } - else - { - std::cout << "[Manus] " << XR_EXT_HAND_TRACKING_EXTENSION_NAME - << " is not supported by the current runtime; HandTracker will not be created." << std::endl; - } + throw std::invalid_argument("ManusTracker requires a plugin session factory for enabled session datasets"); } - - if (m_config.haptic) + m_plugin_session = m_plugin_session_factory(); + m_plugin_session_factory = {}; + if (!m_plugin_session) { - // Registering the reader pulls XR_NVX1_tensor_data into the - // OpenXRSession's required-extension set; the session will fail - // loudly on a runtime that doesn't advertise it. The reader's buffer - // must be >= the producer's collection sample size; we use the shared - // default (matching the producer's PushTensorHapticDevice) rather than - // a Manus-specific size that could drift below it. - m_haptic_reader = std::make_shared(MANUS_GLOVE_COLLECTION_ID); - trackers.push_back(m_haptic_reader); + throw std::runtime_error("The plugin session factory returned no session"); } - - std::vector extensions; - if (!trackers.empty()) + if (m_config.haptic && !m_haptic_reader) { - extensions = core::DeviceIOSession::get_required_extensions(trackers); + throw std::invalid_argument("ManusTracker requires a haptic reader when haptic input is enabled"); } - if (m_config.sensors) - { - for (const auto& ext : core::SchemaPusher::get_required_extensions()) - { - if (std::find(extensions.begin(), extensions.end(), ext) == extensions.end()) - { - extensions.push_back(ext); - } - } - } - - if (m_config.human) - { - extensions.push_back(XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME); - } - - // XR_MNDX_XDEV_SPACE_EXTENSION_NAME is optional: it enables optical (HMD) hand - // tracking as a higher-quality wrist source. If the runtime does not advertise - // it we fall back to controller-based tracking instead of crashing. - bool xdev_extension_supported = false; if (m_config.human) { - xdev_extension_supported = is_openxr_extension_supported(XR_MNDX_XDEV_SPACE_EXTENSION_NAME); - if (xdev_extension_supported) - { - extensions.push_back(XR_MNDX_XDEV_SPACE_EXTENSION_NAME); - } - else - { - std::cout << "[Manus] " << XR_MNDX_XDEV_SPACE_EXTENSION_NAME - << " is not supported by the current runtime; optical hand tracking" - << " will not be available and controller fallback will be used." << std::endl; - } - } - - // Create session with required extensions - constructor automatically begins the session - m_session = std::make_shared(m_config.app_name, extensions); - m_handles = m_session->get_handles(); - - // Initialize time converter now that handles are ready - m_time_converter.emplace(m_handles); - - if (m_config.human) - { - m_left_injector = std::make_unique( - m_handles.instance, m_handles.session, XR_HAND_LEFT_EXT, m_handles.space); - m_right_injector = std::make_unique( - m_handles.instance, m_handles.session, XR_HAND_RIGHT_EXT, m_handles.space); + m_left_hand_pusher = std::make_unique( + m_plugin_session->create_hand_tracking_push_channel(XR_HAND_LEFT_EXT)); + m_right_hand_pusher = std::make_unique( + m_plugin_session->create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT)); } if (m_config.sensors) { - m_left_sensor_pusher = std::make_unique( - m_handles, core::SchemaPusherConfig{ .collection_id = MANUS_SENSORS_LEFT_COLLECTION_ID, - .max_flatbuffer_size = kSensorFlatbufferSize, - .tensor_identifier = "joint_state", - .localized_name = "Manus Sensors Left", - .app_name = m_config.app_name }); - m_right_sensor_pusher = std::make_unique( - m_handles, core::SchemaPusherConfig{ .collection_id = MANUS_SENSORS_RIGHT_COLLECTION_ID, - .max_flatbuffer_size = kSensorFlatbufferSize, - .tensor_identifier = "joint_state", - .localized_name = "Manus Sensors Right", - .app_name = m_config.app_name }); - } - - if (!trackers.empty()) - { - m_deviceio_session = core::DeviceIOSession::run(trackers, m_handles); - } - else - { - // Sensors-only: still need a DeviceIOSession clock for update(); use an empty tracker list. - m_deviceio_session = core::DeviceIOSession::run({}, m_handles); + m_left_sensor_pusher = std::make_unique(m_plugin_session->create_schema_push_channel( + core::SchemaPusherConfig{ .collection_id = MANUS_SENSORS_LEFT_COLLECTION_ID, + .max_flatbuffer_size = kSensorFlatbufferSize, + .tensor_identifier = "joint_state", + .localized_name = "Manus Sensors Left", + .app_name = m_config.app_name })); + m_right_sensor_pusher = std::make_unique(m_plugin_session->create_schema_push_channel( + core::SchemaPusherConfig{ .collection_id = MANUS_SENSORS_RIGHT_COLLECTION_ID, + .max_flatbuffer_size = kSensorFlatbufferSize, + .tensor_identifier = "joint_state", + .localized_name = "Manus Sensors Right", + .app_name = m_config.app_name })); } - // Only attempt XDev hand tracker setup when the extension was actually enabled. - // Skipping here avoids calling xrGetInstanceProcAddr for MNDX entry points that - // the runtime would not have loaded. - if (xdev_extension_supported) + m_pull_channel = m_plugin_session->create_pull_channel(); + if (!m_pull_channel) { - initialize_xdev_hand_trackers(); + throw std::runtime_error("The plugin session could not create a pull channel"); } if (m_config.human) { - std::cout << "[Manus] Initialized with wrist source: " << (m_xdev_available ? "HandTracking" : "Controllers") - << std::endl; + m_wrist_tracking_source = m_pull_channel->create_wrist_tracking_source( + core::WristTrackingSourceConfig{ .mode = core::WristTrackingSourceMode::Auto, + .left_aim_to_wrist = kLeftHandOffset, + .right_aim_to_wrist = kRightHandOffset }); + if (!m_wrist_tracking_source) + { + throw std::runtime_error("The plugin session could not provide a wrist tracking source"); + } + std::cout << "[Manus] Wrist tracking source initialized" << std::endl; } else { - std::cout << "[Manus] OpenXR session ready (human injection disabled)." << std::endl; + std::cout << "[Manus] Plugin session ready (human injection disabled)." << std::endl; } success = true; @@ -440,22 +352,19 @@ void ManusTracker::initialize() noexcept(false) if (!success) { - std::cerr << "[Manus] Warning: OpenXR initialization failed: " << error_msg << std::endl; - std::cerr << "[Manus] Continuing in Manus-only mode (no hand injection, sensor push, or OpenXR positioning)." + std::cerr << "[Manus] Warning: plugin session initialization failed: " << error_msg << std::endl; + std::cerr << "[Manus] Continuing in Manus-only mode (no hand injection, sensor push, or session positioning)." << std::endl; - // Drop every OpenXR-related member that may have been created before the - // throw (trackers/injectors first — they may hold session handles). - cleanup_xdev_hand_trackers(); - m_left_injector.reset(); - m_right_injector.reset(); + // Drop session-created objects before the session that owns their transport. + m_wrist_tracking_source.reset(); + m_left_hand_pusher.reset(); + m_right_hand_pusher.reset(); m_left_sensor_pusher.reset(); m_right_sensor_pusher.reset(); - m_controller_tracker.reset(); - m_hand_tracker.reset(); m_haptic_reader.reset(); - m_deviceio_session.reset(); - m_time_converter.reset(); - m_session.reset(); + m_pull_channel.reset(); + m_plugin_session.reset(); + m_plugin_session_factory = {}; } std::lock_guard lock(m_lifecycle_mutex); @@ -465,9 +374,6 @@ void ManusTracker::initialize() noexcept(false) void ManusTracker::shutdown_sdk() { - // Cleanup XDev hand trackers first - cleanup_xdev_hand_trackers(); - CoreSdk_RegisterCallbackForRawSkeletonStream(nullptr); CoreSdk_RegisterCallbackForLandscapeStream(nullptr); CoreSdk_RegisterCallbackForErgonomicsStream(nullptr); @@ -848,231 +754,6 @@ void ManusTracker::push_sensor_side(bool is_left, core::SchemaPusher& pusher) pusher.push_buffer(builder.GetBufferPointer(), builder.GetSize(), sample_time_ns, sample_time_ns); } -void ManusTracker::initialize_xdev_hand_trackers() -{ - // Load XDev extension function pointers - auto load_func = [this](const char* name, PFN_xrVoidFunction* ptr) -> bool - { - XrResult result = m_handles.xrGetInstanceProcAddr(m_handles.instance, name, ptr); - return XR_SUCCEEDED(result) && *ptr != nullptr; - }; - - // Load XDev extension functions - if (!load_func("xrCreateXDevListMNDX", reinterpret_cast(&m_pfn_create_xdev_list)) || - !load_func("xrDestroyXDevListMNDX", reinterpret_cast(&m_pfn_destroy_xdev_list)) || - !load_func("xrEnumerateXDevsMNDX", reinterpret_cast(&m_pfn_enumerate_xdevs)) || - !load_func("xrGetXDevPropertiesMNDX", reinterpret_cast(&m_pfn_get_xdev_properties))) - { - std::cerr << "[Manus] XR_MNDX_xdev_space extension not available, falling back to controllers" << std::endl; - return; - } - - // Load hand tracking extension functions - if (!load_func("xrCreateHandTrackerEXT", reinterpret_cast(&m_pfn_create_hand_tracker)) || - !load_func("xrDestroyHandTrackerEXT", reinterpret_cast(&m_pfn_destroy_hand_tracker)) || - !load_func("xrLocateHandJointsEXT", reinterpret_cast(&m_pfn_locate_hand_joints))) - { - std::cerr << "[Manus] Hand tracking extension not available, falling back to controllers" << std::endl; - return; - } - - // Create XDev list - XrCreateXDevListInfoMNDX create_info{ XR_TYPE_CREATE_XDEV_LIST_INFO_MNDX }; - XrResult result = m_pfn_create_xdev_list(m_handles.session, &create_info, &m_xdev_list); - if (XR_FAILED(result)) - { - std::cerr << "[Manus] Failed to create XDevList, falling back to controllers" << std::endl; - return; - } - - // Enumerate XDevs - uint32_t xdev_count = 0; - result = m_pfn_enumerate_xdevs(m_xdev_list, 0, &xdev_count, nullptr); - if (XR_FAILED(result) || xdev_count == 0) - { - std::cerr << "[Manus] No XDevs found, falling back to controllers" << std::endl; - return; - } - - std::vector xdev_ids(xdev_count); - result = m_pfn_enumerate_xdevs(m_xdev_list, xdev_count, &xdev_count, xdev_ids.data()); - if (XR_FAILED(result)) - { - return; - } - - // Find native hand tracking devices by matching against their serial strings. - // - // NOTE: The serial values "Head Device (0)" (left) and "Head Device (1)" (right) are - // NOT defined by the XR_MNDX_xdev_space specification. They are an observed runtime- - // specific naming convention (e.g. Monado). If a runtime changes these display names - // across firmware or software updates the match below will silently fail. - // See: https://registry.khronos.org/OpenXR/specs/1.0/html/xrspec.html (XR_MNDX_xdev_space) - XrXDevIdMNDX left_xdev_id = 0; - XrXDevIdMNDX right_xdev_id = 0; - std::vector seen_serials; - - for (const auto& xdev_id : xdev_ids) - { - XrGetXDevInfoMNDX get_info{ XR_TYPE_GET_XDEV_INFO_MNDX }; - get_info.id = xdev_id; - - XrXDevPropertiesMNDX properties{ XR_TYPE_XDEV_PROPERTIES_MNDX }; - result = m_pfn_get_xdev_properties(m_xdev_list, &get_info, &properties); - if (XR_FAILED(result)) - { - continue; - } - - std::string serial_str = properties.serial ? properties.serial : ""; - seen_serials.push_back(serial_str); - - if (serial_str == "Head Device (0)") - { - left_xdev_id = xdev_id; - } - else if (serial_str == "Head Device (1)") - { - right_xdev_id = xdev_id; - } - } - - if (left_xdev_id == 0 || right_xdev_id == 0) - { - std::string serials_list; - for (const auto& s : seen_serials) - { - if (!serials_list.empty()) - serials_list += ", "; - serials_list += '"'; - serials_list += s; - serials_list += '"'; - } - std::cerr << "[Manus] Could not match optical hand-tracking XDevs by serial. " - << "Expected \"Head Device (0)\" (left) and \"Head Device (1)\" (right), " - << "but found: [" << serials_list << "]. " - << "These serial strings are runtime-specific and may have changed." << std::endl; - } - - // Create hand trackers from XDevs - auto create_tracker = [this](XrXDevIdMNDX xdev_id, XrHandEXT hand, XrHandTrackerEXT& out_tracker) -> bool - { - if (xdev_id == 0) - { - return false; - } - - XrCreateHandTrackerXDevMNDX xdev_create_info{ XR_TYPE_CREATE_HAND_TRACKER_XDEV_MNDX }; - xdev_create_info.xdevList = m_xdev_list; - xdev_create_info.id = xdev_id; - - XrHandTrackerCreateInfoEXT create_info{ XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT }; - create_info.next = &xdev_create_info; - create_info.hand = hand; - create_info.handJointSet = XR_HAND_JOINT_SET_DEFAULT_EXT; - - return XR_SUCCEEDED(m_pfn_create_hand_tracker(m_handles.session, &create_info, &out_tracker)); - }; - - bool left_ok = create_tracker(left_xdev_id, XR_HAND_LEFT_EXT, m_native_left_hand_tracker); - bool right_ok = create_tracker(right_xdev_id, XR_HAND_RIGHT_EXT, m_native_right_hand_tracker); - - if (left_ok && right_ok) - { - m_xdev_available = true; - } - else - { - std::cerr << "[Manus] Failed to create native hand trackers, falling back to controllers" << std::endl; - cleanup_xdev_hand_trackers(); - } -} - -void ManusTracker::cleanup_xdev_hand_trackers() -{ - if (m_native_left_hand_tracker != XR_NULL_HANDLE && m_pfn_destroy_hand_tracker) - { - m_pfn_destroy_hand_tracker(m_native_left_hand_tracker); - m_native_left_hand_tracker = XR_NULL_HANDLE; - } - if (m_native_right_hand_tracker != XR_NULL_HANDLE && m_pfn_destroy_hand_tracker) - { - m_pfn_destroy_hand_tracker(m_native_right_hand_tracker); - m_native_right_hand_tracker = XR_NULL_HANDLE; - } - if (m_xdev_list != XR_NULL_HANDLE && m_pfn_destroy_xdev_list) - { - m_pfn_destroy_xdev_list(m_xdev_list); - m_xdev_list = XR_NULL_HANDLE; - } - m_xdev_available = false; -} - -bool ManusTracker::update_xdev_hand(XrHandTrackerEXT tracker, XrTime time, XrPosef& out_wrist_pose, bool& out_is_tracked) -{ - out_is_tracked = false; - - if (tracker == XR_NULL_HANDLE || !m_pfn_locate_hand_joints || time == 0) - { - return false; - } - - XrHandJointsLocateInfoEXT locate_info{ XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT }; - locate_info.baseSpace = m_handles.space; - locate_info.time = time; - - XrHandJointLocationEXT joint_locations[XR_HAND_JOINT_COUNT_EXT]; - - XrHandJointLocationsEXT locations{ XR_TYPE_HAND_JOINT_LOCATIONS_EXT }; - locations.jointCount = XR_HAND_JOINT_COUNT_EXT; - locations.jointLocations = joint_locations; - - XrResult result = m_pfn_locate_hand_joints(tracker, &locate_info, &locations); - if (XR_FAILED(result) || !locations.isActive) - { - return false; - } - - const auto& wrist = joint_locations[XR_HAND_JOINT_WRIST_EXT]; - const bool is_valid = (wrist.locationFlags & XR_SPACE_LOCATION_POSITION_VALID_BIT) && - (wrist.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT); - - if (is_valid) - { - out_wrist_pose = wrist.pose; - // Distinguish actively tracked from valid-but-predicted/stale poses so - // callers can advertise TRACKED bits only when the runtime confirms it. - out_is_tracked = (wrist.locationFlags & XR_SPACE_LOCATION_POSITION_TRACKED_BIT) && - (wrist.locationFlags & XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT); - return true; - } - - return false; -} - -bool ManusTracker::get_controller_wrist_pose(bool is_left, XrPosef& out_wrist_pose) -{ - const auto& tracked = is_left ? m_controller_tracker->get_left_controller(*m_deviceio_session) : - m_controller_tracker->get_right_controller(*m_deviceio_session); - - if (!tracked) - { - return false; - } - - bool aim_valid = false; - XrPosef raw_pose = oxr_utils::get_aim_pose(*tracked, aim_valid); - - if (!aim_valid) - { - return false; - } - - XrPosef offset_pose = is_left ? kLeftHandOffset : kRightHandOffset; - out_wrist_pose = oxr_utils::multiply_poses(raw_pose, offset_pose); - return true; -} - void ManusTracker::inject_hand_data() { std::vector left_nodes; @@ -1084,8 +765,7 @@ void ManusTracker::inject_hand_data() right_nodes = m_right_hand_nodes; } - // Get current XrTime from the system monotonic clock - XrTime time = m_time_converter->os_monotonic_now(); + const int64_t sample_time_ns = core::os_monotonic_now_ns(); auto process_hand = [&](const std::vector& nodes, bool is_left) { @@ -1098,46 +778,18 @@ void ManusTracker::inject_hand_data() XrPosef root_pose = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; bool is_root_tracked = false; - // Get wrist pose - auto-select hand tracking or controllers - XrPosef wrist_pose; - bool xdev_pose_valid = false; - if (m_xdev_available) - { - XrHandTrackerEXT tracker = is_left ? m_native_left_hand_tracker : m_native_right_hand_tracker; - bool xdev_tracked = false; - if (update_xdev_hand(tracker, time, wrist_pose, xdev_tracked)) - { - // Cache the pose (valid even when only predicted/stale) so the - // last good pose is available if tracking is briefly interrupted. - if (is_left) - { - m_left_root_pose = wrist_pose; - } - else - { - m_right_root_pose = wrist_pose; - } - // Only mark as tracked when the runtime confirms active tracking; - // a valid-but-untracked pose must not have TRACKED bits set. - is_root_tracked = xdev_tracked; - xdev_pose_valid = true; - } - } - - // Fall back to controllers only when xdev provided no valid pose at all. - // If xdev gave a valid-but-untracked pose we keep it rather than - // overwriting it with a controller pose that would be falsely marked tracked. - if (!xdev_pose_valid && get_controller_wrist_pose(is_left, wrist_pose)) + const core::WristTrackingSample wrist = m_wrist_tracking_source->query(is_left, sample_time_ns); + if (wrist.valid) { if (is_left) { - m_left_root_pose = wrist_pose; + m_left_root_pose = wrist.pose; } else { - m_right_root_pose = wrist_pose; + m_right_root_pose = wrist.pose; } - is_root_tracked = true; + is_root_tracked = wrist.tracked; } root_pose = is_left ? m_left_root_pose : m_right_root_pose; @@ -1201,11 +853,11 @@ void ManusTracker::inject_hand_data() if (is_left) { - m_left_injector->push(joints, time); + m_left_hand_pusher->push(joints, sample_time_ns); } else { - m_right_injector->push(joints, time); + m_right_hand_pusher->push(joints, sample_time_ns); } }; diff --git a/src/plugins/manus/tools/manus_hand_tracker_printer.cpp b/src/plugins/manus/tools/manus_hand_tracker_printer.cpp index 2d053f9a70..81bd310e78 100644 --- a/src/plugins/manus/tools/manus_hand_tracker_printer.cpp +++ b/src/plugins/manus/tools/manus_hand_tracker_printer.cpp @@ -22,6 +22,9 @@ try plugins::manus::ManusPluginConfig config; config.app_name = "ManusHandPrinter"; + config.human = false; + config.sensors = false; + config.haptic = false; auto& tracker = plugins::manus::ManusTracker::instance(config); // Start Vulkan visualizer in a background thread. diff --git a/src/plugins/noitom_mocap/CMakeLists.txt b/src/plugins/noitom_mocap/CMakeLists.txt index 9c9e1f4038..2d4641902e 100644 --- a/src/plugins/noitom_mocap/CMakeLists.txt +++ b/src/plugins/noitom_mocap/CMakeLists.txt @@ -100,10 +100,10 @@ add_executable(noitom_mocap_plugin ) target_link_libraries(noitom_mocap_plugin PRIVATE + Teleop::plugin_utils NoitomMocapApi::MocapApi deviceio::deviceio_trackers pusherio::pusherio - oxr::oxr_core isaacteleop_schema ) diff --git a/src/plugins/noitom_mocap/main.cpp b/src/plugins/noitom_mocap/main.cpp index 58b43f35f6..b8f58293ea 100644 --- a/src/plugins/noitom_mocap/main.cpp +++ b/src/plugins/noitom_mocap/main.cpp @@ -3,14 +3,18 @@ #include "noitom_mocap_plugin.hpp" +#include + #include #include #include #include #include +#include #include #include #include +#include using namespace plugins::noitom_mocap; @@ -142,7 +146,9 @@ try std::signal(SIGINT, signal_handler); std::signal(SIGTERM, signal_handler); - NoitomMocapPlugin plugin(config); + core::PluginSessionHandle session = std::make_shared( + "NoitomMocapPlugin", core::PluginSessionRequirements{ .schema_push = true }); + NoitomMocapPlugin plugin(std::move(config), std::move(session)); const auto frame_duration = std::chrono::nanoseconds(static_cast(1000000000.0 / rate_hz)); const auto program_start = std::chrono::steady_clock::now(); diff --git a/src/plugins/noitom_mocap/noitom_mocap_plugin.cpp b/src/plugins/noitom_mocap/noitom_mocap_plugin.cpp index bc1bbfbaf0..843ba9da08 100644 --- a/src/plugins/noitom_mocap/noitom_mocap_plugin.cpp +++ b/src/plugins/noitom_mocap/noitom_mocap_plugin.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include @@ -202,10 +201,14 @@ void warn_optional_ptp_missing_once() } // namespace -NoitomMocapPlugin::NoitomMocapPlugin(NoitomMocapPluginConfig config) - : config_(std::move(config)), - session_(std::make_shared("NoitomMocapPlugin", core::SchemaPusher::get_required_extensions())) +NoitomMocapPlugin::NoitomMocapPlugin(NoitomMocapPluginConfig config, core::PluginSessionHandle session) + : config_(std::move(config)), session_(std::move(session)) { + if (!session_) + { + throw std::invalid_argument("NoitomMocapPlugin requires a plugin session"); + } + initialize_mocap(); } @@ -600,14 +603,14 @@ void NoitomMocapPlugin::ensure_pusher(size_t flatbuffer_size) return; } - // Keep the OpenXR tensor collection stable. SchemaPusher pads smaller samples + // Keep the tensor collection stable. A fixed-size transport may pad smaller samples // to max_flatbuffer_size before publishing. - pusher_ = std::make_unique( - session_->get_handles(), core::SchemaPusherConfig{ .collection_id = config_.collection_id, - .max_flatbuffer_size = config_.max_flatbuffer_size, - .tensor_identifier = std::string(FULL_BODY_TENSOR_IDENTIFIER), - .localized_name = "Noitom Full Body", - .app_name = "NoitomMocapPlugin" }); + pusher_ = std::make_unique(session_->create_schema_push_channel( + core::SchemaPusherConfig{ .collection_id = config_.collection_id, + .max_flatbuffer_size = config_.max_flatbuffer_size, + .tensor_identifier = std::string(FULL_BODY_TENSOR_IDENTIFIER), + .localized_name = "Noitom Full Body", + .app_name = "NoitomMocapPlugin" })); std::cout << "NoitomMocapPlugin: push tensor sample size set to " << config_.max_flatbuffer_size << " bytes" << std::endl; } diff --git a/src/plugins/noitom_mocap/noitom_mocap_plugin.hpp b/src/plugins/noitom_mocap/noitom_mocap_plugin.hpp index 19ab6d89e1..b909eb46f4 100644 --- a/src/plugins/noitom_mocap/noitom_mocap_plugin.hpp +++ b/src/plugins/noitom_mocap/noitom_mocap_plugin.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include @@ -13,11 +14,6 @@ #include #include -namespace core -{ -class OpenXRSession; -} - namespace plugins { namespace noitom_mocap @@ -46,7 +42,7 @@ struct NoitomMocapPluginConfig class NoitomMocapPlugin { public: - explicit NoitomMocapPlugin(NoitomMocapPluginConfig config); + NoitomMocapPlugin(NoitomMocapPluginConfig config, core::PluginSessionHandle session); ~NoitomMocapPlugin(); // Returns false when the Noitom SDK connection is lost (caller should exit). @@ -65,7 +61,7 @@ class NoitomMocapPlugin void push_frame(int64_t sample_time_local_common_clock_ns, int64_t sample_time_raw_device_clock_ns); NoitomMocapPluginConfig config_; - std::shared_ptr session_; + core::PluginSessionHandle session_; std::unique_ptr pusher_; MocapApi::IMCPSettings* settings_api_ = nullptr; diff --git a/src/plugins/oak/CMakeLists.txt b/src/plugins/oak/CMakeLists.txt index 8e43feaa98..8a83ac13a8 100644 --- a/src/plugins/oak/CMakeLists.txt +++ b/src/plugins/oak/CMakeLists.txt @@ -222,7 +222,7 @@ target_link_libraries(camera_plugin_oak depthai::core isaacteleop_schema mcap::mcap - oxr::oxr_core + Teleop::plugin_utils pusherio::pusherio SDL2::SDL2-static ) diff --git a/src/plugins/oak/core/frame_sink.cpp b/src/plugins/oak/core/frame_sink.cpp index 7c6dbe7cfe..eb87f62ef8 100644 --- a/src/plugins/oak/core/frame_sink.cpp +++ b/src/plugins/oak/core/frame_sink.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -15,6 +14,7 @@ #include #include #include +#include namespace plugins { @@ -59,19 +59,23 @@ void FrameSink::on_frame(const OakFrame& frame) class SchemaMetadataPusher : public IMetadataPusher { public: - SchemaMetadataPusher(const std::vector& streams, const std::string& collection_prefix) - : m_oxr_session( - std::make_shared("OakCameraPlugin", core::SchemaPusher::get_required_extensions())) + SchemaMetadataPusher(const std::vector& streams, + const std::string& collection_prefix, + core::PluginSessionHandle session) + : m_session(std::move(session)) { + if (!m_session) + throw std::invalid_argument("SchemaMetadataPusher requires a plugin session"); + for (const auto& config : streams) { auto collection_id = collection_prefix + "/" + core::EnumNameStreamType(config.camera); - m_pushers[config.camera] = std::make_unique( - m_oxr_session->get_handles(), core::SchemaPusherConfig{ .collection_id = collection_id, - .max_flatbuffer_size = MAX_FLATBUFFER_SIZE, - .tensor_identifier = "frame_metadata", - .localized_name = "Frame Metadata Pusher", - .app_name = "OakCameraPlugin" }); + m_pushers[config.camera] = std::make_unique(m_session->create_schema_push_channel( + core::SchemaPusherConfig{ .collection_id = collection_id, + .max_flatbuffer_size = MAX_FLATBUFFER_SIZE, + .tensor_identifier = "frame_metadata", + .localized_name = "Frame Metadata Pusher", + .app_name = "OakCameraPlugin" })); std::cout << " Metadata: " << collection_id << std::endl; } } @@ -97,7 +101,7 @@ class SchemaMetadataPusher : public IMetadataPusher private: static constexpr size_t MAX_FLATBUFFER_SIZE = 128; - std::shared_ptr m_oxr_session; + core::PluginSessionHandle m_session; std::map> m_pushers; }; @@ -193,7 +197,8 @@ class McapMetadataPusher : public IMetadataPusher std::unique_ptr create_frame_sink(const std::vector& streams, const std::string& collection_prefix, - const std::string& mcap_filename) + const std::string& mcap_filename, + core::PluginSessionHandle session) { if (!collection_prefix.empty() && !mcap_filename.empty()) throw std::runtime_error("Cannot specify both --collection-prefix and --mcap-filename"); @@ -201,7 +206,7 @@ std::unique_ptr create_frame_sink(const std::vector& st std::unique_ptr pusher; if (!collection_prefix.empty()) - pusher = std::make_unique(streams, collection_prefix); + pusher = std::make_unique(streams, collection_prefix, std::move(session)); else if (!mcap_filename.empty()) pusher = std::make_unique(streams, mcap_filename); diff --git a/src/plugins/oak/core/frame_sink.hpp b/src/plugins/oak/core/frame_sink.hpp index 7c898e7355..e10d4211df 100644 --- a/src/plugins/oak/core/frame_sink.hpp +++ b/src/plugins/oak/core/frame_sink.hpp @@ -6,6 +6,8 @@ #include "oak_camera.hpp" #include "rawdata_writer.hpp" +#include + #include #include #include @@ -60,7 +62,8 @@ class FrameSink */ std::unique_ptr create_frame_sink(const std::vector& streams, const std::string& collection_prefix, - const std::string& mcap_filename); + const std::string& mcap_filename, + core::PluginSessionHandle session); } // namespace oak } // namespace plugins diff --git a/src/plugins/oak/main.cpp b/src/plugins/oak/main.cpp index 62510d8fbb..8f31701184 100644 --- a/src/plugins/oak/main.cpp +++ b/src/plugins/oak/main.cpp @@ -4,13 +4,17 @@ #include "core/frame_sink.hpp" #include "core/oak_camera.hpp" +#include + #include #include #include #include #include +#include #include #include +#include using namespace plugins::oak; @@ -202,7 +206,13 @@ try std::cout << "OAK Camera Plugin Starting" << std::endl; std::cout << "============================================================" << std::endl; - OakCamera camera(camera_config, stream_configs, create_frame_sink(stream_configs, collection_prefix, mcap_filename)); + core::PluginSessionHandle session; + if (!collection_prefix.empty()) + session = std::make_shared( + "OakCameraPlugin", core::PluginSessionRequirements{ .schema_push = true }); + + OakCamera camera(camera_config, stream_configs, + create_frame_sink(stream_configs, collection_prefix, mcap_filename, std::move(session))); std::cout << "------------------------------------------------------------" << std::endl; std::cout << "Running capture loop. Press Ctrl+C to stop." << std::endl; diff --git a/src/plugins/oglo_tactile/CMakeLists.txt b/src/plugins/oglo_tactile/CMakeLists.txt index 5dfd00cffc..e0ab3f0f26 100644 --- a/src/plugins/oglo_tactile/CMakeLists.txt +++ b/src/plugins/oglo_tactile/CMakeLists.txt @@ -56,8 +56,8 @@ add_executable(oglo_tactile_plugin target_include_directories(oglo_tactile_plugin PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(oglo_tactile_plugin PRIVATE + Teleop::plugin_utils pusherio::pusherio - oxr::oxr_core isaacteleop_schema nlohmann_json::nlohmann_json PkgConfig::DBUS diff --git a/src/plugins/oglo_tactile/main.cpp b/src/plugins/oglo_tactile/main.cpp index 1048c3b6f8..a3359e9c5b 100644 --- a/src/plugins/oglo_tactile/main.cpp +++ b/src/plugins/oglo_tactile/main.cpp @@ -3,11 +3,15 @@ #include "oglo_tactile_plugin.hpp" +#include + #include #include #include #include +#include #include +#include using namespace plugins::oglo_tactile; @@ -120,7 +124,9 @@ try << "OGLO Tactile Glove Plugin (" << to_string(opts.side) << ")\n" << "============================================================" << std::endl; - OgloTactilePlugin plugin(std::move(opts)); + core::PluginSessionHandle session = std::make_shared( + "OgloTactilePlugin", core::PluginSessionRequirements{ .schema_push = true }); + OgloTactilePlugin plugin(std::move(opts), std::move(session)); plugin.run(g_stop); return 0; diff --git a/src/plugins/oglo_tactile/oglo_glove_sink.cpp b/src/plugins/oglo_tactile/oglo_glove_sink.cpp index 79a4707c44..1e3f4dd05c 100644 --- a/src/plugins/oglo_tactile/oglo_glove_sink.cpp +++ b/src/plugins/oglo_tactile/oglo_glove_sink.cpp @@ -4,7 +4,6 @@ #include "oglo_glove_sink.hpp" #include -#include #include #include @@ -38,6 +37,21 @@ core::OgloGloveSampleT to_native(const GloveSample& s) return out; } +std::unique_ptr make_push_channel(const core::PluginSessionHandle& session, + Side side, + const std::string& collection_prefix) +{ + if (!session) + throw std::invalid_argument("SchemaPusherGloveSink requires a plugin session"); + + return session->create_schema_push_channel( + core::SchemaPusherConfig{ .collection_id = collection_prefix + "/" + to_string(side), + .max_flatbuffer_size = kMaxFlatbufferSize, + .tensor_identifier = "oglo_tactile", + .localized_name = "OGLO Tactile Glove", + .app_name = "OgloTactilePlugin" }); +} + // ============================================================================= // OpenXR SchemaPusher sink (read by a host tracker into a shared session MCAP) // ============================================================================= @@ -45,15 +59,8 @@ core::OgloGloveSampleT to_native(const GloveSample& s) class SchemaPusherGloveSink final : public IGloveSink { public: - SchemaPusherGloveSink(Side side, const std::string& collection_prefix) - : m_session(std::make_shared( - "OgloTactilePlugin", core::SchemaPusher::get_required_extensions())), - m_pusher(m_session->get_handles(), - core::SchemaPusherConfig{ .collection_id = collection_prefix + "/" + to_string(side), - .max_flatbuffer_size = kMaxFlatbufferSize, - .tensor_identifier = "oglo_tactile", - .localized_name = "OGLO Tactile Glove", - .app_name = "OgloTactilePlugin" }) + SchemaPusherGloveSink(Side side, const std::string& collection_prefix, core::PluginSessionHandle session) + : m_session(std::move(session)), m_pusher(make_push_channel(m_session, side, collection_prefix)) { std::cout << "Pushing collection: " << collection_prefix << "/" << to_string(side) << std::endl; } @@ -68,18 +75,20 @@ class SchemaPusherGloveSink final : public IGloveSink } private: - std::shared_ptr m_session; + core::PluginSessionHandle m_session; core::SchemaPusher m_pusher; }; } // namespace -std::unique_ptr create_glove_sink(Side side, const std::string& collection_prefix) +std::unique_ptr create_glove_sink(Side side, + const std::string& collection_prefix, + core::PluginSessionHandle session) { if (collection_prefix.empty()) throw std::runtime_error("OGLO: --collection-prefix is required"); - return std::make_unique(side, collection_prefix); + return std::make_unique(side, collection_prefix, std::move(session)); } } // namespace oglo_tactile diff --git a/src/plugins/oglo_tactile/oglo_glove_sink.hpp b/src/plugins/oglo_tactile/oglo_glove_sink.hpp index fe3059e042..ef0ba42587 100644 --- a/src/plugins/oglo_tactile/oglo_glove_sink.hpp +++ b/src/plugins/oglo_tactile/oglo_glove_sink.hpp @@ -6,6 +6,8 @@ #include "oglo_config.hpp" #include "oglo_packet_parser.hpp" +#include + #include #include #include @@ -36,7 +38,9 @@ class IGloveSink //! @param collection_prefix OpenXR collection prefix (pushes @c PREFIX/left or //! @c PREFIX/right). //! @throws std::runtime_error if @p collection_prefix is empty. -std::unique_ptr create_glove_sink(Side side, const std::string& collection_prefix); +std::unique_ptr create_glove_sink(Side side, + const std::string& collection_prefix, + core::PluginSessionHandle session); } // namespace oglo_tactile } // namespace plugins diff --git a/src/plugins/oglo_tactile/oglo_tactile_plugin.cpp b/src/plugins/oglo_tactile/oglo_tactile_plugin.cpp index e2fea6e424..aaa26479d9 100644 --- a/src/plugins/oglo_tactile/oglo_tactile_plugin.cpp +++ b/src/plugins/oglo_tactile/oglo_tactile_plugin.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -21,8 +22,12 @@ constexpr int64_t kNsPerUs = 1000; constexpr uint64_t kDeviceUsWrap = (1ull << 32); // device_time_us is uint32 } // namespace -OgloTactilePlugin::OgloTactilePlugin(Options options) : m_opts(std::move(options)) +OgloTactilePlugin::OgloTactilePlugin(Options options, core::PluginSessionHandle session) + : m_opts(std::move(options)), m_session(std::move(session)) { + if (!m_session) + throw std::invalid_argument("OgloTactilePlugin requires a plugin session"); + m_ble = make_ble_client(m_opts.device_name_override); m_ble->on_state_change([this](bool connected) { m_connected.store(connected, std::memory_order_relaxed); }); } @@ -68,7 +73,7 @@ void OgloTactilePlugin::connect_and_subscribe() // The sink is created once (first connect); reconnects reuse it so the // OpenXR collection stays continuous across drops. if (!m_sink) - m_sink = create_glove_sink(m_opts.side, m_opts.collection_prefix); + m_sink = create_glove_sink(m_opts.side, m_opts.collection_prefix, m_session); m_ble->subscribe([this](const uint8_t* data, std::size_t len) { on_notify(data, len); }); m_last_notify_ns.store(core::os_monotonic_now_ns(), std::memory_order_relaxed); diff --git a/src/plugins/oglo_tactile/oglo_tactile_plugin.hpp b/src/plugins/oglo_tactile/oglo_tactile_plugin.hpp index f7dfbf85fc..7af7f8fa19 100644 --- a/src/plugins/oglo_tactile/oglo_tactile_plugin.hpp +++ b/src/plugins/oglo_tactile/oglo_tactile_plugin.hpp @@ -39,7 +39,7 @@ class OgloTactilePlugin std::chrono::milliseconds stall_timeout{ 3000 }; //!< no-notify -> reconnect }; - explicit OgloTactilePlugin(Options options); + OgloTactilePlugin(Options options, core::PluginSessionHandle session); ~OgloTactilePlugin(); //! Connect, then run until @p stop is set. Reconnects automatically on drop. @@ -62,6 +62,7 @@ class OgloTactilePlugin Options m_opts; OgloDeviceConfig m_config; + core::PluginSessionHandle m_session; // Written by the consumer thread on (re)connect, read by the BLE thread in // on_notify(); atomic so the geometry handoff across threads is race-free. std::atomic m_values_per_sample{ kNumTaxels }; diff --git a/src/plugins/plugin_utils/CMakeLists.txt b/src/plugins/plugin_utils/CMakeLists.txt index 536f992291..9dc920dfd1 100644 --- a/src/plugins/plugin_utils/CMakeLists.txt +++ b/src/plugins/plugin_utils/CMakeLists.txt @@ -5,6 +5,10 @@ add_library(teleop_plugin_utils STATIC hand_injector.cpp inc/plugin_utils/hand_injector.hpp + openxr_hand_tracking_push_channel.cpp + openxr_hand_tracking_push_channel.hpp + openxr_plugin_session.cpp + inc/plugin_utils/openxr_plugin_session.hpp wrist_pose_source.cpp inc/plugin_utils/wrist_pose_source.hpp ) @@ -17,8 +21,11 @@ target_link_libraries(teleop_plugin_utils deviceio::deviceio_session deviceio::deviceio_trackers oxr::oxr_utils + pusherio::pusherio Teleop::openxr_extensions OpenXR::openxr_loader + PRIVATE + oxr::oxr_core ) # Include directories diff --git a/src/plugins/plugin_utils/inc/plugin_utils/openxr_plugin_session.hpp b/src/plugins/plugin_utils/inc/plugin_utils/openxr_plugin_session.hpp new file mode 100644 index 0000000000..033041e373 --- /dev/null +++ b/src/plugins/plugin_utils/inc/plugin_utils/openxr_plugin_session.hpp @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include + +namespace core +{ +class ControllerTracker; +class ITracker; +class OpenXRSession; +} + +namespace plugin_utils +{ + +/*! + * @brief Local plugin session that creates OpenXR-backed operation channels. + * + * The adapter owns the OpenXR session. Its owner must destroy every returned + * pull and push channel before destroying this session. OpenXR permits action + * sets to be attached only once, so create_pull_channel() may be called once. + */ +class OpenXRPluginSession final : public core::IPluginSession +{ +public: + OpenXRPluginSession(std::string app_name, + core::PluginSessionRequirements requirements, + std::vector> trackers = {}); + ~OpenXRPluginSession() override; + + OpenXRPluginSession(const OpenXRPluginSession&) = delete; + OpenXRPluginSession& operator=(const OpenXRPluginSession&) = delete; + OpenXRPluginSession(OpenXRPluginSession&&) = delete; + OpenXRPluginSession& operator=(OpenXRPluginSession&&) = delete; + + std::unique_ptr create_pull_channel() override; + std::unique_ptr create_schema_push_channel(const core::SchemaPusherConfig& config) override; + std::unique_ptr create_hand_tracking_push_channel(XrHandEXT hand) override; + +private: + core::PluginSessionRequirements requirements_; + std::vector> trackers_; + std::shared_ptr wrist_controller_tracker_; + bool native_hand_tracking_enabled_ = false; + bool pull_channel_creation_attempted_ = false; + std::shared_ptr session_; +}; + +} // namespace plugin_utils diff --git a/src/plugins/plugin_utils/openxr_hand_tracking_push_channel.cpp b/src/plugins/plugin_utils/openxr_hand_tracking_push_channel.cpp new file mode 100644 index 0000000000..5df5fda3b5 --- /dev/null +++ b/src/plugins/plugin_utils/openxr_hand_tracking_push_channel.cpp @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "openxr_hand_tracking_push_channel.hpp" + +#include "inc/plugin_utils/hand_injector.hpp" + +#include + +#include + +namespace plugin_utils +{ + +namespace +{ + +class OpenXRHandTrackingPushChannel final : public core::IHandTrackingPushChannel +{ +public: + OpenXRHandTrackingPushChannel(const core::OpenXRSessionHandles& handles, XrHandEXT hand) + : time_converter_(handles), injector_(handles.instance, handles.session, hand, handles.space) + { + } + + void push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns) override + { + if (joint_locations == nullptr) + { + throw std::invalid_argument("Hand tracking push requires joint locations"); + } + + injector_.push( + joint_locations, time_converter_.convert_monotonic_ns_to_xrtime(sample_time_local_common_clock_ns)); + } + +private: + core::XrTimeConverter time_converter_; + HandInjector injector_; +}; + +} // namespace + +std::unique_ptr make_openxr_hand_tracking_push_channel( + const core::OpenXRSessionHandles& handles, XrHandEXT hand) +{ + return std::make_unique(handles, hand); +} + +} // namespace plugin_utils diff --git a/src/plugins/plugin_utils/openxr_hand_tracking_push_channel.hpp b/src/plugins/plugin_utils/openxr_hand_tracking_push_channel.hpp new file mode 100644 index 0000000000..dd23a638ed --- /dev/null +++ b/src/plugins/plugin_utils/openxr_hand_tracking_push_channel.hpp @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include + +namespace plugin_utils +{ + +std::unique_ptr make_openxr_hand_tracking_push_channel( + const core::OpenXRSessionHandles& handles, XrHandEXT hand); + +} // namespace plugin_utils diff --git a/src/plugins/plugin_utils/openxr_plugin_session.cpp b/src/plugins/plugin_utils/openxr_plugin_session.cpp new file mode 100644 index 0000000000..84cdd65c13 --- /dev/null +++ b/src/plugins/plugin_utils/openxr_plugin_session.cpp @@ -0,0 +1,246 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/plugin_utils/openxr_plugin_session.hpp" + +#include "inc/plugin_utils/wrist_pose_source.hpp" +#include "openxr_hand_tracking_push_channel.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace plugin_utils +{ + +namespace +{ + +void append_unique(std::vector& extensions, const std::vector& additions) +{ + for (const auto& extension : additions) + { + if (std::find(extensions.begin(), extensions.end(), extension) == extensions.end()) + { + extensions.push_back(extension); + } + } +} + +std::vector make_required_extensions(const core::PluginSessionRequirements& requirements, + const std::vector>& trackers) +{ + auto extensions = core::DeviceIOSession::get_required_extensions(trackers); + + if (requirements.schema_push) + { + append_unique(extensions, core::OpenXRSchemaPushChannel::get_required_extensions()); + } + + if (requirements.hand_tracking_push) + { + append_unique(extensions, { XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME }); + append_unique(extensions, core::XrTimeConverter::get_required_extensions()); + } + + return extensions; +} + +bool is_extension_supported(const char* extension_name) +{ + uint32_t count = 0; + if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, 0, &count, nullptr))) + { + return false; + } + + std::vector properties(count, XrExtensionProperties{ XR_TYPE_EXTENSION_PROPERTIES }); + if (XR_FAILED(xrEnumerateInstanceExtensionProperties(nullptr, count, &count, properties.data()))) + { + return false; + } + + return std::any_of(properties.begin(), properties.end(), + [extension_name](const XrExtensionProperties& property) + { return std::string(property.extensionName) == extension_name; }); +} + +WristSourceMode to_openxr_mode(core::WristTrackingSourceMode mode) +{ + switch (mode) + { + case core::WristTrackingSourceMode::Auto: + return WristSourceMode::Auto; + case core::WristTrackingSourceMode::HandTracking: + return WristSourceMode::HandTracking; + case core::WristTrackingSourceMode::Controller: + return WristSourceMode::Controller; + } + throw std::logic_error("Unknown wrist tracking source mode"); +} + +WristSourceConfig to_openxr_config(const core::WristTrackingSourceConfig& config) +{ + return WristSourceConfig{ .mode = to_openxr_mode(config.mode), + .left_aim_to_wrist = config.left_aim_to_wrist, + .right_aim_to_wrist = config.right_aim_to_wrist }; +} + +class OpenXRWristTrackingSource final : public core::IWristTrackingSource +{ +public: + OpenXRWristTrackingSource(const core::WristTrackingSourceConfig& config, + const core::OpenXRSessionHandles& handles, + core::DeviceIOSession* deviceio_session, + std::shared_ptr controller_tracker) + : time_converter_(handles), + source_(to_openxr_config(config), handles, deviceio_session, std::move(controller_tracker)) + { + } + + core::WristTrackingSample query(bool is_left, int64_t sample_time_local_common_clock_ns) override + { + const WristSample sample = + source_.query(is_left, time_converter_.convert_monotonic_ns_to_xrtime(sample_time_local_common_clock_ns)); + return core::WristTrackingSample{ .pose = sample.pose, .valid = sample.valid, .tracked = sample.tracked }; + } + +private: + core::XrTimeConverter time_converter_; + WristPoseSource source_; +}; + +class OpenXRPluginPullChannel final : public core::IPluginPullChannel +{ +public: + OpenXRPluginPullChannel(const core::OpenXRSessionHandles& handles, + const std::vector>& trackers, + bool wrist_tracking_enabled, + bool native_hand_tracking_enabled, + std::shared_ptr wrist_controller_tracker) + : handles_(handles), + wrist_tracking_enabled_(wrist_tracking_enabled), + native_hand_tracking_enabled_(native_hand_tracking_enabled), + wrist_controller_tracker_(std::move(wrist_controller_tracker)), + deviceio_session_(core::DeviceIOSession::run(trackers, handles_)) + { + } + + void update() override + { + deviceio_session_->update(); + } + + const core::ITrackerImpl& get_tracker_impl(const core::ITracker& tracker) const override + { + return deviceio_session_->get_tracker_impl(tracker); + } + + std::unique_ptr create_wrist_tracking_source( + const core::WristTrackingSourceConfig& config) override + { + if (!wrist_tracking_enabled_) + { + throw std::logic_error( + "OpenXRPluginSession: wrist tracking pull was not declared in PluginSessionRequirements"); + } + + core::WristTrackingSourceConfig effective_config = config; + if (!native_hand_tracking_enabled_) + { + if (effective_config.mode == core::WristTrackingSourceMode::HandTracking) + { + return nullptr; + } + if (effective_config.mode == core::WristTrackingSourceMode::Auto) + { + effective_config.mode = core::WristTrackingSourceMode::Controller; + } + } + + if (effective_config.mode != core::WristTrackingSourceMode::HandTracking && wrist_controller_tracker_ == nullptr) + { + return nullptr; + } + + return std::make_unique( + effective_config, handles_, deviceio_session_.get(), wrist_controller_tracker_); + } + +private: + core::OpenXRSessionHandles handles_; + bool wrist_tracking_enabled_; + bool native_hand_tracking_enabled_; + std::shared_ptr wrist_controller_tracker_; + std::unique_ptr deviceio_session_; +}; + +} // anonymous namespace + +OpenXRPluginSession::OpenXRPluginSession(std::string app_name, + core::PluginSessionRequirements requirements, + std::vector> trackers) + : requirements_(requirements), trackers_(std::move(trackers)) +{ + if (requirements_.wrist_tracking_pull) + { + wrist_controller_tracker_ = std::make_shared(); + trackers_.push_back(wrist_controller_tracker_); + native_hand_tracking_enabled_ = is_extension_supported(XR_EXT_HAND_TRACKING_EXTENSION_NAME) && + is_extension_supported(XR_MNDX_XDEV_SPACE_EXTENSION_NAME); + } + + auto extensions = make_required_extensions(requirements_, trackers_); + if (native_hand_tracking_enabled_) + { + append_unique(extensions, { XR_EXT_HAND_TRACKING_EXTENSION_NAME, XR_MNDX_XDEV_SPACE_EXTENSION_NAME }); + } + session_ = std::make_shared(app_name, extensions); +} + +OpenXRPluginSession::~OpenXRPluginSession() = default; + +std::unique_ptr OpenXRPluginSession::create_pull_channel() +{ + if (pull_channel_creation_attempted_) + { + throw std::logic_error("OpenXRPluginSession supports only one pull channel per session"); + } + pull_channel_creation_attempted_ = true; + + return std::make_unique(session_->get_handles(), trackers_, + requirements_.wrist_tracking_pull, native_hand_tracking_enabled_, + wrist_controller_tracker_); +} + +std::unique_ptr OpenXRPluginSession::create_schema_push_channel( + const core::SchemaPusherConfig& config) +{ + if (!requirements_.schema_push) + { + throw std::logic_error("OpenXRPluginSession: schema push was not declared in PluginSessionRequirements"); + } + return core::make_openxr_schema_push_channel(session_->get_handles(), config); +} + +std::unique_ptr OpenXRPluginSession::create_hand_tracking_push_channel(XrHandEXT hand) +{ + if (!requirements_.hand_tracking_push) + { + throw std::logic_error("OpenXRPluginSession: hand-tracking push was not declared in PluginSessionRequirements"); + } + return make_openxr_hand_tracking_push_channel(session_->get_handles(), hand); +} + +} // namespace plugin_utils diff --git a/src/plugins/rebot_devarm_leader/CMakeLists.txt b/src/plugins/rebot_devarm_leader/CMakeLists.txt index 16ae76127f..aeb4066851 100644 --- a/src/plugins/rebot_devarm_leader/CMakeLists.txt +++ b/src/plugins/rebot_devarm_leader/CMakeLists.txt @@ -9,8 +9,8 @@ add_executable(rebot_devarm_leader_plugin ) target_link_libraries(rebot_devarm_leader_plugin PRIVATE + Teleop::plugin_utils pusherio::pusherio - oxr::oxr_core isaacteleop_schema ) diff --git a/src/plugins/rebot_devarm_leader/main.cpp b/src/plugins/rebot_devarm_leader/main.cpp index f762436c65..6e7a4cd13c 100644 --- a/src/plugins/rebot_devarm_leader/main.cpp +++ b/src/plugins/rebot_devarm_leader/main.cpp @@ -3,12 +3,16 @@ #include "rebot_devarm_leader_plugin.hpp" +#include + #include #include #include #include +#include #include #include +#include using namespace plugins::rebot_devarm_leader; @@ -38,7 +42,9 @@ try << ", collection: " << collection_id << (calibration_path.empty() ? "" : ", calibration: " + calibration_path) << ")" << std::endl; - RebotDevarmLeaderPlugin plugin(device_path, collection_id, calibration_path); + core::PluginSessionHandle session = std::make_shared( + "RebotDevarmLeaderPlugin", core::PluginSessionRequirements{ .schema_push = true }); + RebotDevarmLeaderPlugin plugin(device_path, collection_id, std::move(session), calibration_path); // Push joint state at 90 Hz. const auto frame_duration = std::chrono::nanoseconds(1000000000 / 90); diff --git a/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.cpp b/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.cpp index 1da10a9374..65a01166b1 100644 --- a/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.cpp +++ b/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.cpp @@ -7,7 +7,6 @@ #include "robstride_bus.hpp" #include -#include #include #include @@ -19,8 +18,10 @@ #include #include #include +#include #include #include +#include namespace plugins { @@ -34,6 +35,21 @@ namespace // fixed tensor buffer (7 named joints + velocity fit comfortably). constexpr size_t kMaxFlatbufferSize = 4096; +std::unique_ptr make_push_channel(const core::PluginSessionHandle& session, + const std::string& collection_id) +{ + if (!session) + { + throw std::invalid_argument("RebotDevarmLeaderPlugin requires a plugin session"); + } + + return session->create_schema_push_channel(core::SchemaPusherConfig{ .collection_id = collection_id, + .max_flatbuffer_size = kMaxFlatbufferSize, + .tensor_identifier = "joint_state", + .localized_name = "reBot DevArm Leader", + .app_name = "RebotDevarmLeaderPlugin" }); +} + // reBot DevArm DOF order (matches the reBot-DevArm_fixend URDF joint names; the gripper is the // extra 7th Damiao motor). constexpr std::array kJointNames = { "joint1", "joint2", "joint3", "joint4", @@ -115,17 +131,12 @@ ModelLimits model_limits(const std::string& model) RebotDevarmLeaderPlugin::RebotDevarmLeaderPlugin(const std::string& device_path, const std::string& collection_id, + core::PluginSessionHandle session, const std::string& calibration_path) : device_path_(device_path), collection_id_(collection_id), - session_(std::make_shared( - "RebotDevarmLeaderPlugin", core::SchemaPusher::get_required_extensions())), - pusher_(session_->get_handles(), - core::SchemaPusherConfig{ .collection_id = collection_id, - .max_flatbuffer_size = kMaxFlatbufferSize, - .tensor_identifier = "joint_state", - .localized_name = "reBot DevArm Leader", - .app_name = "RebotDevarmLeaderPlugin" }) + session_(std::move(session)), + pusher_(make_push_channel(session_, collection_id)) { // Defaults: factory ids (1..7 / 0x11..0x17), factory models, no sign flip, zero offset. for (int i = 0; i < kNumJoints; ++i) diff --git a/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.hpp b/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.hpp index b858f3c997..d2b01cdbe6 100644 --- a/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.hpp +++ b/src/plugins/rebot_devarm_leader/rebot_devarm_leader_plugin.hpp @@ -3,17 +3,13 @@ #pragma once +#include #include #include #include #include -namespace core -{ -class OpenXRSession; -} - namespace plugins { namespace rebot_devarm_leader @@ -59,11 +55,13 @@ class RebotDevarmLeaderPlugin * Empty selects the synthetic backend. * @param collection_id Tensor collection id; must match the consumer's JointStateTracker. * Also used as the JointStateOutput.device_id. + * @param session Session that creates the transport channel used by the pusher. * @param calibration_path Optional calibration file (see load_calibration()); empty uses * defaults (motor ids 1..7, feedback ids 0x11..0x17, sign +1, zero offset 0). */ RebotDevarmLeaderPlugin(const std::string& device_path, const std::string& collection_id, + core::PluginSessionHandle session, const std::string& calibration_path = ""); ~RebotDevarmLeaderPlugin(); @@ -112,7 +110,7 @@ class RebotDevarmLeaderPlugin std::unique_ptr bus_; std::unique_ptr rs_bus_; - std::shared_ptr session_; + core::PluginSessionHandle session_; core::SchemaPusher pusher_; }; diff --git a/src/plugins/so101_leader/CMakeLists.txt b/src/plugins/so101_leader/CMakeLists.txt index 7dfa8cdd31..a470cb71dd 100644 --- a/src/plugins/so101_leader/CMakeLists.txt +++ b/src/plugins/so101_leader/CMakeLists.txt @@ -8,8 +8,8 @@ add_executable(so101_leader_plugin ) target_link_libraries(so101_leader_plugin PRIVATE + Teleop::plugin_utils pusherio::pusherio - oxr::oxr_core isaacteleop_schema ) diff --git a/src/plugins/so101_leader/main.cpp b/src/plugins/so101_leader/main.cpp index edaaa244ff..2254185bc7 100644 --- a/src/plugins/so101_leader/main.cpp +++ b/src/plugins/so101_leader/main.cpp @@ -3,11 +3,15 @@ #include "so101_leader_plugin.hpp" +#include + #include #include #include +#include #include #include +#include using namespace plugins::so101_leader; @@ -34,7 +38,9 @@ try << ", collection: " << collection_id << (calibration_path.empty() ? "" : ", calibration: " + calibration_path) << ")" << std::endl; - So101LeaderPlugin plugin(device_path, collection_id, calibration_path); + core::PluginSessionHandle session = std::make_shared( + "So101LeaderPlugin", core::PluginSessionRequirements{ .schema_push = true }); + So101LeaderPlugin plugin(device_path, collection_id, std::move(session), calibration_path); // Push joint state at 90 Hz. const auto frame_duration = std::chrono::nanoseconds(1000000000 / 90); diff --git a/src/plugins/so101_leader/so101_leader_plugin.cpp b/src/plugins/so101_leader/so101_leader_plugin.cpp index 39eb3a46d0..fe6189531c 100644 --- a/src/plugins/so101_leader/so101_leader_plugin.cpp +++ b/src/plugins/so101_leader/so101_leader_plugin.cpp @@ -6,7 +6,6 @@ #include "feetech_bus.hpp" #include -#include #include #include @@ -24,8 +23,10 @@ #include #include #include +#include #include #include +#include #include namespace plugins @@ -40,6 +41,21 @@ namespace // fixed tensor buffer (6 named joints + optional channels fit comfortably). constexpr size_t kMaxFlatbufferSize = 4096; +std::unique_ptr make_push_channel(const core::PluginSessionHandle& session, + const std::string& collection_id) +{ + if (!session) + { + throw std::invalid_argument("So101LeaderPlugin requires a plugin session"); + } + + return session->create_schema_push_channel(core::SchemaPusherConfig{ .collection_id = collection_id, + .max_flatbuffer_size = kMaxFlatbufferSize, + .tensor_identifier = "joint_state", + .localized_name = "SO-101 Leader Arm", + .app_name = "So101LeaderPlugin" }); +} + // SO-101 DOF order (matches Simulation/SO101/so101_new_calib.urdf and the schema name keys). constexpr std::array kJointNames = { "shoulder_pan", "shoulder_lift", "elbow_flex", "wrist_flex", "wrist_roll", "gripper" }; @@ -55,16 +71,12 @@ constexpr double kSynthPeriodFrames = 90.0; // one cycle per ~1 s at 90 Hz So101LeaderPlugin::So101LeaderPlugin(const std::string& device_path, const std::string& collection_id, + core::PluginSessionHandle session, const std::string& calibration_path) : device_path_(device_path), collection_id_(collection_id), - session_(std::make_shared("So101LeaderPlugin", core::SchemaPusher::get_required_extensions())), - pusher_(session_->get_handles(), - core::SchemaPusherConfig{ .collection_id = collection_id, - .max_flatbuffer_size = kMaxFlatbufferSize, - .tensor_identifier = "joint_state", - .localized_name = "SO-101 Leader Arm", - .app_name = "So101LeaderPlugin" }) + session_(std::move(session)), + pusher_(make_push_channel(session_, collection_id)) { // Defaults: servo ids 1..6 in DOF order, no sign flip, centered at the servo midpoint (2048), // full tick range (so the clamp is a no-op until a calibration file narrows it). diff --git a/src/plugins/so101_leader/so101_leader_plugin.hpp b/src/plugins/so101_leader/so101_leader_plugin.hpp index 3b9aa07af6..9fc114b346 100644 --- a/src/plugins/so101_leader/so101_leader_plugin.hpp +++ b/src/plugins/so101_leader/so101_leader_plugin.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include @@ -11,11 +12,6 @@ #include #include -namespace core -{ -class OpenXRSession; -} - namespace plugins { namespace so101_leader @@ -45,11 +41,13 @@ class So101LeaderPlugin * Empty selects the synthetic backend. * @param collection_id Tensor collection id; must match the consumer's JointStateTracker. * Also used as the JointStateOutput.device_id. + * @param session Session that creates the transport channel used by the pusher. * @param calibration_path Optional calibration file (see load_calibration()); empty uses * defaults (servo ids 1..6 in DOF order, sign +1, home tick 2048). */ So101LeaderPlugin(const std::string& device_path, const std::string& collection_id, + core::PluginSessionHandle session, const std::string& calibration_path = ""); ~So101LeaderPlugin(); @@ -98,7 +96,7 @@ class So101LeaderPlugin std::vector read_ok_; // sync-read scratch: per-servo reply flag std::vector lerobot_homing_; // per-DOF homing_offset from a loaded LeRobot JSON (else empty) - std::shared_ptr session_; + core::PluginSessionHandle session_; core::SchemaPusher pusher_; }; diff --git a/src/plugins/vive_se3_tracker/CMakeLists.txt b/src/plugins/vive_se3_tracker/CMakeLists.txt index 5046eaeb37..09f94ada1a 100644 --- a/src/plugins/vive_se3_tracker/CMakeLists.txt +++ b/src/plugins/vive_se3_tracker/CMakeLists.txt @@ -29,9 +29,9 @@ add_executable(vive_se3_tracker_plugin target_include_directories(vive_se3_tracker_plugin PRIVATE "${VUT_SDK_DIR}/include") target_link_libraries(vive_se3_tracker_plugin PRIVATE + Teleop::plugin_utils deviceio::deviceio_trackers pusherio::pusherio - oxr::oxr_core isaacteleop_schema "${_vut_lib}" Threads::Threads diff --git a/src/plugins/vive_se3_tracker/main.cpp b/src/plugins/vive_se3_tracker/main.cpp index 957faf03ea..7abdaac2b0 100644 --- a/src/plugins/vive_se3_tracker/main.cpp +++ b/src/plugins/vive_se3_tracker/main.cpp @@ -3,12 +3,16 @@ #include "vive_se3_tracker_plugin.hpp" +#include + #include #include #include #include #include +#include #include +#include using namespace plugins::vive_se3_tracker; @@ -31,7 +35,9 @@ try std::signal(SIGINT, on_signal); std::signal(SIGTERM, on_signal); - ViveSe3TrackerPlugin plugin; + core::PluginSessionHandle session = std::make_shared( + "ViveSe3TrackerPlugin", core::PluginSessionRequirements{ .schema_push = true }); + ViveSe3TrackerPlugin plugin(std::move(session)); // Poll/push at 90 Hz; valid samples carry the VUT sample time, so the loop // rate only bounds delivery latency, not timestamp accuracy. diff --git a/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.cpp b/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.cpp index fad33c5535..14e7d4a0da 100644 --- a/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.cpp +++ b/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -98,11 +98,14 @@ core::SchemaPusherConfig make_pusher_config(const std::string& collection_id) } // namespace -ViveSe3TrackerPlugin::ViveSe3TrackerPlugin() - : session_( - std::make_shared("ViveSe3TrackerPlugin", core::SchemaPusher::get_required_extensions())), - start_time_ns_(core::os_monotonic_now_ns()) +ViveSe3TrackerPlugin::ViveSe3TrackerPlugin(core::PluginSessionHandle session) + : session_(std::move(session)), start_time_ns_(core::os_monotonic_now_ns()) { + if (!session_) + { + throw std::invalid_argument("ViveSe3TrackerPlugin requires a plugin session"); + } + // Clamp VIVE_SE3_STALE_MS before scaling to ns: a non-numeric env yields 0 // (env_int), which would mark every sample stale, and a huge value would // overflow the * 1'000'000. Reject out-of-range values, use the default. @@ -268,8 +271,8 @@ ViveSe3TrackerPlugin::DeviceStream& ViveSe3TrackerPlugin::stream_for(uint32_t de DeviceStream stream; stream.serial = serial; stream.collection_id = make_collection_id(device_id, serial); - stream.pusher = - std::make_unique(session_->get_handles(), make_pusher_config(stream.collection_id)); + stream.pusher = std::make_unique( + session_->create_schema_push_channel(make_pusher_config(stream.collection_id))); std::cout << "[vive_se3_tracker] created tensor collection '" << stream.collection_id << "' for device_id=" << device_id << std::endl; DeviceStream& ref = streams_.emplace(device_id, std::move(stream)).first->second; @@ -332,7 +335,7 @@ void ViveSe3TrackerPlugin::update() if (synthetic_mode_) generate_synthetic_poses(core::os_monotonic_now_ns()); - // Snapshot under the lock, push outside it (pushes go through OpenXR). + // Snapshot under the lock, push outside it. std::unordered_map snapshot; { std::lock_guard lock(pose_mutex_); diff --git a/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.hpp b/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.hpp index 687b39c187..0d991d2ffd 100644 --- a/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.hpp +++ b/src/plugins/vive_se3_tracker/vive_se3_tracker_plugin.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -13,11 +14,6 @@ #include #include -namespace core -{ -class OpenXRSession; -} - namespace plugins { namespace vive_se3_tracker @@ -87,7 +83,7 @@ constexpr int64_t kMaxStaleMs = 3600000; class ViveSe3TrackerPlugin { public: - ViveSe3TrackerPlugin(); + explicit ViveSe3TrackerPlugin(core::PluginSessionHandle session); ~ViveSe3TrackerPlugin(); ViveSe3TrackerPlugin(const ViveSe3TrackerPlugin&) = delete; @@ -126,7 +122,7 @@ class ViveSe3TrackerPlugin // stream is rebuilt so samples never land in the previous tracker's collection. DeviceStream& stream_for(uint32_t device_id, const std::string& serial); - std::shared_ptr session_; + core::PluginSessionHandle session_; // --- VIVEHub VUT client --- std::unique_ptr vut_client_; diff --git a/src/plugins/wuji_glove/README.md b/src/plugins/wuji_glove/README.md index 2d4cace90a..278ef03c94 100644 --- a/src/plugins/wuji_glove/README.md +++ b/src/plugins/wuji_glove/README.md @@ -5,8 +5,10 @@ SPDX-License-Identifier: Apache-2.0 # Wuji Glove → Isaac Teleop -Drive a **Wuji dexterous hand** from a **Wuji data glove** through the Isaac -Teleop / CloudXR (OpenXR) stack. +Drive a **Wuji dexterous hand** from a **Wuji data glove** through an Isaac +Teleop plugin session. The executable currently selects the local OpenXR +adapter, while the plugin implementation uses transport-neutral pull and hand +tracking channels. Full documentation — components, prerequisites, installation, running, and troubleshooting — lives in the docs tree: diff --git a/src/plugins/wuji_glove/wuji_glove.cpp b/src/plugins/wuji_glove/wuji_glove.cpp index 8b1a1362c7..e3ead70386 100644 --- a/src/plugins/wuji_glove/wuji_glove.cpp +++ b/src/plugins/wuji_glove/wuji_glove.cpp @@ -3,6 +3,8 @@ #include "wuji_glove_plugin.hpp" +#include + #include #include #include @@ -10,6 +12,7 @@ #include #include #include +#include using namespace plugins::wuji_glove; @@ -47,7 +50,9 @@ try std::cout << "Wuji Glove Plugin" << std::endl; std::cout << "Plugin Root ID: " << plugin_root_id << std::endl; - auto plugin = std::make_unique(plugin_root_id); + core::PluginSessionHandle session = std::make_shared( + "WujiGlove", core::PluginSessionRequirements{ .hand_tracking_push = true, .wrist_tracking_pull = true }); + auto plugin = std::make_unique(plugin_root_id, std::move(session)); std::cout << "Plugin running. Press Ctrl+C to stop." << std::endl; while (!g_stop_requested.load(std::memory_order_relaxed) && plugin->is_running()) diff --git a/src/plugins/wuji_glove/wuji_glove_plugin.cpp b/src/plugins/wuji_glove/wuji_glove_plugin.cpp index a9f3c1ba9d..18f2ffd059 100644 --- a/src/plugins/wuji_glove/wuji_glove_plugin.cpp +++ b/src/plugins/wuji_glove/wuji_glove_plugin.cpp @@ -4,6 +4,7 @@ #include "wuji_glove_plugin.hpp" #include +#include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include namespace plugins { @@ -197,23 +199,23 @@ XrPosef pose_from_env(const char* name, const XrPosef& fallback) // Wrist-source selection: WUJI_GLOVE_WRIST_SOURCE = auto (default) | // hand_tracking | controller. -plugin_utils::WristSourceMode wrist_source_mode_from_env() +core::WristTrackingSourceMode wrist_source_mode_from_env() { const char* value = std::getenv("WUJI_GLOVE_WRIST_SOURCE"); if (value == nullptr || *value == '\0' || std::strcmp(value, "auto") == 0) { - return plugin_utils::WristSourceMode::Auto; + return core::WristTrackingSourceMode::Auto; } if (std::strcmp(value, "hand_tracking") == 0) { - return plugin_utils::WristSourceMode::HandTracking; + return core::WristTrackingSourceMode::HandTracking; } if (std::strcmp(value, "controller") == 0) { - return plugin_utils::WristSourceMode::Controller; + return core::WristTrackingSourceMode::Controller; } std::cerr << "WujiGlovePlugin: unknown WUJI_GLOVE_WRIST_SOURCE '" << value << "', using 'auto'" << std::endl; - return plugin_utils::WristSourceMode::Auto; + return core::WristTrackingSourceMode::Auto; } // Resolve the glove's hand side explicitly via the device's "hand_side" GET @@ -245,33 +247,32 @@ const char* safe_err() } // namespace -WujiGlovePlugin::WujiGlovePlugin(const std::string& plugin_root_id) noexcept(false) : m_root_id(plugin_root_id) +WujiGlovePlugin::WujiGlovePlugin(const std::string& plugin_root_id, + core::PluginSessionHandle plugin_session) noexcept(false) + : m_plugin_session(std::move(plugin_session)), m_root_id(plugin_root_id) { std::cout << "Initializing WujiGlovePlugin with root: " << m_root_id << std::endl; - // The glove itself is not an OpenXR upstream tracker — it is read - // out-of-band via wuji_sdk. The tracker list carries only what the wrist - // source needs (the controller tracker for the aim-pose fallback); the - // OpenXR session exists for the push-device (injection) extension, the - // wrist-source queries, and the XrTime base. - plugin_utils::WristSourceConfig wrist_config; + if (!m_plugin_session) + { + throw std::invalid_argument("WujiGlovePlugin requires a plugin session"); + } + + core::WristTrackingSourceConfig wrist_config; wrist_config.mode = wrist_source_mode_from_env(); wrist_config.left_aim_to_wrist = pose_from_env("WUJI_GLOVE_AIM_TO_WRIST_LEFT", kLeftAimToWrist); wrist_config.right_aim_to_wrist = pose_from_env("WUJI_GLOVE_AIM_TO_WRIST_RIGHT", kRightAimToWrist); - auto wrist_requirements = plugin_utils::WristPoseSource::collect_requirements(wrist_config.mode); - - std::vector> trackers = wrist_requirements.trackers; - auto extensions = core::DeviceIOSession::get_required_extensions(trackers); - extensions.push_back(XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME); - extensions.insert(extensions.end(), wrist_requirements.extensions.begin(), wrist_requirements.extensions.end()); - - m_session = std::make_shared("WujiGlove", extensions); - const auto handles = m_session->get_handles(); - - m_deviceio_session = core::DeviceIOSession::run(trackers, handles); - m_time_converter.emplace(handles); - m_wrist_source = std::make_unique( - wrist_config, handles, m_deviceio_session.get(), wrist_requirements.controller_tracker); + m_pull_channel = m_plugin_session->create_pull_channel(); + if (!m_pull_channel) + { + throw std::runtime_error("The plugin session could not create a pull channel"); + } + m_wrist_source = m_pull_channel->create_wrist_tracking_source(wrist_config); + if (!m_wrist_source) + { + std::cout << "WujiGlovePlugin: requested wrist source is unavailable; publishing wrist-relative joints" + << std::endl; + } WujiInitOptions init_opts{}; init_opts.log_level = 2; // warn only; the plugin reports connection and errors itself @@ -546,24 +547,23 @@ void WujiGlovePlugin::invalidate_hand(bool is_left) slot.valid = false; } -void WujiGlovePlugin::pump_hand(std::unique_ptr& injector, +void WujiGlovePlugin::pump_hand(std::unique_ptr& pusher, XrHandEXT hand, const HandFrame& frame, - XrTime time) + int64_t sample_time_ns) { - // Treat data older than 200 ms as "hand absent": drop the injector so the - // runtime reports isActive=false rather than a frozen pose. + // Treat data older than 200 ms as "hand absent": close the stream so the + // receiver reports isActive=false rather than a frozen pose. using namespace std::chrono; const bool fresh = frame.valid && (steady_clock::now() - frame.stamp) < kStaleThreshold; if (!fresh) { - injector.reset(); + pusher.reset(); return; } - if (!injector) + if (!pusher) { - const auto handles = m_session->get_handles(); - injector = std::make_unique(handles.instance, handles.session, hand, handles.space); + pusher = std::make_unique(m_plugin_session->create_hand_tracking_push_channel(hand)); } // Fuse the device wrist pose: place the wrist-relative skeleton at the @@ -571,10 +571,10 @@ void WujiGlovePlugin::pump_hand(std::unique_ptr& inj // actively tracked. With no wrist source available the skeleton stays // wrist-relative at the space origin with VALID-only flags (honest // degradation: consumers see the shape but know the pose is untracked). - plugin_utils::WristSample wrist; + core::WristTrackingSample wrist; if (m_wrist_source) { - wrist = m_wrist_source->query(hand == XR_HAND_LEFT_EXT, time); + wrist = m_wrist_source->query(hand == XR_HAND_LEFT_EXT, sample_time_ns); } std::array joints = frame.joints; @@ -593,7 +593,7 @@ void WujiGlovePlugin::pump_hand(std::unique_ptr& inj } } } - injector->push(joints.data(), time); + pusher->push(joints.data(), sample_time_ns); } void WujiGlovePlugin::worker_thread() @@ -602,31 +602,40 @@ void WujiGlovePlugin::worker_thread() { try { - m_deviceio_session->update(); + m_pull_channel->update(); + + const int64_t sample_time_ns = core::os_monotonic_now_ns(); + + HandFrame left_copy; + HandFrame right_copy; + { + std::lock_guard lock(m_frame_mutex); + left_copy = m_left; + right_copy = m_right; + } + + pump_hand(m_left_pusher, XR_HAND_LEFT_EXT, left_copy, sample_time_ns); + pump_hand(m_right_pusher, XR_HAND_RIGHT_EXT, right_copy, sample_time_ns); } catch (const std::exception& e) { - std::cerr << "WujiGlovePlugin update error: " << e.what() << std::endl; - m_left_injector.reset(); - m_right_injector.reset(); + std::cerr << "WujiGlovePlugin worker error: " << e.what() << std::endl; + m_left_pusher.reset(); + m_right_pusher.reset(); m_failed.store(true, std::memory_order_release); m_running.store(false, std::memory_order_release); return; } - - const XrTime time = m_time_converter->os_monotonic_now(); - - HandFrame left_copy; - HandFrame right_copy; + catch (...) { - std::lock_guard lock(m_frame_mutex); - left_copy = m_left; - right_copy = m_right; + std::cerr << "WujiGlovePlugin worker error: unknown exception" << std::endl; + m_left_pusher.reset(); + m_right_pusher.reset(); + m_failed.store(true, std::memory_order_release); + m_running.store(false, std::memory_order_release); + return; } - pump_hand(m_left_injector, XR_HAND_LEFT_EXT, left_copy, time); - pump_hand(m_right_injector, XR_HAND_RIGHT_EXT, right_copy, time); - std::this_thread::sleep_for(kFramePeriod); } } diff --git a/src/plugins/wuji_glove/wuji_glove_plugin.hpp b/src/plugins/wuji_glove/wuji_glove_plugin.hpp index 79a01291f7..dd83d7426c 100644 --- a/src/plugins/wuji_glove/wuji_glove_plugin.hpp +++ b/src/plugins/wuji_glove/wuji_glove_plugin.hpp @@ -3,12 +3,9 @@ #pragma once -#include #include -#include -#include -#include -#include +#include +#include extern "C" { @@ -30,16 +27,15 @@ namespace plugins namespace wuji_glove { -// Wuji glove -> OpenXR hand-tracking device plugin. +// Wuji glove -> session-backed hand-tracking plugin. // // Reads the glove's 21-joint MediaPipe skeleton via the wuji_sdk C API // (callback-based subscription), converts each frame to a 26-joint -// XrHandJointLocationEXT set, and injects it into the OpenXR hand layer via -// plugin_utils::HandInjector. The existing core::HandTracker consumes it. +// XrHandJointLocationEXT set, and publishes it through the plugin session. class WujiGlovePlugin { public: - explicit WujiGlovePlugin(const std::string& plugin_root_id) noexcept(false); + WujiGlovePlugin(const std::string& plugin_root_id, core::PluginSessionHandle plugin_session) noexcept(false); ~WujiGlovePlugin(); bool is_running() const noexcept; @@ -95,20 +91,17 @@ class WujiGlovePlugin void disconnect_glove(GloveConnection& connection); void discover_gloves(); - // Push (or reset) one hand's injector based on the latest HandFrame. - void pump_hand(std::unique_ptr& injector, + // Push (or reset) one hand's stream based on the latest HandFrame. + void pump_hand(std::unique_ptr& pusher, XrHandEXT hand, const HandFrame& frame, - XrTime time); - - std::shared_ptr m_session; - std::unique_ptr m_deviceio_session; - std::unique_ptr m_left_injector; - std::unique_ptr m_right_injector; - std::optional m_time_converter; - // Declared after m_session/m_deviceio_session: destroyed first, while the - // XR handles and the (non-owned) DeviceIOSession it references are alive. - std::unique_ptr m_wrist_source; + int64_t sample_time_ns); + + core::PluginSessionHandle m_plugin_session; + std::unique_ptr m_pull_channel; + std::unique_ptr m_wrist_source; + std::unique_ptr m_left_pusher; + std::unique_ptr m_right_pusher; // Owned exclusively by m_connection_thread until it is joined. std::vector> m_connections; diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 79c958f895..f5be3b2309 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -5,3 +5,4 @@ add_subdirectory(schema) add_subdirectory(mcap) add_subdirectory(replay_deviceio_session) add_subdirectory(live_trackers) +add_subdirectory(pusherio) diff --git a/tests/cpp/core/pusherio/CMakeLists.txt b/tests/cpp/core/pusherio/CMakeLists.txt new file mode 100644 index 0000000000..1827cdd04c --- /dev/null +++ b/tests/cpp/core/pusherio/CMakeLists.txt @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(pusherio_tests + test_hand_tracking_pusher.cpp + test_plugin_pull_channel.cpp + test_schema_pusher_session.cpp +) + +target_link_libraries(pusherio_tests PRIVATE + deviceio::deviceio_trackers + pusherio::pusherio + Catch2::Catch2WithMain +) + +catch_discover_tests(pusherio_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/pusherio/test_hand_tracking_pusher.cpp b/tests/cpp/core/pusherio/test_hand_tracking_pusher.cpp new file mode 100644 index 0000000000..8c3df9bf34 --- /dev/null +++ b/tests/cpp/core/pusherio/test_hand_tracking_pusher.cpp @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include + +namespace +{ + +struct CapturedHandState +{ + const XrHandJointLocationEXT* joints{ nullptr }; + int64_t sample_time_ns{ 0 }; + bool channel_closed{ false }; +}; + +class CapturingHandChannel final : public core::IHandTrackingPushChannel +{ +public: + explicit CapturingHandChannel(std::shared_ptr state) : state_(std::move(state)) + { + } + + ~CapturingHandChannel() override + { + state_->channel_closed = true; + } + + void push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns) override + { + state_->joints = joint_locations; + state_->sample_time_ns = sample_time_local_common_clock_ns; + } + +private: + std::shared_ptr state_; +}; + +} // namespace + +TEST_CASE("HandTrackingPusher delegates samples to its channel", "[pusherio][unit]") +{ + auto state = std::make_shared(); + core::HandTrackingPusher pusher(std::make_unique(state)); + std::array joints{}; + + pusher.push(joints.data(), 1234); + + REQUIRE(state->joints == joints.data()); + REQUIRE(state->sample_time_ns == 1234); +} + +TEST_CASE("HandTrackingPusher rejects an invalid channel or sample", "[pusherio][unit]") +{ + REQUIRE_THROWS_AS(core::HandTrackingPusher(nullptr), std::invalid_argument); + + auto state = std::make_shared(); + core::HandTrackingPusher pusher(std::make_unique(state)); + REQUIRE_THROWS_AS(pusher.push(nullptr, 1234), std::invalid_argument); +} + +TEST_CASE("HandTrackingPusher closes its logical hand stream on destruction", "[pusherio][unit]") +{ + auto state = std::make_shared(); + { + core::HandTrackingPusher pusher(std::make_unique(state)); + REQUIRE_FALSE(state->channel_closed); + } + REQUIRE(state->channel_closed); +} diff --git a/tests/cpp/core/pusherio/test_plugin_pull_channel.cpp b/tests/cpp/core/pusherio/test_plugin_pull_channel.cpp new file mode 100644 index 0000000000..73896282b3 --- /dev/null +++ b/tests/cpp/core/pusherio/test_plugin_pull_channel.cpp @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace +{ + +struct PullState +{ + int update_count{ 0 }; + int64_t last_update_time_ns{ 0 }; + bool tracker_resolved{ false }; + bool wrist_queried{ false }; + bool pull_channel_destroyed{ false }; + bool session_destroyed{ false }; + bool pull_destroyed_before_session{ false }; +}; + +class FakeControllerTrackerImpl final : public core::IControllerTrackerImpl +{ +public: + explicit FakeControllerTrackerImpl(std::shared_ptr state) : state_(std::move(state)) + { + } + + void update(int64_t monotonic_time_ns) override + { + ++state_->update_count; + state_->last_update_time_ns = monotonic_time_ns; + } + + const core::Serialized& get_left_controller() const override + { + return left_; + } + + const core::Serialized& get_right_controller() const override + { + return right_; + } + + void apply_left_haptic_feedback(float, float, float) const override + { + } + + void apply_right_haptic_feedback(float, float, float) const override + { + } + +private: + std::shared_ptr state_; + core::Serialized left_; + core::Serialized right_; +}; + +class FakeWristTrackingSource final : public core::IWristTrackingSource +{ +public: + explicit FakeWristTrackingSource(std::shared_ptr state) : state_(std::move(state)) + { + } + + core::WristTrackingSample query(bool is_left, int64_t sample_time_local_common_clock_ns) override + { + state_->wrist_queried = is_left && sample_time_local_common_clock_ns == 4242; + core::WristTrackingSample sample; + sample.pose.position.x = 1.0f; + sample.valid = true; + sample.tracked = true; + return sample; + } + +private: + std::shared_ptr state_; +}; + +class FakePullChannel final : public core::IPluginPullChannel +{ +public: + FakePullChannel(std::shared_ptr state, std::shared_ptr controller_tracker) + : state_(std::move(state)), controller_tracker_(std::move(controller_tracker)), controller_impl_(state_) + { + } + + ~FakePullChannel() override + { + state_->pull_channel_destroyed = true; + } + + void update() override + { + controller_impl_.update(1234); + } + + const core::ITrackerImpl& get_tracker_impl(const core::ITracker& tracker) const override + { + if (&tracker != controller_tracker_.get()) + { + throw std::runtime_error("Tracker implementation not found"); + } + state_->tracker_resolved = true; + return controller_impl_; + } + + std::unique_ptr create_wrist_tracking_source(const core::WristTrackingSourceConfig&) override + { + return std::make_unique(state_); + } + +private: + std::shared_ptr state_; + std::shared_ptr controller_tracker_; + FakeControllerTrackerImpl controller_impl_; +}; + +class FakePluginSession final : public core::IPluginSession +{ +public: + FakePluginSession(std::shared_ptr state, std::shared_ptr controller_tracker) + : state_(std::move(state)), controller_tracker_(std::move(controller_tracker)) + { + } + + ~FakePluginSession() override + { + state_->pull_destroyed_before_session = state_->pull_channel_destroyed; + state_->session_destroyed = true; + } + + std::unique_ptr create_pull_channel() override + { + return std::make_unique(state_, controller_tracker_); + } + + std::unique_ptr create_schema_push_channel(const core::SchemaPusherConfig&) override + { + return nullptr; + } + + std::unique_ptr create_hand_tracking_push_channel(XrHandEXT) override + { + return nullptr; + } + +private: + std::shared_ptr state_; + std::shared_ptr controller_tracker_; +}; + +} // namespace + +TEST_CASE("Plugin pull channel supports typed trackers and optional sources", "[pusherio][unit]") +{ + auto state = std::make_shared(); + auto controller_tracker = std::make_shared(); + core::PluginSessionHandle session = std::make_shared(state, controller_tracker); + + { + auto pull_channel = session->create_pull_channel(); + pull_channel->update(); + + const auto& left = controller_tracker->get_left_controller(*pull_channel); + REQUIRE_FALSE(left); + REQUIRE(state->tracker_resolved); + REQUIRE(state->update_count == 1); + REQUIRE(state->last_update_time_ns == 1234); + + auto wrist_source = pull_channel->create_wrist_tracking_source({}); + const core::WristTrackingSample wrist = wrist_source->query(true, 4242); + REQUIRE(state->wrist_queried); + REQUIRE(wrist.valid); + REQUIRE(wrist.tracked); + REQUIRE(wrist.pose.position.x == 1.0f); + } + + REQUIRE(state->pull_channel_destroyed); + REQUIRE_FALSE(state->session_destroyed); + session.reset(); + REQUIRE(state->session_destroyed); + REQUIRE(state->pull_destroyed_before_session); +} diff --git a/tests/cpp/core/pusherio/test_schema_pusher_session.cpp b/tests/cpp/core/pusherio/test_schema_pusher_session.cpp new file mode 100644 index 0000000000..6cbd20145a --- /dev/null +++ b/tests/cpp/core/pusherio/test_schema_pusher_session.cpp @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include + +namespace +{ + +struct CapturedState +{ + core::SchemaPusherConfig config; + std::vector payload; + int64_t local_time_ns{ 0 }; + int64_t device_time_ns{ 0 }; + bool session_destroyed{ false }; + bool channel_destroyed{ false }; + bool channel_destroyed_before_session{ false }; +}; + +class CapturingChannel final : public core::ISchemaPushChannel +{ +public: + explicit CapturingChannel(std::shared_ptr state) : state_(std::move(state)) + { + } + + ~CapturingChannel() override + { + state_->channel_destroyed = true; + } + + const core::SchemaPusherConfig& config() const override + { + return state_->config; + } + + void push_buffer(const uint8_t* buffer, + size_t size, + int64_t sample_time_local_common_clock_ns, + int64_t sample_time_raw_device_clock_ns) override + { + state_->payload.assign(buffer, buffer + size); + state_->local_time_ns = sample_time_local_common_clock_ns; + state_->device_time_ns = sample_time_raw_device_clock_ns; + } + +private: + std::shared_ptr state_; +}; + +class CapturingSession final : public core::IPluginSession +{ +public: + explicit CapturingSession(std::shared_ptr state) : state_(std::move(state)) + { + } + + ~CapturingSession() override + { + state_->channel_destroyed_before_session = state_->channel_destroyed; + state_->session_destroyed = true; + } + + std::unique_ptr create_pull_channel() override + { + return nullptr; + } + + std::unique_ptr create_schema_push_channel(const core::SchemaPusherConfig& config) override + { + state_->config = config; + return std::make_unique(state_); + } + + std::unique_ptr create_hand_tracking_push_channel(XrHandEXT) override + { + return nullptr; + } + +private: + std::shared_ptr state_; +}; + +core::SchemaPusherConfig make_config() +{ + return core::SchemaPusherConfig{ .collection_id = "pedals", + .max_flatbuffer_size = 16, + .tensor_identifier = "pedal_state", + .localized_name = "Pedal state", + .app_name = "PusherTest" }; +} + +} // namespace + +TEST_CASE("SchemaPusher delegates samples to its channel", "[pusherio][unit]") +{ + auto state = std::make_shared(); + core::PluginSessionHandle session = std::make_shared(state); + core::SchemaPusher pusher(session->create_schema_push_channel(make_config())); + const std::vector payload{ 1, 2, 3 }; + + pusher.push_buffer(payload.data(), payload.size(), 100, 80); + + REQUIRE(state->config.collection_id == "pedals"); + REQUIRE(state->config.tensor_identifier == "pedal_state"); + REQUIRE(state->payload == payload); + REQUIRE(state->local_time_ns == 100); + REQUIRE(state->device_time_ns == 80); +} + +TEST_CASE("SchemaPusher leaves session ownership with its caller", "[pusherio][unit]") +{ + auto state = std::make_shared(); + auto session = std::make_shared(state); + + { + core::SchemaPusher pusher(session->create_schema_push_channel(make_config())); + REQUIRE(session.use_count() == 1); + REQUIRE_FALSE(state->session_destroyed); + REQUIRE_FALSE(state->channel_destroyed); + } + + REQUIRE(state->channel_destroyed); + REQUIRE_FALSE(state->session_destroyed); + + session.reset(); + REQUIRE(state->session_destroyed); + REQUIRE(state->channel_destroyed_before_session); +} + +TEST_CASE("SchemaPusher validates transport-independent buffer metadata", "[pusherio][unit]") +{ + auto state = std::make_shared(); + auto session = std::make_shared(state); + core::SchemaPusher pusher(session->create_schema_push_channel(make_config())); + const std::vector oversized(17, 0); + + REQUIRE_THROWS_AS(pusher.push_buffer(oversized.data(), oversized.size(), 100, 80), std::runtime_error); + REQUIRE_THROWS_AS(pusher.push_buffer(nullptr, 1, 100, 80), std::invalid_argument); + REQUIRE_THROWS_AS(pusher.push_buffer(nullptr, 0, 100, -1), std::runtime_error); +}