MAYA-125723 - Implement display layer's bbox mode for instanced geometry

lib/mayaUsd/render/vp2RenderDelegate/draw_item.cpp

@@ -104,6 +104,7 @@ HdVP2DrawItem::AddRenderItem(MHWRender::MRenderItem* item, const HdGeomSubset* g
 
     renderItemData._renderItem = item;
     renderItemData._renderItemName = item->name();
+    renderItemData._enabled = item->isEnabled();
 #if MAYA_API_VERSION >= 20220000
     auto sharingData = HdVP2DrawItemUserDataPtr::dynamic_pointer_cast<>(item->getCustomData());
 #else

lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.cpp

@@ -319,6 +319,16 @@ TfToken MayaUsdRPrim::_GetMaterialNetworkToken(const TfToken& reprToken) const
     return _displayLayerModes._texturing ? reprToken : TfToken();
 }
 
+HdReprSharedPtr MayaUsdRPrim::_FindRepr(const ReprVector& reprs, const TfToken& reprToken)
+{
+    ReprVector::const_iterator it
+        = std::find_if(reprs.begin(), reprs.end(), [reprToken](ReprVector::const_reference e) {
+              return reprToken == e.first;
+          });
+
+    return (it != reprs.end() ? it->second : nullptr);
+}
+
 HdReprSharedPtr MayaUsdRPrim::_InitReprCommon(
     HdRprim&       refThis,
     TfToken const& reprToken,
@@ -330,38 +340,49 @@ HdReprSharedPtr MayaUsdRPrim::_InitReprCommon(
         _FirstInitRepr(dirtyBits, id);
     }
 
-    _SyncDisplayLayerModes(refThis);
+    auto* const param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
+    auto&       drawScene = param->GetDrawScene();
+
+    // display layers handling
+    if (!drawScene.GetUsdImagingDelegate()->GetInstancerId(id).IsEmpty()) {
+        _displayLayerModes = DisplayLayerModes();
+
+        // Instanced primitives with instances in display layers use 'forced' representations to
+        // draw those specific instances, so the 'forced' representations should be inited alongside
+        if (reprToken != HdVP2ReprTokens->forcedBbox) {
+            refThis.InitRepr(
+                drawScene.GetUsdImagingDelegate(), HdVP2ReprTokens->forcedBbox, dirtyBits);
+        }
+    } else {
+        // Sync display layer modes for non-instanced prims.
+        // For instanced prims, this will be done inside Sync method on a per-instance basis
+        _SyncDisplayLayerModes();
+    }
 
     _UpdateReprOverrides(reprs);
 
-    // Finally we can get to actually init/dirty the repr
-    ReprVector::const_iterator it
-        = std::find_if(reprs.begin(), reprs.end(), [reprToken](ReprVector::const_reference e) {
-              return reprToken == e.first;
-          });
-    HdReprSharedPtr curRepr = (it != reprs.end() ? it->second : nullptr);
+    // Find the current representation in the array of all inited representations
+    HdReprSharedPtr curRepr = _FindRepr(reprs, reprToken);
 
     // In repr override mode, call InitRepr for the representation that overrides.
     if (_reprOverride != kNone) {
         TfToken overrideToken = _GetOverrideToken(reprToken);
         if (!overrideToken.IsEmpty() && (overrideToken != reprToken)) {
-            auto* const param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
-            auto&       drawScene = param->GetDrawScene();
             refThis.InitRepr(drawScene.GetUsdImagingDelegate(), overrideToken, dirtyBits);
             if (curRepr) {
                 return nullptr; // if the overriden repr is already created, we can safely exit here
             }
         }
     }
 
+    // Finalize initialization
+
     if (curRepr) {
         _SetDirtyRepr(curRepr);
         return nullptr;
     }
 
-    // set dirty bit to say we need to sync a new repr
-    *dirtyBits |= HdChangeTracker::NewRepr;
-
+    *dirtyBits |= HdChangeTracker::NewRepr; // set dirty bit to say we need to sync a new repr
     reprs.emplace_back(reprToken, std::make_shared<HdRepr>());
     return reprs.back().second;
 }
@@ -746,7 +767,7 @@ void MayaUsdRPrim::_PopulateDisplayLayerModes(const MString&, DisplayLayerModes&
 }
 #endif
 
-void MayaUsdRPrim::_SyncDisplayLayerModes(const HdRprim& refThis)
+void MayaUsdRPrim::_SyncDisplayLayerModes()
 {
     auto* const          param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
     ProxyRenderDelegate& drawScene = param->GetDrawScene();
@@ -757,14 +778,34 @@ void MayaUsdRPrim::_SyncDisplayLayerModes(const HdRprim& refThis)
     }
 
     _displayLayerModesFrame = drawScene.GetFrameCounter();
+    MString pathString = drawScene.GetUfePathPrefix() + _PrimSegmentString[0];
+    _PopulateDisplayLayerModes(pathString, _displayLayerModes);
+}
 
-    // If the prim is not instanced, populate _displayLayerModes.
-    // Otherwise display layer modes will be handled later, in the instancing loop.
-    if (refThis.GetInstancerId().IsEmpty()) {
-        MString pathString = drawScene.GetUfePathPrefix() + _PrimSegmentString[0];
-        _PopulateDisplayLayerModes(pathString, _displayLayerModes);
-    } else {
-        _displayLayerModes = DisplayLayerModes();
+void MayaUsdRPrim::_SyncDisplayLayerModesInstanced(SdfPath const& id, unsigned int instanceCount)
+{
+    auto* const          param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
+    ProxyRenderDelegate& drawScene = param->GetDrawScene();
+
+    // First check if the status need updating
+    if (drawScene.GetFrameCounter() == _displayLayerModesFrame) {
+        return;
+    }
+
+    _displayLayerModesFrame = drawScene.GetFrameCounter();
+    const MString ufePathPrefix = drawScene.GetUfePathPrefix();
+
+    _needForcedBBox = false;
+    _displayLayerModesInstanced.resize(instanceCount);
+    for (unsigned int usdInstanceId = 0; usdInstanceId < instanceCount; usdInstanceId++) {
+        const MString instancePath
+            = ufePathPrefix + drawScene.GetScenePrimPath(id, usdInstanceId).GetString().c_str();
+        auto& displayLayerModes = _displayLayerModesInstanced[usdInstanceId];
+        _PopulateDisplayLayerModes(instancePath, displayLayerModes);
+
+        if (displayLayerModes._reprOverride == kBBox) {
+            _needForcedBBox = true;
+        }
     }
 }
 
@@ -988,4 +1029,61 @@ bool MayaUsdRPrim::_GetMaterialPrimvars(
     return true;
 }
 
+bool MayaUsdRPrim::_ShouldSkipInstance(unsigned int usdInstanceId, const TfToken& reprToken) const
+{
+    const auto& displayLayerModes = _displayLayerModesInstanced[usdInstanceId];
+    if (!displayLayerModes._visibility) {
+        return true;
+    }
+
+    if (reprToken == HdVP2ReprTokens->forcedBbox) {
+        if (displayLayerModes._reprOverride != kBBox) {
+            return true;
+        }
+    } else {
+        if (displayLayerModes._reprOverride == kBBox) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void MayaUsdRPrim::_SyncForcedReprs(
+    HdRprim&          refThis,
+    HdSceneDelegate*  delegate,
+    HdRenderParam*    renderParam,
+    HdDirtyBits*      dirtyBits,
+    ReprVector const& reprs)
+{
+    // Forced representations work only for instanced primitives
+    if (refThis.GetInstancerId().IsEmpty()) {
+        return;
+    }
+
+    auto* const          param = static_cast<HdVP2RenderParam*>(_delegate->GetRenderParam());
+    ProxyRenderDelegate& drawScene = param->GetDrawScene();
+
+    // First check if the sync still needs to be performed
+    if (drawScene.GetFrameCounter() == _forcedReprsFrame) {
+        return;
+    }
+
+    _forcedReprsFrame = drawScene.GetFrameCounter();
+
+    RenderItemFunc hideDrawItem = [this](HdVP2DrawItem::RenderItemData& renderItemData) {
+        if (renderItemData._enabled) {
+            renderItemData._enabled = false;
+            _delegate->GetVP2ResourceRegistry().EnqueueCommit(
+                [&renderItemData]() { renderItemData._renderItem->enable(false); });
+        }
+    };
+
+    if (_needForcedBBox) {
+        refThis.Sync(delegate, renderParam, dirtyBits, HdVP2ReprTokens->forcedBbox);
+    } else {
+        _ForEachRenderItemInRepr(_FindRepr(reprs, HdVP2ReprTokens->forcedBbox), hideDrawItem);
+    }
+}
+
 PXR_NAMESPACE_CLOSE_SCOPE

lib/mayaUsd/render/vp2RenderDelegate/mayaPrimCommon.h

@@ -216,6 +216,8 @@ class MayaUsdRPrim
     static void
     _PopulateDisplayLayerModes(const MString& pathString, DisplayLayerModes& displayLayerModes);
 
+    static HdReprSharedPtr _FindRepr(const ReprVector& reprs, const TfToken& reprToken);
+
     void _CommitMVertexBuffer(MHWRender::MVertexBuffer* const, void*) const;
 
     void _UpdateTransform(
@@ -258,7 +260,17 @@ class MayaUsdRPrim
         ReprVector const&  reprs,
         TfToken const&     renderTag);
 
-    void _SyncDisplayLayerModes(const HdRprim& refThis);
+    void _SyncDisplayLayerModes();
+    void _SyncDisplayLayerModesInstanced(SdfPath const& id, unsigned int instanceCount);
+
+    bool _ShouldSkipInstance(unsigned int usdInstanceId, const TfToken& reprToken) const;
+
+    void _SyncForcedReprs(
+        HdRprim&          refThis,
+        HdSceneDelegate*  delegate,
+        HdRenderParam*    renderParam,
+        HdDirtyBits*      dirtyBits,
+        ReprVector const& reprs);
 
     void _UpdatePrimvarSourcesGeneric(
         HdSceneDelegate*       sceneDelegate,
@@ -324,8 +336,13 @@ class MayaUsdRPrim
     HdVP2SelectionStatus _selectionStatus { kUnselected };
 
     //! Modes requested by display layer along with the frame they are updated on
-    DisplayLayerModes _displayLayerModes;
-    uint64_t          _displayLayerModesFrame { 0 };
+    DisplayLayerModes              _displayLayerModes;
+    std::vector<DisplayLayerModes> _displayLayerModesInstanced;
+    uint64_t                       _displayLayerModesFrame { 0 };
+
+    // forced representations runtime state
+    bool     _needForcedBBox = false;
+    uint64_t _forcedReprsFrame { 0 };
 
     //! HideOnPlayback status of the Rprim
     bool _hideOnPlayback { false };

lib/mayaUsd/render/vp2RenderDelegate/mesh.cpp

@@ -1039,6 +1039,8 @@ void HdVP2Mesh::Sync(
 
     // Draw item update is controlled by its own dirty bits.
     _UpdateRepr(delegate, reprToken);
+
+    _SyncForcedReprs(*this, delegate, renderParam, dirtyBits, _reprs);
 }
 
 /*! \brief  Returns the minimal set of dirty bits to place in the
@@ -1245,13 +1247,18 @@ void HdVP2Mesh::_InitRepr(const TfToken& reprToken, HdDirtyBits* dirtyBits)
                 drawItem->AddUsage(HdVP2DrawItem::kSelectionHighlight);
             }
             // The item is used for bbox display and selection highlight.
-            else if (reprToken == HdVP2ReprTokens->bbox) {
+            else if (
+                reprToken == HdVP2ReprTokens->bbox || reprToken == HdVP2ReprTokens->forcedBbox) {
                 renderItem = _CreateBoundingBoxRenderItem(
                     renderItemName,
                     kOpaqueBlue,
                     MSelectionMask::kSelectMeshes,
                     MHWRender::MFrameContext::kExcludeMeshes);
                 drawItem->AddUsage(HdVP2DrawItem::kSelectionHighlight);
+                if (reprToken == HdVP2ReprTokens->forcedBbox) {
+                    renderItem->enable(false);
+                    renderItem->setDrawMode(MHWRender::MGeometry::kAll);
+                }
             }
             break;
 #ifndef MAYA_NEW_POINT_SNAPPING_SUPPORT
@@ -1860,8 +1867,7 @@ void HdVP2Mesh::_UpdateDrawItem(
             }
 #endif
 
-            DisplayLayerModes displayLayerModes;
-            const MString     ufePathPrefix = drawScene.GetUfePathPrefix();
+            _SyncDisplayLayerModesInstanced(id, instanceCount);
 
             stateToCommit._instanceTransforms = std::make_shared<MMatrixArray>();
             stateToCommit._instanceColors = std::make_shared<MFloatArray>();
@@ -1871,10 +1877,7 @@ void HdVP2Mesh::_UpdateDrawItem(
                     continue;
 
                 // Check display layer modes of this instance
-                const MString instancePath = ufePathPrefix
-                    + drawScene.GetScenePrimPath(id, usdInstanceId).GetString().c_str();
-                _PopulateDisplayLayerModes(instancePath, displayLayerModes);
-                if (!displayLayerModes._visibility)
+                if (_ShouldSkipInstance(usdInstanceId, reprToken))
                     continue;
 
 #ifndef MAYA_UPDATE_UFE_IDENTIFIER_SUPPORT

lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.cpp

@@ -291,6 +291,9 @@ void _ConfigureReprs()
     // Edge desc for bbox display.
     HdMesh::ConfigureRepr(HdVP2ReprTokens->bbox, reprDescEdge);
 
+    // Forced representations are used for instanced geometry with display layer overrides
+    HdMesh::ConfigureRepr(HdVP2ReprTokens->forcedBbox, reprDescEdge);
+
     // smooth hull for untextured display
     HdBasisCurves::ConfigureRepr(
         HdVP2ReprTokens->smoothHullUntextured, HdBasisCurvesGeomStylePatch);

lib/mayaUsd/render/vp2RenderDelegate/tokens.h

@@ -25,7 +25,8 @@ PXR_NAMESPACE_OPEN_SCOPE
 #define HDVP2_REPR_TOKENS \
     (bbox) \
     (defaultMaterial) \
-    (smoothHullUntextured)
+    (smoothHullUntextured) \
+    (forcedBbox)
 
 #define HDVP2_TOKENS \
     (displayColorAndOpacity) \