Merge topic 'vr-physical-geometry-rework'

803d3a32 ENH: Use pose matrices instead of camera parameters for VR controllers
Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: Ken Martin <ken.martin@kitware.com>
Merge-request: !4706

Rendering/Core/vtkRenderWindowInteractor3D.cxx

@@ -25,9 +25,8 @@
 #include "vtkActor.h"
 #include "vtkObjectFactory.h"
 #include "vtkCommand.h"
-#include "vtkNew.h"
-// #include "vtkPropPicker.h"
 #include "vtkMath.h"
+#include "vtkMatrix4x4.h"
 
 vtkStandardNewMacro(vtkRenderWindowInteractor3D);
 
@@ -345,3 +344,113 @@ void vtkRenderWindowInteractor3D::RightButtonReleaseEvent()
   }
   this->InvokeEvent(vtkCommand::RightButtonReleaseEvent, nullptr);
 }
+
+//------------------------------------------------------------------
+void vtkRenderWindowInteractor3D::SetPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex)
+{
+  if (!poseMatrix || pointerIndex < 0 || pointerIndex >= VTKI_MAX_POINTERS)
+  {
+    return;
+  }
+
+  bool poseDifferent = false;
+  for (int i = 0; i < 4; i++)
+  {
+    for (int j = 0; j < 4; j++)
+    {
+      if ( fabs(this->PhysicalEventPoses[pointerIndex]->GetElement(i, j) - poseMatrix->GetElement(i, j)) >= 1e-3 )
+      {
+        poseDifferent = true;
+        break;
+      }
+    }
+  }
+
+
+  if (poseDifferent)
+  {
+    this->LastPhysicalEventPoses[pointerIndex]->DeepCopy(PhysicalEventPoses[pointerIndex]);
+    this->PhysicalEventPoses[pointerIndex]->DeepCopy(poseMatrix);
+    this->Modified();
+  }
+}
+
+//------------------------------------------------------------------
+void vtkRenderWindowInteractor3D::SetWorldEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex)
+{
+  if (!poseMatrix || pointerIndex < 0 || pointerIndex >= VTKI_MAX_POINTERS)
+  {
+    return;
+  }
+
+  bool poseDifferent = false;
+  for (int i = 0; i < 4; i++)
+  {
+    for (int j = 0; j < 4; j++)
+    {
+      if ( fabs(this->WorldEventPoses[pointerIndex]->GetElement(i, j) - poseMatrix->GetElement(i, j)) >= 1e-3 )
+      {
+        poseDifferent = true;
+        break;
+      }
+    }
+  }
+
+
+  if (poseDifferent)
+  {
+    this->LastWorldEventPoses[pointerIndex]->DeepCopy(WorldEventPoses[pointerIndex]);
+    this->WorldEventPoses[pointerIndex]->DeepCopy(poseMatrix);
+    this->Modified();
+  }
+}
+
+//------------------------------------------------------------------
+void vtkRenderWindowInteractor3D::GetWorldEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex)
+{
+  if (pointerIndex >= VTKI_MAX_POINTERS || !poseMatrix)
+  {
+    return;
+  }
+  poseMatrix->DeepCopy(WorldEventPoses[pointerIndex]);
+}
+
+//------------------------------------------------------------------
+void vtkRenderWindowInteractor3D::GetLastWorldEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex)
+{
+  if (pointerIndex >= VTKI_MAX_POINTERS || !poseMatrix)
+  {
+    return;
+  }
+  poseMatrix->DeepCopy(LastWorldEventPoses[pointerIndex]);
+}
+
+//------------------------------------------------------------------
+void vtkRenderWindowInteractor3D::GetPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex)
+{
+  if (pointerIndex >= VTKI_MAX_POINTERS || !poseMatrix)
+  {
+    return;
+  }
+  poseMatrix->DeepCopy(PhysicalEventPoses[pointerIndex]);
+}
+
+//------------------------------------------------------------------
+void vtkRenderWindowInteractor3D::GetLastPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex)
+{
+  if (pointerIndex >= VTKI_MAX_POINTERS || !poseMatrix)
+  {
+    return;
+  }
+  poseMatrix->DeepCopy(LastPhysicalEventPoses[pointerIndex]);
+}
+
+//------------------------------------------------------------------
+void vtkRenderWindowInteractor3D::GetStartingPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex)
+{
+  if (pointerIndex >= VTKI_MAX_POINTERS || !poseMatrix)
+  {
+    return;
+  }
+  poseMatrix->DeepCopy(StartingPhysicalEventPoses[pointerIndex]);
+}

Rendering/Core/vtkRenderWindowInteractor3D.h

@@ -32,7 +32,10 @@
 #include "vtkRenderingCoreModule.h" // For export macro
 #include "vtkRenderWindowInteractor.h"
 
+#include "vtkNew.h" // ivars
+
 class vtkCamera;
+class vtkMatrix4x4;
 enum class vtkEventDataDevice;
 enum class vtkEventDataDeviceInput;
 
@@ -70,11 +73,9 @@ public:
 
   //@{
   /**
-   * With VR we know the world coordinate positions
-   * and orientations of events. These methods
-   * support querying them instead of going through
-   * a display X,Y coordinate approach as is standard
-   * for mouse/touch events
+   * With VR we know the world coordinate positions and orientations of events.
+   * These methods support querying them instead of going through a display X,Y
+   * coordinate approach as is standard for mouse/touch events
    */
   virtual double *GetWorldEventPosition(int pointerIndex)
   {
@@ -108,13 +109,15 @@ public:
     }
     return this->LastWorldEventOrientations[pointerIndex];
   }
+  virtual void GetWorldEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex);
+  virtual void GetLastWorldEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex);
   //@}
 
   //@{
   /**
    * With VR we know the physical/room coordinate positions
-   * and orientations of events. These methods
-   * support setting them.
+   * and orientations of events.
+   * These methods support setting them.
    */
   virtual void SetPhysicalEventPosition(double x, double y, double z, int pointerIndex)
   {
@@ -143,6 +146,18 @@ public:
       this->Modified();
     }
   }
+  virtual void SetPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex);
+  //@}
+
+  //@{
+  /**
+   * With VR we know the physical/room coordinate positions
+   * and orientations of events.
+   * These methods support getting them.
+   */
+  virtual void GetPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex);
+  virtual void GetLastPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex);
+  virtual void GetStartingPhysicalEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex);
   //@}
 
   //@{
@@ -209,6 +224,7 @@ public:
       this->Modified();
     }
   }
+  virtual void SetWorldEventPose(vtkMatrix4x4* poseMatrix, int pointerIndex);
   //@}
 
   //@{
@@ -280,6 +296,11 @@ protected:
   double   StartingPhysicalEventPositions[VTKI_MAX_POINTERS][3];
   double   WorldEventOrientations[VTKI_MAX_POINTERS][4];
   double   LastWorldEventOrientations[VTKI_MAX_POINTERS][4];
+  vtkNew<vtkMatrix4x4> WorldEventPoses[VTKI_MAX_POINTERS];
+  vtkNew<vtkMatrix4x4> LastWorldEventPoses[VTKI_MAX_POINTERS];
+  vtkNew<vtkMatrix4x4> PhysicalEventPoses[VTKI_MAX_POINTERS];
+  vtkNew<vtkMatrix4x4> LastPhysicalEventPoses[VTKI_MAX_POINTERS];
+  vtkNew<vtkMatrix4x4> StartingPhysicalEventPoses[VTKI_MAX_POINTERS];
   void RecognizeGesture(vtkCommand::EventIds) override;
 
 private:

Rendering/OpenVR/vtkOpenVRControlsHelper.cxx

@@ -30,7 +30,7 @@ PURPOSE.  See the above copyright notice for more information.
 #include "vtkWindow.h"
 #include "vtkCamera.h"
 #include "vtkLineSource.h"
-
+#include "vtkTransform.h"
 #include "vtkSmartPointer.h"
 
 vtkStandardNewMacro(vtkOpenVRControlsHelper);

Rendering/OpenVR/vtkOpenVRHardwarePicker.cxx

@@ -23,16 +23,7 @@ PURPOSE.  See the above copyright notice for more information.
 #include "vtkOpenVRRenderWindowInteractor.h"
 #include "vtkRenderer.h"
 #include "vtkSelection.h"
-
-// #include "vtkAssemblyNode.h"
-// #include "vtkAssemblyPath.h"
-// #include "vtkRenderer.h"
-// #include "vtkMath.h"
-// #include "vtkCamera.h"
-// #include "vtkBox.h"
-// #include "vtkRenderWindow.h"
-// #include "vtkRenderWindowInteractor3D.h"
-// #include "vtkTransform.h"
+#include "vtkTransform.h"
 
 vtkStandardNewMacro(vtkOpenVRHardwarePicker);
 

Rendering/OpenVR/vtkOpenVRPanelRepresentation.cxx

@@ -394,21 +394,10 @@ void vtkOpenVRPanelRepresentation::ComputeMatrix(vtkRenderer *ren)
       rw->GetTrackedDevicePose(vtkEventDataDevice::LeftController, &tdPose);
       if (tdPose && tdPose->bPoseIsValid)
       {
-        double pos[3];
-        double ppos[3];
-        double wxyz[4];
-        double wdir[3];
+        vtkNew<vtkMatrix4x4> poseMatrixWorld;
         static_cast<vtkOpenVRRenderWindowInteractor *>(rw->GetInteractor())
-          ->ConvertPoseToWorldCoordinates(*tdPose, pos, wxyz, ppos, wdir);
-
-        double scale = rw->GetPhysicalScale();
-
-        this->TempTransform->Identity();
-        this->TempTransform->PreMultiply();
-        this->TempTransform->Translate(pos[0], pos[1], pos[2]);
-        this->TempTransform->Scale(scale, scale, scale);
-        this->TempTransform->RotateWXYZ(wxyz[0], wxyz[1], wxyz[2], wxyz[3]);
-        this->TextActor->GetUserMatrix()->DeepCopy(this->TempTransform->GetMatrix());
+          ->ConvertOpenVRPoseToMatrices(*tdPose, poseMatrixWorld);
+        this->TextActor->GetUserMatrix()->DeepCopy(poseMatrixWorld);
       }
     }
 
@@ -418,21 +407,10 @@ void vtkOpenVRPanelRepresentation::ComputeMatrix(vtkRenderer *ren)
       rw->GetTrackedDevicePose(vtkEventDataDevice::RightController, &tdPose);
       if (tdPose && tdPose->bPoseIsValid)
       {
-        double pos[3];
-        double ppos[3];
-        double wxyz[4];
-        double wdir[3];
+        vtkNew<vtkMatrix4x4> poseMatrixWorld;
         static_cast<vtkOpenVRRenderWindowInteractor *>(rw->GetInteractor())
-          ->ConvertPoseToWorldCoordinates(*tdPose, pos, wxyz, ppos, wdir);
-
-        double scale = rw->GetPhysicalScale();
-
-        this->TempTransform->Identity();
-        this->TempTransform->PreMultiply();
-        this->TempTransform->Translate(pos[0], pos[1], pos[2]);
-        this->TempTransform->Scale(scale, scale, scale);
-        this->TempTransform->RotateWXYZ(wxyz[0], wxyz[1], wxyz[2], wxyz[3]);
-        this->TextActor->GetUserMatrix()->DeepCopy(this->TempTransform->GetMatrix());
+          ->ConvertOpenVRPoseToMatrices(*tdPose, poseMatrixWorld);
+        this->TextActor->GetUserMatrix()->DeepCopy(poseMatrixWorld);
       }
     }
   }

Rendering/OpenVR/vtkOpenVRRenderWindow.cxx

@@ -425,7 +425,6 @@ void vtkOpenVRRenderWindow::UpdateHMDMatrixPose()
       double hmdY_Physical[3] = { tdPose.mDeviceToAbsoluteTracking.m[0][1],
                                   tdPose.mDeviceToAbsoluteTracking.m[1][1],
                                   tdPose.mDeviceToAbsoluteTracking.m[2][1] };
-      double hmdZ_Physical[3] = {0.0};
       double hmdPosition_Physical[3] = { tdPose.mDeviceToAbsoluteTracking.m[0][3],
                                          tdPose.mDeviceToAbsoluteTracking.m[1][3],
                                          tdPose.mDeviceToAbsoluteTracking.m[2][3] };
@@ -787,6 +786,73 @@ void vtkOpenVRRenderWindow::GetTrackedDevicePose(
   }
 }
 
+void vtkOpenVRRenderWindow::SetPhysicalViewDirection(double x, double y, double z)
+{
+  if ( this->PhysicalViewDirection[0] != x
+    || this->PhysicalViewDirection[1] != y
+    || this->PhysicalViewDirection[2] != z )
+  {
+    this->PhysicalViewDirection[0] = x;
+    this->PhysicalViewDirection[1] = y;
+    this->PhysicalViewDirection[2] = z;
+    this->InvokeEvent(vtkOpenVRRenderWindow::PhysicalToWorldMatrixModified);
+    this->Modified();
+  }
+}
+
+void vtkOpenVRRenderWindow::SetPhysicalViewDirection(double dir[3])
+{
+  this->SetPhysicalViewDirection(dir[0], dir[1], dir[2]);
+}
+
+void vtkOpenVRRenderWindow::SetPhysicalViewUp(double x, double y, double z)
+{
+  if ( this->PhysicalViewUp[0] != x
+    || this->PhysicalViewUp[1] != y
+    || this->PhysicalViewUp[2] != z )
+  {
+    this->PhysicalViewUp[0] = x;
+    this->PhysicalViewUp[1] = y;
+    this->PhysicalViewUp[2] = z;
+    this->InvokeEvent(vtkOpenVRRenderWindow::PhysicalToWorldMatrixModified);
+    this->Modified();
+  }
+}
+
+void vtkOpenVRRenderWindow::SetPhysicalViewUp(double dir[3])
+{
+  this->SetPhysicalViewUp(dir[0], dir[1], dir[2]);
+}
+
+void vtkOpenVRRenderWindow::SetPhysicalTranslation(double x, double y, double z)
+{
+  if ( this->PhysicalTranslation[0] != x
+    || this->PhysicalTranslation[1] != y
+    || this->PhysicalTranslation[2] != z )
+  {
+    this->PhysicalTranslation[0] = x;
+    this->PhysicalTranslation[1] = y;
+    this->PhysicalTranslation[2] = z;
+    this->InvokeEvent(vtkOpenVRRenderWindow::PhysicalToWorldMatrixModified);
+    this->Modified();
+  }
+}
+
+void vtkOpenVRRenderWindow::SetPhysicalTranslation(double trans[3])
+{
+  this->SetPhysicalTranslation(trans[0], trans[1], trans[2]);
+}
+
+void vtkOpenVRRenderWindow::SetPhysicalScale(double scale)
+{
+  if (this->PhysicalScale != scale)
+  {
+    this->PhysicalScale = scale;
+    this->InvokeEvent(vtkOpenVRRenderWindow::PhysicalToWorldMatrixModified);
+    this->Modified();
+  }
+}
+
 void vtkOpenVRRenderWindow::SetPhysicalToWorldMatrix(vtkMatrix4x4* matrix)
 {
   if (!matrix)
@@ -797,7 +863,11 @@ void vtkOpenVRRenderWindow::SetPhysicalToWorldMatrix(vtkMatrix4x4* matrix)
   vtkNew<vtkTransform> hmdToWorldTransform;
   hmdToWorldTransform->SetMatrix(matrix);
 
-  hmdToWorldTransform->GetPosition(this->PhysicalTranslation);
+  double translation[3] = {0.0};
+  hmdToWorldTransform->GetPosition(translation);
+  this->PhysicalTranslation[0] = (-1.0) * translation[0];
+  this->PhysicalTranslation[1] = (-1.0) * translation[1];
+  this->PhysicalTranslation[2] = (-1.0) * translation[2];
 
   double scale[3] = {0.0};
   hmdToWorldTransform->GetScale(scale);
@@ -811,6 +881,9 @@ void vtkOpenVRRenderWindow::SetPhysicalToWorldMatrix(vtkMatrix4x4* matrix)
   this->PhysicalViewDirection[1] = (-1.0) * matrix->GetElement(1,2);
   this->PhysicalViewDirection[2] = (-1.0) * matrix->GetElement(2,2);
   vtkMath::Normalize(this->PhysicalViewDirection);
+
+  this->InvokeEvent(vtkOpenVRRenderWindow::PhysicalToWorldMatrixModified);
+  this->Modified();
 }
 
 void vtkOpenVRRenderWindow::GetPhysicalToWorldMatrix(vtkMatrix4x4* physicalToWorldMatrix)
@@ -835,6 +908,6 @@ void vtkOpenVRRenderWindow::GetPhysicalToWorldMatrix(vtkMatrix4x4* physicalToWor
     physicalToWorldMatrix->SetElement(row, 0, physicalX_NonscaledWorld[row]*this->PhysicalScale);
     physicalToWorldMatrix->SetElement(row, 1, physicalY_NonscaledWorld[row]*this->PhysicalScale);
     physicalToWorldMatrix->SetElement(row, 2, physicalZ_NonscaledWorld[row]*this->PhysicalScale);
-    physicalToWorldMatrix->SetElement(row, 3, this->PhysicalTranslation[row]);
+    physicalToWorldMatrix->SetElement(row, 3, -this->PhysicalTranslation[row]);
   }
 }

Rendering/OpenVR/vtkOpenVRRenderWindow.h

@@ -58,6 +58,11 @@ class vtkTransform;
 class VTKRENDERINGOPENVR_EXPORT vtkOpenVRRenderWindow : public vtkOpenGLRenderWindow
 {
 public:
+  enum
+    {
+    PhysicalToWorldMatrixModified = vtkCommand::UserEvent + 200
+    };
+
   static vtkOpenVRRenderWindow *New();
   vtkTypeMacro(vtkOpenVRRenderWindow,vtkOpenGLRenderWindow);
   void PrintSelf(ostream& os, vtkIndent indent);
@@ -147,13 +152,16 @@ public:
    * some cases users may not want the Y axis to be up
    * and these methods allow them to control it.
    */
-  vtkSetVector3Macro(PhysicalViewDirection, double);
-  vtkSetVector3Macro(PhysicalViewUp, double);
+  virtual void SetPhysicalViewDirection(double,double,double);
+  virtual void SetPhysicalViewDirection(double[3]);
   vtkGetVector3Macro(PhysicalViewDirection, double);
+  virtual void SetPhysicalViewUp(double,double,double);
+  virtual void SetPhysicalViewUp(double[3]);
   vtkGetVector3Macro(PhysicalViewUp, double);
-  vtkSetVector3Macro(PhysicalTranslation, double);
+  virtual void SetPhysicalTranslation(double,double,double);
+  virtual void SetPhysicalTranslation(double[3]);
   vtkGetVector3Macro(PhysicalTranslation, double);
-  vtkSetMacro(PhysicalScale, double);
+  virtual void SetPhysicalScale(double);
   vtkGetMacro(PhysicalScale, double);
   //@}
 
@@ -361,9 +369,13 @@ protected:
 
   // used in computing the pose
   vtkTransform *HMDTransform;
+  /// -Z axis of the Physical to World matrix
   double PhysicalViewDirection[3];
+  /// Y axis of the Physical to World matrix
   double PhysicalViewUp[3];
+  /// Inverse of the translation component of the Physical to World matrix, in mm
   double PhysicalTranslation[3];
+  /// Scale of the Physical to World matrix
   double PhysicalScale;
 
   // for the overlay

Rendering/OpenVR/vtkOpenVRRenderWindowInteractor.cxx

@@ -32,6 +32,8 @@
 #include "vtkObjectFactory.h"
 #include "vtkOpenVRCamera.h"
 #include "vtkTextureObject.h"
+#include "vtkTransform.h"
+#include "vtkMatrix4x4.h"
 
 vtkStandardNewMacro(vtkOpenVRRenderWindowInteractor);
 
@@ -88,12 +90,44 @@ double *vtkOpenVRRenderWindowInteractor::GetPhysicalTranslation(vtkCamera *)
   return win->GetPhysicalTranslation();
 }
 
+void vtkOpenVRRenderWindowInteractor::ConvertOpenVRPoseToMatrices(
+  const vr::TrackedDevicePose_t &tdPose,
+  vtkMatrix4x4* poseMatrixWorld,
+  vtkMatrix4x4* poseMatrixPhysical/*=nullptr*/)
+{
+  if (!poseMatrixWorld && !poseMatrixPhysical)
+  {
+    return;
+  }
+
+  vtkNew<vtkMatrix4x4> poseMatrixPhysicalTemp;
+  for (int row=0; row<3; ++row)
+  {
+    for (int col=0; col<4; ++col)
+    {
+      poseMatrixPhysicalTemp->SetElement(row, col, tdPose.mDeviceToAbsoluteTracking.m[row][col]);
+    }
+  }
+  if (poseMatrixPhysical)
+  {
+    poseMatrixPhysical->DeepCopy(poseMatrixPhysicalTemp);
+  }
+
+  if (poseMatrixWorld)
+  {
+    vtkOpenVRRenderWindow* win = vtkOpenVRRenderWindow::SafeDownCast(this->RenderWindow);
+    vtkNew<vtkMatrix4x4> physicalToWorldMatrix;
+    win->GetPhysicalToWorldMatrix(physicalToWorldMatrix);
+    vtkMatrix4x4::Multiply4x4(physicalToWorldMatrix, poseMatrixPhysicalTemp, poseMatrixWorld);
+  }
+}
+
 void vtkOpenVRRenderWindowInteractor::ConvertPoseToWorldCoordinates(
   const vr::TrackedDevicePose_t &tdPose,
-  double pos[3],
-  double wxyz[4],
-  double ppos[3],
-  double wdir[3])
+  double pos[3],  // Output world position
+  double wxyz[4], // Output world orientation quaternion
+  double ppos[3], // Output physical position
+  double wdir[3]) // Output world view direction (-Z)
 {
   vtkOpenVRRenderWindow *win =
     vtkOpenVRRenderWindow::SafeDownCast(this->RenderWindow);
@@ -154,7 +188,7 @@ void vtkOpenVRRenderWindowInteractor::ConvertPoseToWorldCoordinates(
   vtkMath::Matrix3x3ToQuaternion(ortho, wxyz);
 
   // calc the return value wxyz
- double mag = sqrt( wxyz[1] * wxyz[1] + wxyz[2] * wxyz[2] + wxyz[3] * wxyz[3] );
+  double mag = sqrt( wxyz[1] * wxyz[1] + wxyz[2] * wxyz[2] + wxyz[3] * wxyz[3] );
 
   if ( mag != 0.0 )
   {
@@ -348,11 +382,20 @@ void vtkOpenVRRenderWindowInteractor::DoOneEvent(vtkOpenVRRenderWindow *renWin,
         this->PointerIndexLookup[pointerIndex] = tdi;
 
         vr::TrackedDevicePose_t &tdPose = renWin->GetTrackedDevicePose(tdi);
-        double pos[3];
-        double ppos[3];
-        double wxyz[4];
-        double wdir[3];
+        double pos[3] = {0.0};
+        double ppos[3] = {0.0};
+        double wxyz[4] = {0.0};
+        double wdir[3] = {0.0};
         this->ConvertPoseToWorldCoordinates(tdPose, pos, wxyz, ppos, wdir);
+        this->SetWorldEventPosition(pos[0],pos[1],pos[2],pointerIndex);
+        this->SetPhysicalEventPosition(ppos[0], ppos[1], ppos[2], pointerIndex);
+        this->SetWorldEventOrientation(wxyz[0],wxyz[1],wxyz[2],wxyz[3],pointerIndex);
+
+        vtkNew<vtkMatrix4x4> poseMatrixWorld;
+        vtkNew<vtkMatrix4x4> poseMatrixPhysical;
+        this->ConvertOpenVRPoseToMatrices(tdPose, poseMatrixWorld, poseMatrixPhysical);
+        this->SetWorldEventPose(poseMatrixWorld,pointerIndex);
+        this->SetPhysicalEventPose(poseMatrixPhysical,pointerIndex);
 
         // so even though we have world coordinates we have to convert them to
         // screen coordinates because all of VTKs picking code is currently
@@ -360,11 +403,7 @@ void vtkOpenVRRenderWindowInteractor::DoOneEvent(vtkOpenVRRenderWindow *renWin,
         ren->SetWorldPoint(pos[0],pos[1],pos[2],1.0);
         ren->WorldToDisplay();
         double *displayCoords = ren->GetDisplayPoint();
-
         this->SetEventPosition(displayCoords[0],displayCoords[1],pointerIndex);
-        this->SetWorldEventPosition(pos[0],pos[1],pos[2],pointerIndex);
-        this->SetPhysicalEventPosition(ppos[0], ppos[1], ppos[2], pointerIndex);
-        this->SetWorldEventOrientation(wxyz[0],wxyz[1],wxyz[2],wxyz[3],pointerIndex);
         this->SetPointerIndex(pointerIndex);
 
         vtkNew<vtkEventDataButton3D> ed;
@@ -444,7 +483,7 @@ void vtkOpenVRRenderWindowInteractor::DoOneEvent(vtkOpenVRRenderWindow *renWin,
         {
           this->InvokeEvent(vtkCommand::Button3DEvent, ed);
           //----------------------------------------------------------------------------
-          //Handle Multitouch
+          // Handle Multitouch
           if (this->RecognizeGestures)
           {
             int iInput = static_cast<int>(ed->GetInput());
@@ -502,11 +541,20 @@ void vtkOpenVRRenderWindowInteractor::DoOneEvent(vtkOpenVRRenderWindow *renWin,
       this->PointerIndexLookup[pointerIndex] = unTrackedDevice;
       this->SetPointerIndex(pointerIndex);
 
-      double pos[3];
-      double ppos[3];
-      double wxyz[4];
-      double wdir[3];
+      double pos[3] = {0.0};
+      double ppos[3] = {0.0};
+      double wxyz[4] = {0.0};
+      double wdir[3] = {0.0};
       this->ConvertPoseToWorldCoordinates(tdPose, pos, wxyz, ppos, wdir);
+      this->SetWorldEventPosition(pos[0],pos[1],pos[2],pointerIndex);
+      this->SetWorldEventOrientation(wxyz[0],wxyz[1],wxyz[2],wxyz[3],pointerIndex);
+      this->SetPhysicalEventPosition(ppos[0], ppos[1], ppos[2], pointerIndex);
+
+      vtkNew<vtkMatrix4x4> poseMatrixWorld;
+      vtkNew<vtkMatrix4x4> poseMatrixPhysical;
+      this->ConvertOpenVRPoseToMatrices(tdPose, poseMatrixWorld, poseMatrixPhysical);
+      this->SetWorldEventPose(poseMatrixWorld,pointerIndex);
+      this->SetPhysicalEventPose(poseMatrixPhysical,pointerIndex);
 
       // so even though we have world coordinates we have to convert them to
       // screen coordinates because all of VTKs picking code is currently
@@ -515,9 +563,7 @@ void vtkOpenVRRenderWindowInteractor::DoOneEvent(vtkOpenVRRenderWindow *renWin,
       ren->WorldToDisplay();
       double *displayCoords = ren->GetDisplayPoint();
       this->SetEventPosition(displayCoords[0], displayCoords[1], pointerIndex);
-      this->SetWorldEventPosition(pos[0],pos[1],pos[2],pointerIndex);
-      this->SetWorldEventOrientation(wxyz[0],wxyz[1],wxyz[2],wxyz[3],pointerIndex);
-      this->SetPhysicalEventPosition(ppos[0], ppos[1], ppos[2], pointerIndex);
+
       vtkNew<vtkEventDataMove3D> ed;
       ed->SetDevice(pointerIndex ? vtkEventDataDevice::LeftController : vtkEventDataDevice::RightController);
       ed->SetWorldPosition(pos);
@@ -541,7 +587,7 @@ void vtkOpenVRRenderWindowInteractor::DoOneEvent(vtkOpenVRRenderWindow *renWin,
 //----------------------------------------------------------------------------
 void vtkOpenVRRenderWindowInteractor::RecognizeComplexGesture(vtkEventDataDevice3D* edata)
 {
-  //Recognize gesture only if one button is pressed per controller
+  // Recognize gesture only if one button is pressed per controller
   if (this->DeviceInputDownCount[this->PointerIndex] > 2 ||
     this->DeviceInputDownCount[this->PointerIndex] == 0)
   {
@@ -554,27 +600,34 @@ void vtkOpenVRRenderWindowInteractor::RecognizeComplexGesture(vtkEventDataDevice
   {
     if (edata->GetAction() == vtkEventDataAction::Press)
     {
-        int iInput = static_cast<int>(vtkEventDataDeviceInput::Grip);
+      int iInput = static_cast<int>(vtkEventDataDeviceInput::Grip);
 
-        this->StartingPhysicalEventPositions[this->PointerIndex][0] =
-          this->PhysicalEventPositions[this->PointerIndex][0];
-        this->StartingPhysicalEventPositions[this->PointerIndex][1] =
-          this->PhysicalEventPositions[this->PointerIndex][1];
-        this->StartingPhysicalEventPositions[this->PointerIndex][2] =
-          this->PhysicalEventPositions[this->PointerIndex][2];
+      this->StartingPhysicalEventPositions[this->PointerIndex][0] =
+        this->PhysicalEventPositions[this->PointerIndex][0];
+      this->StartingPhysicalEventPositions[this->PointerIndex][1] =
+        this->PhysicalEventPositions[this->PointerIndex][1];
+      this->StartingPhysicalEventPositions[this->PointerIndex][2] =
+        this->PhysicalEventPositions[this->PointerIndex][2];
 
-        //Both controllers have the grip down, start multitouch
-        if (this->DeviceInputDown[iInput][0] && this->DeviceInputDown[iInput][1])
-        {
-          // we do not know what the gesture is yet
-          this->CurrentGesture = vtkCommand::StartEvent;
-        }
+      this->StartingPhysicalEventPoses[this->PointerIndex]->DeepCopy(
+        this->PhysicalEventPoses[this->PointerIndex] );
+
+      vtkOpenVRRenderWindow* renWin =
+        vtkOpenVRRenderWindow::SafeDownCast(this->RenderWindow);
+      renWin->GetPhysicalToWorldMatrix(this->StartingPhysicalToWorldMatrix);
+
+      //Both controllers have the grip down, start multitouch
+      if (this->DeviceInputDown[iInput][0] && this->DeviceInputDown[iInput][1])
+      {
+        // we do not know what the gesture is yet
+        this->CurrentGesture = vtkCommand::StartEvent;
+      }
       return;
     }
     // end the gesture if needed
     if (edata->GetAction() == vtkEventDataAction::Release)
     {
-      if (edata->GetInput() == vtkEventDataDeviceInput::Trigger)
+      if (edata->GetInput() == vtkEventDataDeviceInput::Grip)
       {
         if (this->CurrentGesture == vtkCommand::PinchEvent)
         {
@@ -824,6 +877,7 @@ void vtkOpenVRRenderWindowInteractor::ExitCallback()
   this->TerminateApp();
 }
 
+//----------------------------------------------------------------------------
 vtkEventDataDevice vtkOpenVRRenderWindowInteractor::GetPointerDevice()
 {
   if (this->PointerIndex == 0)
@@ -836,3 +890,14 @@ vtkEventDataDevice vtkOpenVRRenderWindowInteractor::GetPointerDevice()
   }
   return vtkEventDataDevice::Unknown;
 }
+
+//----------------------------------------------------------------------------
+void vtkOpenVRRenderWindowInteractor::GetStartingPhysicalToWorldMatrix(
+  vtkMatrix4x4* startingPhysicalToWorldMatrix)
+{
+  if (!startingPhysicalToWorldMatrix)
+  {
+    return;
+  }
+  startingPhysicalToWorldMatrix->DeepCopy(this->StartingPhysicalToWorldMatrix);
+}

Rendering/OpenVR/vtkOpenVRRenderWindowInteractor.h

@@ -28,7 +28,9 @@
 
 #include "vtkOpenVRRenderWindow.h" // ivars
 #include "vtkNew.h" // ivars
-#include "vtkTransform.h" // ivars
+
+class vtkTransform;
+class vtkMatrix4x4;
 
 class VTKRENDERINGOPENVR_EXPORT vtkOpenVRRenderWindowInteractor : public vtkRenderWindowInteractor3D
 {
@@ -84,12 +86,27 @@ public:
   virtual void DoOneEvent(vtkOpenVRRenderWindow *renWin, vtkRenderer *ren);
 
   /*
-   * returns the pointer index as a device
+   * Return the pointer index as a device
    */
   vtkEventDataDevice GetPointerDevice();
 
-  // converts a device pose to a world coordinate
-  // position and orientation
+  /*
+   * Convert a device pose to pose matrices
+   * \param poseMatrixPhysical Optional output pose matrix in physical frame
+   * \param poseMatrixWorld    Optional output pose matrix in world frame
+   */
+  void ConvertOpenVRPoseToMatrices(
+    const vr::TrackedDevicePose_t &tdPose,
+    vtkMatrix4x4* poseMatrixWorld,
+    vtkMatrix4x4* poseMatrixPhysical=nullptr);
+
+  /*