////
//// BEGIN-EXAMPLE DeviceAdapterTagCxx11Thread.h
////
#include <vtkm/cont/internal/DeviceAdapterTag.h>

// If this device adapter were to be contributed to VTK-m, then this macro
// declaration should be moved to DeviceAdapterTag.h and given a unique
// number.
#define VTKM_DEVICE_ADAPTER_CXX11_THREAD 101

VTKM_VALID_DEVICE_ADAPTER(Cxx11Thread, VTKM_DEVICE_ADAPTER_CXX11_THREAD);
////
//// END-EXAMPLE DeviceAdapterTagCxx11Thread.h
////

#include <vtkm/cont/internal/ArrayManagerExecution.h>

////
//// BEGIN-EXAMPLE ArrayManagerExecutionPrototype.cxx
////
namespace vtkm {
namespace cont {
namespace internal {

template<typename T, typename StorageTag, typename DeviceAdapterTag>
class ArrayManagerExecution;

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE ArrayManagerExecutionPrototype.cxx
////

////
//// BEGIN-EXAMPLE ArrayManagerExecutionCxx11Thread.h
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/ArrayManagerExecution.h>
#include <vtkm/cont/internal/ArrayManagerExecutionShareWithControl.h>

namespace vtkm {
namespace cont {
namespace internal {

template<typename T, typename StorageTag>
class ArrayManagerExecution<
      T, StorageTag, vtkm::cont::DeviceAdapterTagCxx11Thread>
    : public vtkm::cont::internal::ArrayManagerExecutionShareWithControl<
        T, StorageTag>
{
  using Superclass =
    vtkm::cont::internal::ArrayManagerExecutionShareWithControl<T, StorageTag>;

public:
  VTKM_CONT
  ArrayManagerExecution(typename Superclass::StorageType *storage)
    : Superclass(storage) {  }
};

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE ArrayManagerExecutionCxx11Thread.h
////

////
//// BEGIN-EXAMPLE ExecutionPortalFactoryBasicPrototype.cxx
////
namespace vtkm {
namespace cont {
namespace internal {

template<typename T, typename DeviceAdapterTag>
struct ExecutionPortalFactoryBasic;

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE ExecutionPortalFactoryBasicPrototype.cxx
////

////
//// BEGIN-EXAMPLE ExecutionPortalFactoryBasicCxx11Thread.cxx
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/ArrayManagerExecutionShareWithControl.h>

namespace vtkm {
namespace cont {
namespace internal {

template <typename T>
class ExecutionPortalFactoryBasic<T, vtkm::cont::DeviceAdapterTagCxx11Thread>
    : public vtkm::cont::internal::ExecutionPortalFactoryBasicShareWithControl<T>
{
};

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE ExecutionPortalFactoryBasicCxx11Thread.cxx
////

#include <vtkm/cont/internal/VirtualObjectTransfer.h>

////
//// BEGIN-EXAMPLE ExecutionArrayInterfaceBasicPrototype.cxx
////
namespace vtkm {
namespace cont {
namespace internal {

template<typename DeviceAdapterTag>
struct ExecutionArrayInterfaceBasic;

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE ExecutionArrayInterfaceBasicPrototype.cxx
////

////
//// BEGIN-EXAMPLE ExecutionArrayInterfaceBasicCxx11Thread.cxx
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/ArrayManagerExecutionShareWithControl.h>

namespace vtkm {
namespace cont {
namespace internal {

template <>
class ExecutionArrayInterfaceBasic<vtkm::cont::DeviceAdapterTagCxx11Thread>
    : public vtkm::cont::internal::ExecutionArrayInterfaceBasicShareWithControl
{
public:
  using Superclass =
    vtkm::cont::internal::ExecutionArrayInterfaceBasicShareWithControl;

  VTKM_CONT
  ExecutionArrayInterfaceBasic(vtkm::cont::internal::StorageBasicBase& storage)
    : Superclass(storage)
  {
  }

  VTKM_CONT
  virtual vtkm::cont::DeviceAdapterId GetDeviceId() const final override
  {
    return VTKM_DEVICE_ADAPTER_CXX11_THREAD;
  }
};

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE ExecutionArrayInterfaceBasicCxx11Thread.cxx
////

////
//// BEGIN-EXAMPLE VirtualObjectTransferPrototype.cxx
////
namespace vtkm {
namespace cont {
namespace internal {

template<typename VirtualDerivedType, typename DeviceAdapter>
struct VirtualObjectTransfer;

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE VirtualObjectTransferPrototype.cxx
////

////
//// BEGIN-EXAMPLE VirtualObjectTransferCxx11Thread.h
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/VirtualObjectTransfer.h>
#include <vtkm/cont/internal/VirtualObjectTransferShareWithControl.h>

namespace vtkm {
namespace cont {
namespace internal {

template<typename VirtualDerivedType>
struct VirtualObjectTransfer<
        VirtualDerivedType, vtkm::cont::DeviceAdapterTagCxx11Thread>
  : VirtualObjectTransferShareWithControl<VirtualDerivedType>
{
  VTKM_CONT VirtualObjectTransfer(const VirtualDerivedType* virtualObject)
    : VirtualObjectTransferShareWithControl<VirtualDerivedType>(virtualObject)
  {
  }
};

}
}
} // namespace vtkm::cont::internal
////
//// END-EXAMPLE VirtualObjectTransferCxx11Thread.h
////

////
//// BEGIN-EXAMPLE DeviceAdapterAlgorithmCxx11Thread.h
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/DeviceAdapterAlgorithm.h>
#include <vtkm/cont/ErrorExecution.h>
#include <vtkm/cont/internal/DeviceAdapterAlgorithmGeneral.h>

#include <thread>

namespace vtkm {
namespace cont {

template<>
struct DeviceAdapterAlgorithm<vtkm::cont::DeviceAdapterTagCxx11Thread>
    : vtkm::cont::internal::DeviceAdapterAlgorithmGeneral<
          DeviceAdapterAlgorithm<vtkm::cont::DeviceAdapterTagCxx11Thread>,
          vtkm::cont::DeviceAdapterTagCxx11Thread>
{
private:
  template<typename FunctorType>
  struct ScheduleKernel1D
  {
    VTKM_CONT
    ScheduleKernel1D(const FunctorType &functor)
      : Functor(functor)
    {  }

    VTKM_EXEC
    void operator()() const
    {
      try
      {
        for (vtkm::Id threadId = this->BeginId;
             threadId < this->EndId;
             threadId++)
        {
          this->Functor(threadId);
          // If an error is raised, abort execution.
          if (this->ErrorMessage.IsErrorRaised()) { return; }
        }
      }
      catch (vtkm::cont::Error error)
      {
        this->ErrorMessage.RaiseError(error.GetMessage().c_str());
      }
      catch (std::exception error)
      {
        this->ErrorMessage.RaiseError(error.what());
      }
      catch (...)
      {
        this->ErrorMessage.RaiseError("Unknown exception raised.");
      }
    }

    FunctorType Functor;
    vtkm::exec::internal::ErrorMessageBuffer ErrorMessage;
    vtkm::Id BeginId;
    vtkm::Id EndId;
  };

  template<typename FunctorType>
  struct ScheduleKernel3D
  {
    VTKM_CONT
    ScheduleKernel3D(const FunctorType &functor, vtkm::Id3 maxRange)
      : Functor(functor), MaxRange(maxRange)
    {  }

    VTKM_EXEC
    void operator()() const
    {
      vtkm::Id3 threadId3D(this->BeginId%this->MaxRange[0],
                           (this->BeginId/this->MaxRange[0])%this->MaxRange[1],
                           this->BeginId/(this->MaxRange[0]*this->MaxRange[1]));

      try
      {
        for (vtkm::Id threadId = this->BeginId;
             threadId < this->EndId;
             threadId++)
        {
          this->Functor(threadId3D);
          // If an error is raised, abort execution.
          if (this->ErrorMessage.IsErrorRaised()) { return; }

          threadId3D[0]++;
          if (threadId3D[0] >= MaxRange[0])
          {
            threadId3D[0] = 0;
            threadId3D[1]++;
            if (threadId3D[1] >= MaxRange[1])
            {
              threadId3D[1] = 0;
              threadId3D[2]++;
            }
          }
        }
      }
      catch (vtkm::cont::Error error)
      {
        this->ErrorMessage.RaiseError(error.GetMessage().c_str());
      }
      catch (std::exception error)
      {
        this->ErrorMessage.RaiseError(error.what());
      }
      catch (...)
      {
        this->ErrorMessage.RaiseError("Unknown exception raised.");
      }
    }

    FunctorType Functor;
    vtkm::exec::internal::ErrorMessageBuffer ErrorMessage;
    vtkm::Id BeginId;
    vtkm::Id EndId;
    vtkm::Id3 MaxRange;
  };

  template<typename KernelType>
  VTKM_CONT
  static void DoSchedule(KernelType kernel,
                         vtkm::Id numInstances)
  {
    if (numInstances < 1) { return; }

    const vtkm::Id MESSAGE_SIZE = 1024;
    char errorString[MESSAGE_SIZE];
    errorString[0] = '\0';
    vtkm::exec::internal::ErrorMessageBuffer errorMessage(errorString,
                                                          MESSAGE_SIZE);
    kernel.Functor.SetErrorMessageBuffer(errorMessage);
    kernel.ErrorMessage = errorMessage;

    vtkm::Id numThreads =
        static_cast<vtkm::Id>(std::thread::hardware_concurrency());
    if (numThreads > numInstances)
    {
      numThreads = numInstances;
    }
    vtkm::Id numInstancesPerThread = (numInstances+numThreads-1)/numThreads;

    std::thread *threadPool = new std::thread[numThreads];
    vtkm::Id beginId = 0;
    for (vtkm::Id threadIndex = 0; threadIndex < numThreads; threadIndex++)
    {
      vtkm::Id endId = std::min(beginId+numInstancesPerThread, numInstances);
      KernelType threadKernel = kernel;
      threadKernel.BeginId = beginId;
      threadKernel.EndId = endId;
      std::thread newThread(threadKernel);
      threadPool[threadIndex].swap(newThread);
      beginId = endId;
    }

    for (vtkm::Id threadIndex = 0; threadIndex < numThreads; threadIndex++)
    {
      threadPool[threadIndex].join();
    }

    delete[] threadPool;

    if (errorMessage.IsErrorRaised())
    {
      throw vtkm::cont::ErrorExecution(errorString);
    }
  }

public:
  template<typename FunctorType>
  VTKM_CONT
  static void Schedule(FunctorType functor, vtkm::Id numInstances)
  {
    DoSchedule(ScheduleKernel1D<FunctorType>(functor), numInstances);
  }

  template<typename FunctorType>
  VTKM_CONT
  static void Schedule(FunctorType functor, vtkm::Id3 maxRange)
  {
    vtkm::Id numInstances = maxRange[0]*maxRange[1]*maxRange[2];
    DoSchedule(ScheduleKernel3D<FunctorType>(functor, maxRange), numInstances);
  }

  VTKM_CONT
  static void Synchronize()
  {
    // Nothing to do. This device schedules all of its operations using a
    // split/join paradigm. This means that the if the control threaad is
    // calling this method, then nothing should be running in the execution
    // environment.
  }
};

}
} // namespace vtkm::cont
////
//// END-EXAMPLE DeviceAdapterAlgorithmCxx11Thread.h
////

////
//// BEGIN-EXAMPLE DeviceAdapterTimerImplementationCxx11Thread.h
////
#include <chrono>

namespace vtkm {
namespace cont {

template<>
class DeviceAdapterTimerImplementation<vtkm::cont::DeviceAdapterTagCxx11Thread>
{
public:
  VTKM_CONT
  DeviceAdapterTimerImplementation()
  {
    this->Reset();
  }

  VTKM_CONT
  void Reset()
  {
    vtkm::cont::DeviceAdapterAlgorithm<vtkm::cont::DeviceAdapterTagCxx11Thread>
        ::Synchronize();
    this->StartTime = std::chrono::high_resolution_clock::now();
  }

  VTKM_CONT
  vtkm::Float64 GetElapsedTime()
  {
    vtkm::cont::DeviceAdapterAlgorithm<vtkm::cont::DeviceAdapterTagCxx11Thread>
        ::Synchronize();
    std::chrono::high_resolution_clock::time_point endTime =
        std::chrono::high_resolution_clock::now();

    std::chrono::high_resolution_clock::duration elapsedTicks =
        endTime - this->StartTime;

    std::chrono::duration<vtkm::Float64> elapsedSeconds(elapsedTicks);

    return elapsedSeconds.count();
  }

private:
  std::chrono::high_resolution_clock::time_point StartTime;
};

}
} // namespace vtkm::cont
////
//// END-EXAMPLE DeviceAdapterTimerImplementationCxx11Thread.h
////

////
//// BEGIN-EXAMPLE UnitTestDeviceAdapterCxx11Thread.cxx
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/DeviceAdapterCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/testing/TestingDeviceAdapter.h>

int UnitTestDeviceAdapterCxx11Thread(int, char *[])
{
  return vtkm::cont::testing::TestingDeviceAdapter<
      vtkm::cont::DeviceAdapterTagCxx11Thread>::Run();
}
////
//// END-EXAMPLE UnitTestDeviceAdapterCxx11Thread.cxx
////

int CustomDeviceAdapter(int argc, char *argv[])
{
  return UnitTestDeviceAdapterCxx11Thread(argc, argv);
}
