////
//// BEGIN-EXAMPLE DeviceAdapterTagCxx11Thread.h
////
#include <vtkm/cont/internal/DeviceAdapterTag.h>

// If this device adapter were to be contributed to VTK-m, then this macro
// declaration should be moved to DeviceAdapterTag.h and given a unique
// number. It also has te be less than VTK_MAX_DEVICE_ADAPTER_ID
#define VTKM_DEVICE_ADAPTER_CXX11_THREAD 7

VTKM_VALID_DEVICE_ADAPTER(Cxx11Thread, VTKM_DEVICE_ADAPTER_CXX11_THREAD);
////
//// END-EXAMPLE DeviceAdapterTagCxx11Thread.h
////

#include <vtkm/cont/DeviceAdapterAlgorithm.h>

////
//// BEGIN-EXAMPLE DeviceAdapterRuntimeDetectorPrototype.cxx
////
namespace vtkm
{
namespace cont
{

template<typename DeviceAdapterTag>
class DeviceAdapterRuntimeDetector;
}
} // namespace vtkm
////
//// END-EXAMPLE DeviceAdapterRuntimeDetectorPrototype.cxx
////

////
//// BEGIN-EXAMPLE DeviceAdapterRuntimeDetectorCxx11Thread.cxx
////
namespace vtkm
{
namespace cont
{

template<>
class DeviceAdapterRuntimeDetector<vtkm::cont::DeviceAdapterTagCxx11Thread>
{
public:
  VTKM_CONT bool Exists() const
  {
    return vtkm::cont::DeviceAdapterTagCxx11Thread::IsEnabled;
  }
};

} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE DeviceAdapterRuntimeDetectorCxx11Thread.cxx
////

#include <vtkm/cont/internal/ArrayManagerExecution.h>

////
//// BEGIN-EXAMPLE ArrayManagerExecutionPrototype.cxx
////
namespace vtkm
{
namespace cont
{
namespace internal
{

template<typename T, typename StorageTag, typename DeviceAdapterTag>
class ArrayManagerExecution;
}
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE ArrayManagerExecutionPrototype.cxx
////

////
//// BEGIN-EXAMPLE ArrayManagerExecutionCxx11Thread.h
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/ArrayManagerExecution.h>
#include <vtkm/cont/internal/ArrayManagerExecutionShareWithControl.h>

namespace vtkm
{
namespace cont
{
namespace internal
{

template<typename T, typename StorageTag>
class ArrayManagerExecution<T, StorageTag, vtkm::cont::DeviceAdapterTagCxx11Thread>
  : public vtkm::cont::internal::ArrayManagerExecutionShareWithControl<T, StorageTag>
{
  using Superclass =
    vtkm::cont::internal::ArrayManagerExecutionShareWithControl<T, StorageTag>;

public:
  VTKM_CONT
  ArrayManagerExecution(typename Superclass::StorageType* storage)
    : Superclass(storage)
  {
  }
};

} // namespace internal
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE ArrayManagerExecutionCxx11Thread.h
////

////
//// BEGIN-EXAMPLE ExecutionPortalFactoryBasicPrototype.cxx
////
namespace vtkm
{
namespace cont
{
namespace internal
{

template<typename T, typename DeviceAdapterTag>
struct ExecutionPortalFactoryBasic;
}
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE ExecutionPortalFactoryBasicPrototype.cxx
////

////
//// BEGIN-EXAMPLE ExecutionPortalFactoryBasicCxx11Thread.cxx
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/ArrayManagerExecutionShareWithControl.h>

namespace vtkm
{
namespace cont
{
namespace internal
{

template<typename T>
struct ExecutionPortalFactoryBasic<T, vtkm::cont::DeviceAdapterTagCxx11Thread>
  : public vtkm::cont::internal::ExecutionPortalFactoryBasicShareWithControl<T>
{
};

} // namespace internal
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE ExecutionPortalFactoryBasicCxx11Thread.cxx
////

#include <vtkm/cont/internal/VirtualObjectTransfer.h>

////
//// BEGIN-EXAMPLE ExecutionArrayInterfaceBasicPrototype.cxx
////
namespace vtkm
{
namespace cont
{
namespace internal
{

template<typename DeviceAdapterTag>
struct ExecutionArrayInterfaceBasic;
}
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE ExecutionArrayInterfaceBasicPrototype.cxx
////

////
//// BEGIN-EXAMPLE ExecutionArrayInterfaceBasicCxx11Thread.cxx
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/ArrayManagerExecutionShareWithControl.h>

namespace vtkm
{
namespace cont
{
namespace internal
{

template<>
struct ExecutionArrayInterfaceBasic<vtkm::cont::DeviceAdapterTagCxx11Thread>
  : public vtkm::cont::internal::ExecutionArrayInterfaceBasicShareWithControl
{
public:
  using Superclass =
    vtkm::cont::internal::ExecutionArrayInterfaceBasicShareWithControl;

  VTKM_CONT
  ExecutionArrayInterfaceBasic(vtkm::cont::internal::StorageBasicBase& storage)
    : Superclass(storage)
  {
  }

  VTKM_CONT
  virtual vtkm::cont::DeviceAdapterId GetDeviceId() const final override
  {
    return vtkm::cont::DeviceAdapterTagCxx11Thread();
  }
};

} // namespace internal
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE ExecutionArrayInterfaceBasicCxx11Thread.cxx
////

////
//// BEGIN-EXAMPLE VirtualObjectTransferPrototype.cxx
////
namespace vtkm
{
namespace cont
{
namespace internal
{

template<typename VirtualDerivedType, typename DeviceAdapter>
struct VirtualObjectTransfer;
}
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE VirtualObjectTransferPrototype.cxx
////

////
//// BEGIN-EXAMPLE VirtualObjectTransferCxx11Thread.h
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/internal/VirtualObjectTransfer.h>
#include <vtkm/cont/internal/VirtualObjectTransferShareWithControl.h>

namespace vtkm
{
namespace cont
{
namespace internal
{

template<typename VirtualDerivedType>
struct VirtualObjectTransfer<VirtualDerivedType,
                             vtkm::cont::DeviceAdapterTagCxx11Thread>
  : VirtualObjectTransferShareWithControl<VirtualDerivedType>
{
  VTKM_CONT VirtualObjectTransfer(const VirtualDerivedType* virtualObject)
    : VirtualObjectTransferShareWithControl<VirtualDerivedType>(virtualObject)
  {
  }
};

} // namespace internal
} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE VirtualObjectTransferCxx11Thread.h
////

////
//// BEGIN-EXAMPLE DeviceAdapterAlgorithmCxx11Thread.h
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/internal/DeviceAdapterTagCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/cont/DeviceAdapterAlgorithm.h>
#include <vtkm/cont/ErrorExecution.h>
#include <vtkm/cont/internal/DeviceAdapterAlgorithmGeneral.h>

#include <thread>

namespace vtkm
{
namespace cont
{

template<>
struct DeviceAdapterAlgorithm<vtkm::cont::DeviceAdapterTagCxx11Thread>
  : vtkm::cont::internal::DeviceAdapterAlgorithmGeneral<
      DeviceAdapterAlgorithm<vtkm::cont::DeviceAdapterTagCxx11Thread>,
      vtkm::cont::DeviceAdapterTagCxx11Thread>
{
private:
  template<typename FunctorType>
  struct ScheduleKernel1D
  {
    VTKM_CONT
    ScheduleKernel1D(const FunctorType& functor)
      : Functor(functor)
    {
    }

    VTKM_EXEC
    void operator()() const
    {
      try
      {
        for (vtkm::Id threadId = this->BeginId; threadId < this->EndId; threadId++)
        {
          this->Functor(threadId);
          // If an error is raised, abort execution.
          if (this->ErrorMessage.IsErrorRaised())
          {
            return;
          }
        }
      }
      catch (vtkm::cont::Error error)
      {
        this->ErrorMessage.RaiseError(error.GetMessage().c_str());
      }
      catch (std::exception error)
      {
        this->ErrorMessage.RaiseError(error.what());
      }
      catch (...)
      {
        this->ErrorMessage.RaiseError("Unknown exception raised.");
      }
    }

    FunctorType Functor;
    vtkm::exec::internal::ErrorMessageBuffer ErrorMessage;
    vtkm::Id BeginId;
    vtkm::Id EndId;
  };

  template<typename FunctorType>
  struct ScheduleKernel3D
  {
    VTKM_CONT
    ScheduleKernel3D(const FunctorType& functor, vtkm::Id3 maxRange)
      : Functor(functor)
      , MaxRange(maxRange)
    {
    }

    VTKM_EXEC
    void operator()() const
    {
      vtkm::Id3 threadId3D(this->BeginId % this->MaxRange[0],
                           (this->BeginId / this->MaxRange[0]) % this->MaxRange[1],
                           this->BeginId / (this->MaxRange[0] * this->MaxRange[1]));

      try
      {
        for (vtkm::Id threadId = this->BeginId; threadId < this->EndId; threadId++)
        {
          this->Functor(threadId3D);
          // If an error is raised, abort execution.
          if (this->ErrorMessage.IsErrorRaised())
          {
            return;
          }

          threadId3D[0]++;
          if (threadId3D[0] >= MaxRange[0])
          {
            threadId3D[0] = 0;
            threadId3D[1]++;
            if (threadId3D[1] >= MaxRange[1])
            {
              threadId3D[1] = 0;
              threadId3D[2]++;
            }
          }
        }
      }
      catch (vtkm::cont::Error error)
      {
        this->ErrorMessage.RaiseError(error.GetMessage().c_str());
      }
      catch (std::exception error)
      {
        this->ErrorMessage.RaiseError(error.what());
      }
      catch (...)
      {
        this->ErrorMessage.RaiseError("Unknown exception raised.");
      }
    }

    FunctorType Functor;
    vtkm::exec::internal::ErrorMessageBuffer ErrorMessage;
    vtkm::Id BeginId;
    vtkm::Id EndId;
    vtkm::Id3 MaxRange;
  };

  template<typename KernelType>
  VTKM_CONT static void DoSchedule(KernelType kernel, vtkm::Id numInstances)
  {
    if (numInstances < 1)
    {
      return;
    }

    const vtkm::Id MESSAGE_SIZE = 1024;
    char errorString[MESSAGE_SIZE];
    errorString[0] = '\0';
    vtkm::exec::internal::ErrorMessageBuffer errorMessage(errorString, MESSAGE_SIZE);
    kernel.Functor.SetErrorMessageBuffer(errorMessage);
    kernel.ErrorMessage = errorMessage;

    vtkm::Id numThreads = static_cast<vtkm::Id>(std::thread::hardware_concurrency());
    if (numThreads > numInstances)
    {
      numThreads = numInstances;
    }
    vtkm::Id numInstancesPerThread = (numInstances + numThreads - 1) / numThreads;

    std::thread* threadPool = new std::thread[numThreads];
    vtkm::Id beginId = 0;
    for (vtkm::Id threadIndex = 0; threadIndex < numThreads; threadIndex++)
    {
      vtkm::Id endId = std::min(beginId + numInstancesPerThread, numInstances);
      KernelType threadKernel = kernel;
      threadKernel.BeginId = beginId;
      threadKernel.EndId = endId;
      std::thread newThread(threadKernel);
      threadPool[threadIndex].swap(newThread);
      beginId = endId;
    }

    for (vtkm::Id threadIndex = 0; threadIndex < numThreads; threadIndex++)
    {
      threadPool[threadIndex].join();
    }

    delete[] threadPool;

    if (errorMessage.IsErrorRaised())
    {
      throw vtkm::cont::ErrorExecution(errorString);
    }
  }

public:
  template<typename FunctorType>
  VTKM_CONT static void Schedule(FunctorType functor, vtkm::Id numInstances)
  {
    DoSchedule(ScheduleKernel1D<FunctorType>(functor), numInstances);
  }

  template<typename FunctorType>
  VTKM_CONT static void Schedule(FunctorType functor, vtkm::Id3 maxRange)
  {
    vtkm::Id numInstances = maxRange[0] * maxRange[1] * maxRange[2];
    DoSchedule(ScheduleKernel3D<FunctorType>(functor, maxRange), numInstances);
  }

  VTKM_CONT
  static void Synchronize()
  {
    // Nothing to do. This device schedules all of its operations using a
    // split/join paradigm. This means that the if the control threaad is
    // calling this method, then nothing should be running in the execution
    // environment.
  }
};

} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE DeviceAdapterAlgorithmCxx11Thread.h
////

////
//// BEGIN-EXAMPLE DeviceAdapterTimerImplementationCxx11Thread.h
////
#include <chrono>

namespace vtkm
{
namespace cont
{

template<>
class DeviceAdapterTimerImplementation<vtkm::cont::DeviceAdapterTagCxx11Thread>
{
public:
  VTKM_CONT
  DeviceAdapterTimerImplementation() { this->Reset(); }

  VTKM_CONT
  void Reset()
  {
    vtkm::cont::DeviceAdapterAlgorithm<
      vtkm::cont::DeviceAdapterTagCxx11Thread>::Synchronize();
    this->StartTime = std::chrono::high_resolution_clock::now();
  }

  VTKM_CONT
  vtkm::Float64 GetElapsedTime()
  {
    vtkm::cont::DeviceAdapterAlgorithm<
      vtkm::cont::DeviceAdapterTagCxx11Thread>::Synchronize();
    std::chrono::high_resolution_clock::time_point endTime =
      std::chrono::high_resolution_clock::now();

    std::chrono::high_resolution_clock::duration elapsedTicks =
      endTime - this->StartTime;

    std::chrono::duration<vtkm::Float64> elapsedSeconds(elapsedTicks);

    return elapsedSeconds.count();
  }

private:
  std::chrono::high_resolution_clock::time_point StartTime;
};

} // namespace cont
} // namespace vtkm
////
//// END-EXAMPLE DeviceAdapterTimerImplementationCxx11Thread.h
////

////
//// BEGIN-EXAMPLE UnitTestDeviceAdapterCxx11Thread.cxx
////
//// PAUSE-EXAMPLE
// We did not really put the device adapter components in separate header
// files, but for the purposes of an example we are pretending we are.
#if 0
//// RESUME-EXAMPLE
#include <vtkm/cont/cxx11/DeviceAdapterCxx11Thread.h>
//// PAUSE-EXAMPLE
#endif
//// RESUME-EXAMPLE

#include <vtkm/Types.h>
#include <vtkm/cont/testing/TestingDeviceAdapter.h>
#include <vtkm/cont/vtkm_cont_export.h>

int UnitTestDeviceAdapterCxx11Thread(int, char* [])
{
  return vtkm::cont::testing::TestingDeviceAdapter<
    vtkm::cont::DeviceAdapterTagCxx11Thread>::Run();
}
////
//// END-EXAMPLE UnitTestDeviceAdapterCxx11Thread.cxx
////

int CustomDeviceAdapter(int argc, char* argv[])
{
  return UnitTestDeviceAdapterCxx11Thread(argc, argv);
}
