// **********************************************************************************************
// ***                                     !!!WARNING!!!                                      ***
// *** WARNING: AUTO GENERATED FILE! DO NOT MODIFY BY HAND! YOUR CHANGES WILL BE OVERWRITTEN! ***
// ***                                     !!!WARNING!!!                                      ***
// ***                  Generated by Peano's Python API: www.peano-framework.org              ***
// **********************************************************************************************
#include "ElasticWave_FVEnclaveTask.h"


#include "ElasticWaveKernels.h"

#include "ElasticWaveKernels.h"

#include "exahype2/EnclaveBookkeeping.h"
#include "exahype2/EnclaveTask.h"

#include "exahype2/fv/rusanov/rusanov.h"


#include "config.h"

#include "exahype2/CellData.h"
#include "exahype2/EnclaveBookkeeping.h"
#include "exahype2/EnclaveTask.h"

#include "exahype2/enumerators/enumerators.h"

#include "peano4/parallel/parallel.h"

#include "peano4/utils/Loop.h"

#include "tarch/compiler/CompilerSpecificSettings.h"

#include "tarch/multicore/smartScheduling.h"
#include "tarch/multicore/otter.h"

#include "tarch/mpi/DoubleMessage.h"
#include "tarch/mpi/IntegerMessage.h"

#include "tarch/timing/Measurement.h"

#if defined(UseSmartMPI)
#include "communication/Tags.h"
#endif

#include <string.h>
#include <memory>

tarch::logging::Log                applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::_log( "applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask" );
int                                applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::_enclaveTaskTypeId(peano4::parallel::getTaskType("applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask"));

int applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::getEnclaveTaskTypeId() {
  return _enclaveTaskTypeId;
}

double applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::applyKernelToCell(
  const ::peano4::datamanagement::CellMarker& marker,
  double                                      t,
  double                                      dt,
  double* __restrict__                        reconstructedPatch,
  double* __restrict__                        targetPatch
) {
  ::exahype2::CellData patchData( reconstructedPatch, marker.x(), marker.h(), t, dt, targetPatch );

  
  if ( repositories::InstanceOfElasticWave_FV.patchCanUseStatelessPDETerms(
      marker.x(),
      marker.h(),
      t,
      dt
  )) {
    ::exahype2::fv::rusanov::timeStepWithRusanovPatchwiseHeapStateless<
      ElasticWave_FV,
      4,
      1,
      5,
      0,
       true ,
       false ,
       true ,
       true ,
      ::exahype2::enumerators::AoSLexicographicEnumerator
    >(patchData);
  
  }
  else
  

  ::exahype2::fv::rusanov::timeStepWithRusanovPatchwiseHeapFunctors<
      4,
      1,
      5,
      0,
       true ,
       false ,
       true ,
       true ,
      ::exahype2::enumerators::AoSLexicographicEnumerator
    >(patchData,
  [&](
    [[maybe_unused]] const double* __restrict__                   Q, // Q[5+0]
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& faceCentre,
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& volumeH,
    [[maybe_unused]] double                                       t,
    [[maybe_unused]] double                                       dt,
    [[maybe_unused]] int                                          normal,
    [[maybe_unused]] double* __restrict__                         F // F[5]
  )->void {
    
    repositories::InstanceOfElasticWave_FV.flux( Q, faceCentre, volumeH, t, dt, normal, F );
    
  },
  [&](
    [[maybe_unused]] const double* __restrict__                   Q, // Q[5+0]
    [[maybe_unused]] const double* __restrict__                   deltaQ, // deltaQ[5+0]
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& faceCentre,
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& volumeH,
    [[maybe_unused]] double                                       t,
    [[maybe_unused]] double                                       dt,
    [[maybe_unused]] int                                          normal,
    [[maybe_unused]] double* __restrict__                         BTimesDeltaQ // BTimesDeltaQ[5]
  )->void {
    
  },
  [&](
    [[maybe_unused]] const double* __restrict__                   Q, // Q[5+0]
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& volumeX,
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& volumeH,
    [[maybe_unused]] double                                       t,
    [[maybe_unused]] double                                       dt,
    [[maybe_unused]] double* __restrict__                         S // S[5]
  )->void {
    
    repositories::InstanceOfElasticWave_FV.sourceTerm( Q, volumeX, volumeH, t, dt, S );
    
  },
  [&](
    [[maybe_unused]] const double* __restrict__                   Q, // Q[5+0]
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& faceCentre,
    [[maybe_unused]] const tarch::la::Vector<Dimensions, double>& volumeH,
    [[maybe_unused]] double                                       t,
    [[maybe_unused]] double                                       dt,
    [[maybe_unused]] int                                          normal
  )->double {
    return repositories::InstanceOfElasticWave_FV.maxEigenvalue( Q, faceCentre, volumeH, t, dt, normal);
  }
);
  

  ::exahype2::fv::validatePatch(
    targetPatch,
    5,
    0,
    4,
    0, // halo
    std::string(__FILE__) + "(" + std::to_string(__LINE__) + "): " + marker.toString()
  ); // outcome has to be valid

  tarch::freeMemory(reconstructedPatch,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );

  
  return patchData.maxEigenvalue[0];
  
}

applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::ElasticWave_FVEnclaveTask(
  const ::peano4::datamanagement::CellMarker& marker,
  double                                      t,
  double                                      dt,
  double* __restrict__                        reconstructedPatch
):
  ::exahype2::EnclaveTask(
    _enclaveTaskTypeId,
    marker,
    t,
    dt,
    reconstructedPatch,
    #if Dimensions==2
    180,
    80,
    #else
    1080,
    320,
    #endif
    [&]() -> void {
      _maxEigenvalue = applyKernelToCell(
        _marker,
        _t,
        _dt,
        _inputValues,
        _outputValues
      );
    }
  )
  #ifdef UseSmartMPI
  , smartmpi::Task(_enclaveTaskTypeId)
  #endif
{
  setPriority( tarch::multicore::Task::DefaultPriority-1 );
}


#ifdef UseSmartMPI
bool applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::isSmartMPITask() const {
  #ifdef UseSmartMPI
  return true;
  #else
  return false;
  #endif
}


void applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::runLocally() {
  computeTask();
  if (_remoteTaskId != -1) {
    ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(_remoteTaskId,_numberOfResultValues,_outputValues,_maxEigenvalue);
  } else {
    ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(getTaskId(),_numberOfResultValues,_outputValues,_maxEigenvalue);
  }
}

void applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::moveTask(int rank, int tag, MPI_Comm communicator) {
  ::tarch::mpi::DoubleMessage  tMessage(_t);
  ::tarch::mpi::DoubleMessage  dtMessage(_dt);
  ::tarch::mpi::IntegerMessage taskIdMessage;

  if ( tag != smartmpi::communication::MoveTaskToMyServerForEvaluationTag &&
       tag != smartmpi::communication::MoveTaskToComputesComputeRankTag ) {
    taskIdMessage.setValue(_remoteTaskId);
  } else {
    taskIdMessage.setValue(getTaskId());
  }

  ::peano4::datamanagement::CellMarker::send( _marker, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::send( tMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::send( dtMessage, rank, tag, communicator );
  ::tarch::mpi::IntegerMessage::send( taskIdMessage, rank, tag, communicator );

  MPI_Request request;
  MPI_Isend( _inputValues, _numberOfInputValues, MPI_DOUBLE, rank, tag, communicator, &request );

  logInfo(
    "moveTask(...)",
    "sent (" << _marker.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << _numberOfInputValues <<
    "," << taskIdMessage.toString() << ") to rank " << rank <<
    " via tag " << tag
  );
}

smartmpi::Task* applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::receiveTask(int rank, int tag, MPI_Comm communicator) {
  peano4::grid::GridTraversalEvent dummyEvent;
  const int NumberOfInputValues =
    #if Dimensions==2
    180;
    #else
    1080;
    #endif

  ::tarch::mpi::DoubleMessage tMessage;
  ::tarch::mpi::DoubleMessage dtMessage;
  ::tarch::mpi::IntegerMessage taskIdMessage;
  ::peano4::datamanagement::CellMarker markerMessage(dummyEvent);
  double* inputValues = tarch::allocateMemory( NumberOfInputValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );

  ::peano4::datamanagement::CellMarker::receive( markerMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::receive( tMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::receive( dtMessage, rank, tag, communicator );
  ::tarch::mpi::IntegerMessage::receive( taskIdMessage, rank, tag, communicator );

  logInfo(
    "receiveTask(...)",
    "received (" << markerMessage.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << taskIdMessage.toString() << ") from rank " << rank <<
    " via tag " << tag << " and will now receive " << NumberOfInputValues << " doubles"
  );

  MPI_Recv( inputValues, NumberOfInputValues, MPI_DOUBLE, rank, tag, communicator,
    MPI_STATUS_IGNORE
  );

  ElasticWave_FVEnclaveTask* result = new ElasticWave_FVEnclaveTask(
    markerMessage,
    tMessage.getValue(),
    dtMessage.getValue(),
    inputValues
  );
  result->_remoteTaskId = taskIdMessage.getValue();
  return result;
}

void applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::runLocallyAndSendTaskOutputToRank(int rank, int tag, MPI_Comm communicator) {
  _outputValues = tarch::allocateMemory( _numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );

//  _functor(_inputValues,_outputValues,_marker,_t,_dt,_maxEigenvalue);
//  tarch::freeMemory(_inputValues,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
  _functor();

  logInfo(
    "runLocallyAndSendTaskOutputToRank(...)",
    "executed remote task on this rank. Will start to send result back"
  );

  forwardTaskOutputToRank(rank, tag, communicator);
}

void applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::forwardTaskOutputToRank(int rank, int tag, MPI_Comm communicator) {
  logInfo(
    "forwardTaskOutputToRank(...)",
    "will start to forward task output (which has already been computed)"
  );

  ::tarch::mpi::DoubleMessage  tMessage(_t);
  ::tarch::mpi::DoubleMessage  dtMessage(_dt);
  ::tarch::mpi::DoubleMessage  eValueMessage(_maxEigenvalue);
  ::tarch::mpi::IntegerMessage taskIdMessage(_remoteTaskId);

  ::peano4::datamanagement::CellMarker::send( _marker, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::send( tMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::send( dtMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::send( eValueMessage, rank, tag, communicator );
  ::tarch::mpi::IntegerMessage::send( taskIdMessage, rank, tag, communicator );

  MPI_Request request;
  MPI_Isend( _outputValues, _numberOfResultValues, MPI_DOUBLE, rank, tag, communicator, &request );

  logInfo(
    "forwardTaskOutputToRank(...)",
    "sent (" << _marker.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << _numberOfResultValues <<
    "," << taskIdMessage.toString() << ") to rank " << rank <<
    " via tag " << tag
  );

  // tarch::freeMemory(_outputValues,tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
}

smartmpi::Task* applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::receiveOutcome(int rank, int tag, MPI_Comm communicator, const bool intentionToForward) {
  logInfo( "receiveOutcome(...)", "rank=" << rank << ", tag=" << tag );
  peano4::grid::GridTraversalEvent dummyEvent;
  const int NumberOfResultValues =
    #if Dimensions==2
    80;
    #else
    320;
    #endif

  ::tarch::mpi::DoubleMessage tMessage;
  ::tarch::mpi::DoubleMessage dtMessage;
  ::tarch::mpi::DoubleMessage eValueMessage;
  ::tarch::mpi::IntegerMessage taskIdMessage;
  ::peano4::datamanagement::CellMarker markerMessage(dummyEvent);
  double* outputValues = tarch::allocateMemory( NumberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );

  ::peano4::datamanagement::CellMarker::receive( markerMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::receive( tMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::receive( dtMessage, rank, tag, communicator );
  ::tarch::mpi::DoubleMessage::receive( eValueMessage, rank, tag, communicator );
  ::tarch::mpi::IntegerMessage::receive( taskIdMessage, rank, tag, communicator );

  logInfo(
    "receiveOutcome(...)",
    "received (" << markerMessage.toString() << "," << tMessage.toString() << "," << dtMessage.toString() << "," << taskIdMessage.toString() << ") from rank " << rank <<
    " via tag " << tag << " and will now receive " << NumberOfResultValues << " doubles"
  );

  MPI_Recv( outputValues, NumberOfResultValues, MPI_DOUBLE, rank, tag, communicator,
    MPI_STATUS_IGNORE
  );

  /**
   * Having received the output there are two further options:
   * we may need to forward it yet again to another rank - in this case
   * we need a pointer to the task which contains the output;
   * alternatively we bookmark the output and return a nullptr
   */
  if(intentionToForward) {
    double* inputValues = nullptr; // no input as already computed

    ElasticWave_FVEnclaveTask* result = new ElasticWave_FVEnclaveTask(
      markerMessage,
      tMessage.getValue(),
      dtMessage.getValue(),
      inputValues
    );
    result->_remoteTaskId = taskIdMessage.getValue();
    result->_outputValues = outputValues;
    result->_maxEigenvalue = eValueMessage.getValue();
    return result;
  }
  logInfo(
    "receiveOutcome(...)",
    "bookmark outcome of task " << taskIdMessage.getValue()
  );
  ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(taskIdMessage.getValue(),NumberOfResultValues,outputValues,eValueMessage.getValue());
  return nullptr;
}
#endif



bool applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::canFuse() const {
  return repositories::InstanceOfElasticWave_FV.patchCanUseStatelessPDETerms(
    _marker.x(),
    _marker.h(),
    _t,
    _dt
  );
}

/**
 * Also merge the current task
 */
bool applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask::fuse( const std::list<Task*>& otherTasks, int targetDevice ) {
  logDebug( "fuse(...)", "asked to fuse " << (otherTasks.size()+1) << " tasks into one large GPU task on device " << targetDevice );

  ::exahype2::CellData patchData(otherTasks.size()+1);
  int currentTask = 0;
  for (auto& p: otherTasks) {
    patchData.QIn[currentTask]         = static_cast<applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask*>(p)->_inputValues;
    patchData.cellCentre[currentTask]  = static_cast<applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask*>(p)->_marker.x();
    patchData.cellSize[currentTask]    = static_cast<applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask*>(p)->_marker.h();
    patchData.t[currentTask]           = static_cast<applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask*>(p)->_t;
    patchData.dt[currentTask]          = static_cast<applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask*>(p)->_dt;
    patchData.id[currentTask]          = static_cast<applications::exahype2::elasticwave::tasks::ElasticWave_FVEnclaveTask*>(p)->_taskNumber;
    patchData.QOut[currentTask]        = tarch::allocateMemory<double>( _numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
    delete p;
    currentTask++;
  }
  patchData.QIn[currentTask]         = _inputValues;
  patchData.cellCentre[currentTask]  = _marker.x();
  patchData.cellSize[currentTask]    = _marker.h();
  patchData.t[currentTask]           = _t;
  patchData.dt[currentTask]          = _dt;
  patchData.id[currentTask]          = _taskNumber;
  patchData.QOut[currentTask]        = tarch::allocateMemory<double>( _numberOfResultValues, tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );

  //
  // ==============
  // Invoke kernels
  // ==============
  //
  bool foundOffloadingBranch = false;

  #if defined(GPUOffloadingOMP)
  if (targetDevice>=0) {
    foundOffloadingBranch = true;
    ::exahype2::fv::rusanov::omp::timeStepWithRusanovPatchwiseUSMStateless<
      ElasticWave_FV,
      4,
      1,
      5,
      0,
       true ,
       false ,
       true ,
       true ,
      ::exahype2::enumerators::AoSLexicographicEnumerator
      >(targetDevice, patchData);
  
  }
  #endif

  #if defined(GPUOffloadingHIP)
  if (targetDevice>=0) {
    foundOffloadingBranch = true;
    ::exahype2::fv::rusanov::hip::timeStepWithRusanovPatchwiseUSMStateless<
      ElasticWave_FV,
      4,
      1,
      5,
      0,
       true ,
       false ,
       true ,
       true ,
      ::exahype2::enumerators::AoSLexicographicEnumerator
      >(targetDevice, patchData);
  
  }
  #endif

  #if defined(GPUOffloadingSYCL)
  if (targetDevice>=0 or targetDevice==Host) {
    foundOffloadingBranch = true;
    ::exahype2::fv::rusanov::sycl::timeStepWithRusanovPatchwiseUSMStateless<
      ElasticWave_FV,
      4,
      1,
      5,
      0,
       true ,
       false ,
       true ,
       true ,
      ::exahype2::enumerators::AoSLexicographicEnumerator
      >(targetDevice, patchData);
  
  }
  #endif

  #if defined(GPUOffloadingCPP)
  if (targetDevice>=0 or targetDevice==Host) {
    foundOffloadingBranch = true;
    ::exahype2::fv::rusanov::cpp::timeStepWithRusanovPatchwiseUSMStateless<
      ElasticWave_FV,
      4,
      1,
      5,
      0,
       true ,
       false ,
       true ,
       true ,
      ::exahype2::enumerators::AoSLexicographicEnumerator
      >(targetDevice, patchData);
  
  }
  #endif

  if (not foundOffloadingBranch) {
    ::exahype2::fv::rusanov::timeStepWithRusanovPatchwiseHeapStateless<
      ElasticWave_FV,
      4,
      1,
      5,
      0,
       true ,
       false ,
       true ,
       true ,
      ::exahype2::enumerators::AoSLexicographicEnumerator
    >(patchData);
  
  }

  //
  // ==================
  // Bring back results
  // ==================
  //
  for (int i=0; i<patchData.numberOfCells; i++) {
    tarch::freeMemory( patchData.QIn[i], tarch::MemoryLocation::ManagedSharedAcceleratorDeviceMemory );
    ::exahype2::EnclaveBookkeeping::getInstance().finishedTask(patchData.id[i], _numberOfResultValues, patchData.QOut[i], patchData.maxEigenvalue[i]);
  }

  // Don't rerun, so return false.
  return false;
}