Process.cpp

/****************************************************************************
* Copyright (c) 2026, CEA
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*****************************************************************************/
 
#include <Process.h>
#include <Comm_Group.h>
#include <PE_Groups.h>
#include <communications.h>
#include <Sortie_Nulle.h>
#include <Journal.h>
#include <SFichier.h>
#include <Synonyme_info.h>
#include <petsc_for_kernel.h>
#include <comm_incl.h>
#include <TRUST_Error.h>
#include <Comm_Group_MPI.h>
#include <unistd.h> // sleep() on some machines
#include <SChaine.h>
#include <FichierHDFPar.h>
#include <EChaineJDD.h>
#include <DeviceMemory.h>
#include <kokkos++.h>
#include <fstream>
 
// Each of Cerr, Cout and Journal(i) can be redirected to one of the following four streams:
// Null output instance (equivalent of /dev/null)
static Sortie_Nulle  journal_zero_;
// Output instance pointing to cerr
static Sortie        std_err_(cerr);
// Output instance pointing to cout
static Sortie        std_out_(cout);
// Journal file instances
static SFichier     journal_file_;
 
static int          journal_file_open_;
static Nom          journal_file_name_;
// Maximum level of messages written. The initial value determines
// whether messages written before journal initialization are written or not.
static int        verbose_level_ = 0;
static int        disable_stop_ = 0;
 
// Flag indicating whether cerr and cout should be redirected to the journal file
static int        cerr_to_journal_ = 0;
int Process::exception_sur_exit=0;
int Process::multiple_files=5120; // Can be modified via the TRUST_MultipleFiles environment variable
bool Process::force_single_file(const int ranks, const Nom& filename)
{
  char* theValue = getenv("TRUST_MultipleFiles");
  if (theValue != nullptr) multiple_files=atoi(theValue);
  if (ranks>multiple_files)
    {
      if (Process::je_suis_maitre())   // Necessary: called very early in main.cpp before Cerr is fully defined on all processes
        {
          Cerr << "======================================================================================================" << finl;
          Cerr << "Warning! Single file option is forced for " << filename << " above " << multiple_files << " MPI ranks." << finl;
          Cerr << "for I/O performance reasons on cluster and inodes number limitation." << finl;
          Cerr << "If you want to keep multiple files, add at the beginning of your data file to outpass the limitation:" << finl;
          Cerr << "MultipleFiles " << ranks << finl;
          Cerr << "=====================================================================================================" << finl;
        }
      return true;
    }
  else
    return false;
}
 
/*! @brief Returns 1 if on the master processor of the current group (i.e. me() == 0), 0 otherwise.
 *
 * @return 1 if master processor, 0 otherwise.
 */
int Process::je_suis_maitre()
{
  const int r = PE_Groups::current_group().rank();
  return r == 0;
}
 
/*! @brief Returns 1 if on the NUMA node master processor, 0 otherwise.
 *
 * @return 1 if node master processor, 0 otherwise.
 */
int Process::node_master()
{
  const int r = PE_Groups::get_node_group().rank();
  return r == 0;
}
 
/*! @brief Returns the number of processors in the current group. See Comm_Group::nproc() and PE_Groups::current_group().
 *
 * @return Number of processors in the current group.
 */
int Process::nproc()
{
  const int n = PE_Groups::current_group().nproc();
  return n;
}
 
bool Process::is_parallel()
{
  return Process::nproc() > 1;
}
 
bool Process::is_sequential()
{
  return Process::nproc() == 1;
}
 
/*! @brief Returns the rank of the local processor in the current communication group. See Comm_Group::rank() and PE_Groups::current_group().
 *
 * @return Rank in the current group.
 */
int Process::me()
{
  const int r = PE_Groups::current_group().rank();
  return r;
}
 
/*! @brief Synchronizes all processors in the current group (waits until all processors have reached the barrier).
 *
 * Must be executed on all processors in the group.
 *
 */
void Process::barrier()
{
  PE_Groups::current_group().barrier(0);
}
 
 
/*! @brief Computes the sum of x over all processors in the current group.
 *
 * @param x Value to sum across all processors.
 * @return Sum of x over all processors in the current group.
 * @sa mp_max()
 */
double Process::mp_sum(double x)
{
  const Comm_Group& grp = PE_Groups::current_group();
  double y;
  grp.mp_collective_op(&x, &y, 1, Comm_Group::COLL_SUM);
  return y;
}
 
float Process::mp_sum(float x)
{
  const Comm_Group& grp = PE_Groups::current_group();
  float y;
  grp.mp_collective_op(&x, &y, 1, Comm_Group::COLL_SUM);
  return y;
}
 
/*! @brief Computes the sum of x over all processors in the current group.
 *
 * !!! Note that the sum of many int might result in a long !!!
 *
 * @param x Value to sum across all processors.
 * @return Sum of x over all processors in the current group.
 * @sa mp_max()
 */
trustIdType Process::mp_sum(trustIdType x)
{
  const Comm_Group& grp = PE_Groups::current_group();
  trustIdType y;
  grp.mp_collective_op(&x, &y, 1, Comm_Group::COLL_SUM);
  return y;
}
 
template<typename _TYPE_>
void mp_collective_op_arr(TRUSTArray<_TYPE_>& x, Comm_Group::Collective_Op op, int n)
{
  int sz = n==-1 ? x.size_array() : n;
  assert_parallel<_TYPE_>(sz);
  if (sz > 0)
    {
      _TYPE_ *data = x.addr();
      _TYPE_ *tmp = new _TYPE_[sz];
      const Comm_Group& grp = PE_Groups::current_group();
      grp.mp_collective_op(data, tmp, sz, op);
      memcpy(data, tmp, sz * sizeof(_TYPE_));
      delete[] tmp;
    }
}
 
template<typename _TYPE_>
void Process::mp_sum_for_each_item(TRUSTArray<_TYPE_>& x, int n) { mp_collective_op_arr(x, Comm_Group::COLL_SUM, n); }
 
template<typename _TYPE_>
void Process::mp_max_for_each_item(TRUSTArray<_TYPE_>& x, int n) { mp_collective_op_arr(x, Comm_Group::COLL_MAX, n); }
 
template<typename _TYPE_>
void Process::mp_min_for_each_item(TRUSTArray<_TYPE_>& x, int n) { mp_collective_op_arr(x, Comm_Group::COLL_MIN, n); }
 
/*! @brief C++14 compatible mp_sum_for_each: combine multiple mp_sum calls into one collective operation
 *  Usage: mp_sum_for_each(a, b); mp_sum_for_each(a, b, c); mp_sum_for_each(a, b, c, d); mp_sum_for_each(a, b, c, d, e);
 *  All arguments must be of the same type (double or int) and are modified in place.
 *  Supports 2-5 parameters.
 */
template<typename T>
void Process::mp_sum_for_each(T& arg1, T& arg2)
{
  T data[2] = {arg1, arg2};
  T tmp[2];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 2, Comm_Group::COLL_SUM);
  arg1 = tmp[0];
  arg2 = tmp[1];
}
 
template<typename T>
void Process::mp_sum_for_each(T& arg1, T& arg2, T& arg3)
{
  T data[3] = {arg1, arg2, arg3};
  T tmp[3];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 3, Comm_Group::COLL_SUM);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
}
 
template<typename T>
void Process::mp_sum_for_each(T& arg1, T& arg2, T& arg3, T& arg4)
{
  T data[4] = {arg1, arg2, arg3, arg4};
  T tmp[4];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 4, Comm_Group::COLL_SUM);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
  arg4 = tmp[3];
}
 
/*! @brief C++14 compatible mp_max_for_each: combine multiple mp_max calls into one collective operation */
template<typename T>
void Process::mp_max_for_each(T& arg1, T& arg2)
{
  T data[2] = {arg1, arg2};
  T tmp[2];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 2, Comm_Group::COLL_MAX);
  arg1 = tmp[0];
  arg2 = tmp[1];
}
 
template<typename T>
void Process::mp_max_for_each(T& arg1, T& arg2, T& arg3)
{
  T data[3] = {arg1, arg2, arg3};
  T tmp[3];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 3, Comm_Group::COLL_MAX);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
}
 
template<typename T>
void Process::mp_max_for_each(T& arg1, T& arg2, T& arg3, T& arg4)
{
  T data[4] = {arg1, arg2, arg3, arg4};
  T tmp[4];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 4, Comm_Group::COLL_MAX);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
  arg4 = tmp[3];
}
 
/*! @brief C++14 compatible mp_min_for_each: combine multiple mp_min calls into one collective operation */
template<typename T>
void Process::mp_min_for_each(T& arg1, T& arg2)
{
  T data[2] = {arg1, arg2};
  T tmp[2];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 2, Comm_Group::COLL_MIN);
  arg1 = tmp[0];
  arg2 = tmp[1];
}
 
template<typename T>
void Process::mp_min_for_each(T& arg1, T& arg2, T& arg3)
{
  T data[3] = {arg1, arg2, arg3};
  T tmp[3];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 3, Comm_Group::COLL_MIN);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
}
 
template<typename T>
void Process::mp_min_for_each(T& arg1, T& arg2, T& arg3, T& arg4)
{
  T data[4] = {arg1, arg2, arg3, arg4};
  T tmp[4];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 4, Comm_Group::COLL_MIN);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
  arg4 = tmp[3];
}
 
// 5-parameter versions
template<typename T>
void Process::mp_sum_for_each(T& arg1, T& arg2, T& arg3, T& arg4, T& arg5)
{
  T data[5] = {arg1, arg2, arg3, arg4, arg5};
  T tmp[5];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 5, Comm_Group::COLL_SUM);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
  arg4 = tmp[3];
  arg5 = tmp[4];
}
 
template<typename T>
void Process::mp_max_for_each(T& arg1, T& arg2, T& arg3, T& arg4, T& arg5)
{
  T data[5] = {arg1, arg2, arg3, arg4, arg5};
  T tmp[5];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 5, Comm_Group::COLL_MAX);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
  arg4 = tmp[3];
  arg5 = tmp[4];
}
 
template<typename T>
void Process::mp_min_for_each(T& arg1, T& arg2, T& arg3, T& arg4, T& arg5)
{
  T data[5] = {arg1, arg2, arg3, arg4, arg5};
  T tmp[5];
  const Comm_Group& grp = PE_Groups::current_group();
  grp.mp_collective_op(data, tmp, 5, Comm_Group::COLL_MIN);
  arg1 = tmp[0];
  arg2 = tmp[1];
  arg3 = tmp[2];
  arg4 = tmp[3];
  arg5 = tmp[4];
}
 
namespace
{
 
template <typename T>
T mp_operations_commun_(T x, Comm_Group::Collective_Op op)
{
  const Comm_Group& grp = PE_Groups::current_group();
  T y;
  grp.mp_collective_op(&x, &y, 1, op);
  return y;
}
}
 
/*! @brief Returns the maximum value of x across all processors in the current group.
 *
 * @param x Value to reduce with max operation.
 * @return Maximum value of x across all processors.
 */
int Process::mp_max(int x) { return mp_operations_commun_(x,Comm_Group::COLL_MAX); }
double Process::mp_max(double x) { return mp_operations_commun_(x,Comm_Group::COLL_MAX); }
#if INT_is_64_ == 2
trustIdType Process::mp_max(trustIdType x) { return mp_operations_commun_(x,Comm_Group::COLL_MAX); }
#endif
 
 
/*! @brief Returns the minimum value of x across all processors in the current group.
 *
 * @param x Value to reduce with min operation.
 * @return Minimum value of x across all processors.
 */
int Process::mp_min(int x) { return mp_operations_commun_(x,Comm_Group::COLL_MIN); }
double Process::mp_min(double x) { return mp_operations_commun_(x,Comm_Group::COLL_MIN); }
#if INT_is_64_ == 2
trustIdType Process::mp_min(trustIdType x) { return mp_operations_commun_(x,Comm_Group::COLL_MIN); }
#endif
 
/*! @brief Computes the partial sum of x over processors 0 to me()-1 (returns 0 on processor 0).
 *
 * @param x Value to include in the partial sum.
 * @return Partial sum of x over processors 0 to me()-1.
 * @sa Comm_Group::mppartial_sum()
 *
 */
trustIdType Process::mppartial_sum(trustIdType x)
{
  const Comm_Group& grp = PE_Groups::current_group();
  trustIdType xx = x;
  trustIdType y;
 
  grp.mp_collective_op(&xx, &y, 1, Comm_Group::COLL_PARTIAL_SUM);
  return y;
}
 
/*! @brief Computes the logical AND of b across all processors in the current group.
 *
 * @param b Boolean value to reduce with AND operation.
 * @return True if b is true on all processors, false otherwise.
 */
bool Process::mp_and(bool b)
{
  const Comm_Group& grp = PE_Groups::current_group();
  int x = b ? 1 : 0;
  int y;
  grp.mp_collective_op(&x, &y, 1, Comm_Group::COLL_MIN);
  return y == 1;
}
 
bool Process::mp_or(bool b)
{
  const Comm_Group& grp = PE_Groups::current_group();
  int x = b ? 1 : 0;
  int y;
  grp.mp_collective_op(&x, &y, 1, Comm_Group::COLL_MAX);
  return y == 1;
}
 
 
int Process::check_int_overflow(trustIdType v)
{
  if (v >= std::numeric_limits<int>::max())
    Process::exit("Value too big - above 32b and can not be converted to int!!");
  return static_cast<int>(v);
}
 
/*! @brief Exit routine for TRUST within a Kokkos region.
 *
 */
/*
KOKKOS_FUNCTION
void Process::Kokkos_exit(const char* str)
{
#ifdef TRUST_USE_GPU
 // ToDo Kokkos: try to exit more properly on device...
 Kokkos::abort(str);
 //Kokkos::finalize();
#else
    Process::exit(str);
#endif
}*/
 
/*! @brief Exit routine for TRUST on error.
 *
 * Saves memory and hierarchy information to files "memoire.dump" and "hierarchie.dump".
 *
 * @param i Exit code (forced to -1 if 0).
 */
void Process::exit(int i)
{
  Nom message="=========================================\nTRUST has caused an error and will stop.\nUnexpected error during TRUST calculation.";
  std::string jddLine = "\nError triggered at line " + std::to_string(EChaineJDD::file_cur_line_) + " in " + Objet_U::nom_du_cas().getString() + ".data";
  message+=jddLine;
  exit(message,i);
}
void Process::exit(const Nom& message ,int i)
{
  if (exception_sur_exit == 2)
    {
      ::exit(-1); // ND 11/01/23 a second ::exit(-1) is used in TRUST because if write permissions are missing, Process::exit() would be called recursively
    }
 
  if(je_suis_maitre())
    {
      Cerr << message << finl;
      Cerr.flush();
      if (!get_disable_stop() && Process::je_suis_maitre())
        {
          Nom nomfic( Objet_U::nom_du_cas() );
          nomfic += ".stop";
          {
            SFichier ficstop( nomfic );
            ficstop <<message<<finl;
          }
        }
    }
  Journal() << message << finl;
 
  if (exception_sur_exit)
    {
      // Throw an exception (used by Execute_parallel)
      throw TRUST_Error("Error in trust ",Process::me());
    }
  else
    {
      int abort=0;
#ifdef MPI_
      if (Process::is_parallel())
        {
          // user defined groups (if any !)
          if (PE_Groups::has_user_defined_group())
            {
              auto& grp = PE_Groups::get_user_defined_group();
              if (sub_type(Comm_Group_MPI,grp))
                ref_cast_non_const(Comm_Group_MPI,grp).free_all(); // free comm + group
            }
 
          const MPI_Comm& mpi_comm=ref_cast(Comm_Group_MPI,PE_Groups::groupe_TRUST()).get_mpi_comm();
          int tag = 666;
          int buffer[1]= {1};
          MPI_Request request;
 
          // Non-blocking send to me()+1
          int to_pe = (me()==nproc()-1?0:me()+1);
          MPI_Isend(buffer, 1, MPI_ENTIER, to_pe, tag, mpi_comm, &request);
 
          // Non-blocking receive from me()-1
          int from_pe = (me()==0?nproc()-1:me()-1);
          MPI_Irecv(buffer, 1, MPI_ENTIER, from_pe, tag, mpi_comm, &request);
 
          // Wait
          sleep(1);
 
          // Check if me() received from me()-1
          int ok;
          MPI_Status status;
          MPI_Test(&request,&ok,&status);
          if (!ok)
            abort=1;
        }
#endif
      if (abort)
        {
          if (!je_suis_maitre())
            {
              std_err_ << "!!! TRUST process number " << Process::me() << " exited unexpectedly ! See error message at the end of the file " << journal_file_name_ << " -> Aborting calculation..." << finl;
            }
          PE_Groups::groupe_TRUST().abort();
        }
      else
        {
#ifdef MPI_
          // On MPI_Finalize si MPI_Initialized and not MPI_Finalized
          int flag;
          MPI_Initialized(&flag);
          if (flag)
            {
              MPI_Finalized(&flag);
              if (!flag)
                MPI_Finalize();
            }
#endif
          PE_Groups::finalize();
        }
    }
  Kokkos::finalize();
  // Force exit
  if (i==0) i=-1;
  ::exit(i); // Only ::exit used in the code until 01/23. A second one was added because Process::exit is called recursively if the study directory write permissions are missing.
}
 
/*! @brief Abort routine for TRUST on a fatal error.
 *
 * In optimized mode, exits cleanly. In debug mode, aborts abruptly to allow debugger inspection.
 */
void Process::abort()
{
#ifdef NDEBUG
  // In optimized mode, exit cleanly.
  exit();
#else
  // In debug mode, abort abruptly to capture debugger info.
  ::abort(); // Only ::abort() used in the code
#endif
}
 
/*! @brief Returns a static Sortie object used as an event journal.
 *
 * If message_level <= verbose_level_, the message is written; otherwise it is discarded to a Sortie_Nulle.
 * If the journal file is open, messages are written to the file; otherwise to stderr.
 *
 * @param message_level Level of the message. Written only if <= verbose_level_.
 * @return Reference to the appropriate Sortie output stream.
 */
Sortie& Process::Journal(int message_level)
{
  if (message_level <= verbose_level_ && verbose_level_ > 0)
    {
      if (journal_file_open_)
        return journal_file_;
      else
        return std_err_;
    }
  return journal_zero_;
}
 
// Returns the RAM used by the current processor.
double Process::ram_processeur()
{
#ifdef PETSCKSP_H
  PetscLogDouble memoire;
  PetscMemoryGetCurrentUsage(&memoire);
  return memoire;
#else
  return 0;
#endif
}
#include <sys/resource.h>
double ru_maxrss()
{
  // Best to track OOM
  struct rusage usage;
  getrusage(RUSAGE_SELF, &usage);
  // ru_maxrss is in kilobytes
  long rss_kb = usage.ru_maxrss;
  return static_cast<double>(rss_kb*1024);
}
 
#ifndef __APPLE__
#include <malloc.h>
 
/*
struct mallinfo2 {
    size_t hblkhd;    // Space in mmapped regions (bytes)
    size_t hblks;     // Number of mmapped regions
    size_t usmblks;   // Always 0 (obsolete field)
    size_t fsmblks;   // Always 0 (obsolete field)
    size_t uordblks;  // Total allocated space (bytes)
    size_t fordblks;  // Total free space (bytes)
    size_t keepcost;  // Top-most, releasable space (bytes)
};
*/
double heap_allocation()
{
  // Best to track memory leak
#if defined(__GLIBC__) && __GLIBC_PREREQ(2, 33)
  struct mallinfo2 info = mallinfo2();
#else
  struct mallinfo info = mallinfo();
#endif
  return (double)info.uordblks;
}
 
static double heap_allocated_old=0;
 
#endif /* ndef __APPLE__ */
 
void Process::imprimer_ram_totale(int all_process)
{
  double memoire;
  //memoire = ram_processeur();
  memoire = ru_maxrss();
 
#ifndef __APPLE__
  double heap_allocated = heap_allocation();
#endif
 
  if (memoire)
    {
      //Cout << "RAM preliminary: PETSc " << ram_processeur() << "  ru_maxrss " << memoire << " mallinfo " << heap_allocated << finl;
      int Mo=1024*1024;
      if (all_process) Journal() << (int)(memoire/Mo) << " MBytes of RAM taken by the processor " << Process::me() << finl;
      {
        double max_memoire=Process::mp_max(memoire);
        double total_memoire=Process::mp_sum(memoire);
        Cout << (int)(total_memoire/Mo) << " MBytes of RAM taken by the calculation (max on a rank: "<<(int)(max_memoire/Mo)<<" MB)." << finl;
#ifndef __APPLE__
        Cout << "[RAM] Allocated heap on master rank: " << (int)(heap_allocated/Mo) << " Mbytes";
        double delta = heap_allocated - heap_allocated_old;
        if (delta!=0 && heap_allocated_old>0) Cout << " (" << (delta>0 ? "+" : "") << (long)delta << " bytes)";
        Cout << finl;
        heap_allocated_old = heap_allocated;
#ifdef TRUST_USE_GPU
        int Go = 1024 * Mo;
        double allocated = mp_max((double)DeviceMemory::allocatedBytesOnDevice());
        double total = static_cast<double>(DeviceMemory::deviceMemGetInfo(1));
        Cout << 0.1*(int)(10*allocated/Go) << " GBytes of maximal RAM allocated on a GPU (" <<  int(100 * allocated / total) << "%)" << finl;
#endif
#endif /* ndef __APPLE__ */
      }
#ifdef TRUST_USE_ROCM /* Only on adastra */
      // SUnreclaim on each process:
      std::ifstream meminfo("/proc/meminfo");
      std::string line;
      size_t sunreclaim_kb = 0;
      size_t mem_available_kb = 0;
      size_t mem_total_kb = 0;
      while (std::getline(meminfo, line))
        {
          if (line.substr(0, 9) == "MemTotal:")
            {
              size_t pos = line.find_first_of("0123456789");
              mem_total_kb = std::stoull(line.substr(pos));
            }
          if (line.substr(0, 13) == "MemAvailable:")
            {
              size_t pos = line.find_first_of("0123456789");
              mem_available_kb = std::stoull(line.substr(pos));
            }
          if (line.substr(0, 11) == "SUnreclaim:")
            {
              size_t pos = line.find_first_of("0123456789");
              sunreclaim_kb = std::stoull(line.substr(pos));
              break;
            }
        }
      Process::Journal() << "[RAM] SUnreclaim: " << sunreclaim_kb/1024 << " MB MemAvailable: " << mem_available_kb/1024 << " MB MemTotal: " << mem_total_kb/1024 << " MB " << finl;
#endif
    }
}
 
/*! @brief Initializes the journal file.
 *
 * @param verbose_level Messages at level <= verbose_level will be printed; others are discarded.
 * @param file_name If null pointer, everyone writes to cerr; otherwise the name of the file (must be different on each processor).
 * @param append Indicates whether the file is opened in append mode or not.
 */
void init_journal_file(int verbose_level, const char * file_name, int append)
{
  end_journal(verbose_level);
 
  if (verbose_level > 0)
    {
      if (file_name)
        {
          IOS_OPEN_MODE mode = ios::out;
          if (append)
            mode = ios::app;
          if (!journal_file_.ouvrir(file_name, mode))
            {
              Cerr << "Fatal error in init_journal_file: cannot open journal file" << finl;
              Process::exit();
            }
 
          journal_file_open_ = 1;
          journal_file_name_ = file_name;
        }
    }
  verbose_level_ = verbose_level;
}
 
void end_journal(int verbose_level)
{
  // Note: careful handling so that it "fails cleanly" if the destructor
  // writes to the journal!
  journal_file_.close();
  journal_file_open_ = 0;
}
 
/*! @brief Returns the Sortie object to which Cerr output is redirected.
 *
 * This can be std_err_ or journal_file_.
 *
 * @return Reference to the output stream used for Cerr.
 */
Sortie& get_Cerr()
{
  if (journal_file_open_ && cerr_to_journal_)
    return journal_file_;
  else
    {
      // If groups have not been initialized yet,
      // we cannot test whether we are the master.
      if (PE_Groups::get_nb_groups()==0)
        return std_err_;
      // Only the master processor writes to std_err_; others write to journal_file_
      if (Process::je_suis_maitre())
        return std_err_;
      else if (verbose_level_)
        return journal_file_;
      else
        return journal_zero_;
    }
}
 
/*! @brief Returns cout or the journal file on the master processor, or journal_zero_ otherwise.
 *
 * @return Reference to the output stream used for Cout.
 * @sa cerr_to_journal_
 *
 */
Sortie& get_Cout()
{
  if (Process::je_suis_maitre())
    {
      if (journal_file_open_ && cerr_to_journal_)
        return journal_file_;
      else
        return std_out_;
    }
  else
    {
      return journal_zero_;
    }
}
 
/*! @brief Changes the destination of Cerr and Cout.
 *
 * If flag=0, output goes to stderr and stdout; otherwise, if the journal file is open, output goes to
 * the journal; otherwise output goes to Sortie_Nulle.
 *
 * @param flag If 0, use stderr/stdout; if non-zero, redirect to journal or Sortie_Nulle.
 */
void set_Cerr_to_journal(int flag)
{
  cerr_to_journal_ = flag;
}
 
int get_journal_level()
{
  return verbose_level_;
}
 
void change_journal_level(int level)
{
  verbose_level_ = level;
}
 
/*! @brief Returns the disable_stop_ flag (Disable or not the writing of the .
 *
 * stop file)
 *
 */
int get_disable_stop()
{
  return disable_stop_;
}
 
/*! @brief Affects a new value to disable_stop_ flag (Disable or not the writing of the .
 *
 * stop file)
 *
 */
void change_disable_stop(int new_stop)
{
  disable_stop_ = new_stop;
}
 
// Explicit template instantiations for mp_*_for_each_item
template void Process::mp_sum_for_each_item<double>(TRUSTArray<double>&, int);
template void Process::mp_sum_for_each_item<trustIdType>(TRUSTArray<trustIdType>&, int);
template void Process::mp_sum_for_each_item<int>(TRUSTArray<int>&, int);
template void Process::mp_max_for_each_item<double>(TRUSTArray<double>&, int);
template void Process::mp_max_for_each_item<int>(TRUSTArray<int>&, int);
template void Process::mp_min_for_each_item<double>(TRUSTArray<double>&, int);
template void Process::mp_min_for_each_item<int>(TRUSTArray<int>&, int);
 
// Explicit template instantiations for mp_*_for_each (C++14 overloads)
// mp_sum_for_each
template void Process::mp_sum_for_each<double>(double&, double&);
template void Process::mp_sum_for_each<double>(double&, double&, double&);
template void Process::mp_sum_for_each<double>(double&, double&, double&, double&);
template void Process::mp_sum_for_each<double>(double&, double&, double&, double&, double&);
template void Process::mp_sum_for_each<int>(int&, int&);
template void Process::mp_sum_for_each<int>(int&, int&, int&);
template void Process::mp_sum_for_each<int>(int&, int&, int&, int&);
template void Process::mp_sum_for_each<int>(int&, int&, int&, int&, int&);
// mp_max_for_each
template void Process::mp_max_for_each<double>(double&, double&);
template void Process::mp_max_for_each<double>(double&, double&, double&);
template void Process::mp_max_for_each<double>(double&, double&, double&, double&);
template void Process::mp_max_for_each<double>(double&, double&, double&, double&, double&);
template void Process::mp_max_for_each<int>(int&, int&);
template void Process::mp_max_for_each<int>(int&, int&, int&);
template void Process::mp_max_for_each<int>(int&, int&, int&, int&);
template void Process::mp_max_for_each<int>(int&, int&, int&, int&, int&);
// mp_min_for_each
template void Process::mp_min_for_each<double>(double&, double&);
template void Process::mp_min_for_each<double>(double&, double&, double&);
template void Process::mp_min_for_each<double>(double&, double&, double&, double&);
template void Process::mp_min_for_each<double>(double&, double&, double&, double&, double&);
template void Process::mp_min_for_each<int>(int&, int&);
template void Process::mp_min_for_each<int>(int&, int&, int&);
template void Process::mp_min_for_each<int>(int&, int&, int&, int&);
template void Process::mp_min_for_each<int>(int&, int&, int&, int&, int&);