Add two more caliper configs

conf/caliper/diy-aggregate-mpi.conf

+# This config will printout at the end of the code (technically, on
+# MPI_Finalize) the min/max/avg runtime of all marked sections and MPI functions
+# across all ranks.
+#
+# To run:
+#   CALI_CONFIG_FILE=diy-aggregate-mpi.conf mpirun ...
+
+# This is just a verbatim copy of caliper's built-in mpi-runtime-report It's
+# here for the tweaking (e.g., turning off the MPI function reporting); this
+# particular config could be equivalently set via
+# CALI_CONFIG_PROFILE=mpi-runtime-report
+CALI_SERVICES_ENABLE=aggregate,event,mpi,mpireport,timestamp
+CALI_MPI_BLACKLIST=MPI_Comm_rank,MPI_Comm_size,MPI_Wtick,MPI_Wtime
+CALI_EVENT_ENABLE_SNAPSHOT_INFO=false
+CALI_TIMER_SNAPSHOT_DURATION=true
+CALI_TIMER_INCLUSIVE_DURATION=false
+CALI_TIMER_UNIT=sec
+CALI_MPIREPORT_CONFIG="select min(sum#time.duration) as \"Min time/rank\",max(sum#time.duration) as \"Max time/rank\", avg(sum#time.duration) as \"Avg time/rank\", percent_total(sum#time.duration) as \"Time % (total)\" group by prop:nested format tree"
+CALI_MPIREPORT_FILENAME=stderr

conf/caliper/diy-aggregate-serial.conf

+# This config prints out, for each rank, all annotations (for each block or
+# exchange round, or across all of them, depending on the CALI_AGGREGATE_KEY
+# below). By default, the output goes to stdout, but it can be split into
+# separate files with MPI rank encoded in the filename by setting
+# CALI_REPORT_FILENAME below.
+#
+# To run:
+#   CALI_CONFIG_FILE=diy-aggregate-serial.conf mpirun ...
+
+CALI_SERVICES_ENABLE=event,aggregate,report,mpi,timestamp
+#CALI_AGGREGATE_KEY=annotation                                   # just annotation (across all blocks and rounds)
+CALI_AGGREGATE_KEY=annotation,diy.block                         # annotation for each block (across all rounds)
+#CALI_AGGREGATE_KEY=annotation,diy.block,diy.exchange-round      # annotation for each block for each round
+#CALI_AGGREGATE_KEY=annotation,mpi.function                      # annotation and mpi function
+
+#CALI_REPORT_FILENAME=diy-caliper-profile-%mpi.rank%.cali
+
+# Uncommenting one of these functions will profile MPI calls
+#CALI_MPI_BLACKLIST=MPI_Init                                    # record everything, but MPI_Init
+#CALI_MPI_WHITELIST=MPI_Test,MPI_Issend,MPI_Isend,MPI_Recv      # record only specified functions
+#CALI_MPI_MSG_TRACING=true                                      # detailed profiling of individual messages (powerful, but overwhelming)

include/diy/detail/master/execution.hpp

@@ -27,9 +27,7 @@ struct diy::Master::ProcessBlock
 
       int i   = blocks[cur];
       int gid = master.gid(i);
-#if defined(DIY_USE_CALIPER)
-      cali::Annotation::Guard g( cali::Annotation("diy.block").set(gid) );
-#endif
+      stats::Annotation::Guard g( stats::Annotation("diy.block").set(gid) );
 
       if (master.block(i))
       {

include/diy/master.hpp

@@ -347,9 +347,7 @@ namespace diy
     public:
       std::shared_ptr<spd::logger>  log = get_logger();
       stats::Profiler               prof;
-#if defined(DIY_USE_CALIPER)
-      cali::Annotation              exchange_round_annotation { "diy.exchange-round" };
-#endif
+      stats::Annotation             exchange_round_annotation { "diy.exchange-round" };
   };
 
   struct Master::SkipNoIncoming
@@ -622,9 +620,7 @@ void
 diy::Master::
 foreach_(const Callback<Block>& f, const Skip& skip)
 {
-#if defined(DIY_USE_CALIPER)
     exchange_round_annotation.set(exchange_round_);
-#endif
 
     auto scoped = prof.scoped("foreach");
     DIY_UNUSED(scoped);
@@ -701,10 +697,7 @@ iexchange_(const    ICallback<Block>&   f,
     // prepare for next round
     incoming_.erase(exchange_round_);
     ++exchange_round_;
-#if defined(DIY_USE_CALIPER)
     exchange_round_annotation.set(exchange_round_);
-#endif
-
 
     //IExchangeInfoDUD iexchange(comm_, min_queue_size, max_hold_time, fine, prof);
     IExchangeInfoCollective iexchange(comm_, min_queue_size, max_hold_time, fine, prof);
@@ -903,7 +896,7 @@ send_queue(int              from_gid,
            bool             remote,
            IExchangeInfo*   iexchange)
 {
-    cali::Annotation::Guard g( cali::Annotation("diy.block").set(from_gid) );
+    stats::Annotation::Guard g( stats::Annotation("diy.block").set(from_gid) );
 
     // skip empty queues and hold queues shorter than some limit for some time
     if ( iexchange && (out_queue.size() == 0 || iexchange->hold(out_queue.size())) )
@@ -1194,9 +1187,7 @@ flush(bool remote)
   // prepare for next round
   incoming_.erase(exchange_round_);
   ++exchange_round_;
-#if defined(DIY_USE_CALIPER)
   exchange_round_annotation.set(exchange_round_);
-#endif
 
 
   if (remote)

include/diy/stats.hpp

@@ -166,8 +166,26 @@ struct Profiler
         DurationAccumulator total;
 };
 #endif  // DIY_PROFILE
+
+// Annotations don't do anything without Caliper
+struct Annotation
+{
+    struct Guard
+    {
+                    Guard(Annotation& a)            {}
+    };
+
+                    Annotation(const char*)         {}
+
+    template<class T>
+    Annotation&     set(T)                          { return *this; }
+
+};
+
 #else   // DIY_USE_CALIPER
 
+using Annotation = cali::Annotation;
+
 struct Profiler
 {
     using   Scoped = ScopedProfile<Profiler>;