Commit e666bb47 authored by pugmire's avatar pugmire
Browse files

Various tweaks to the streamline algorithms.

Modify how timings are done.
Added back latency hiding in the masterSlave streamline algorithm.



git-svn-id: http://visit.ilight.com/svn/visit/trunk/src@6658 18c085ea-50e0-402c-830e-de6fd14e8384
parent a123eff3
......@@ -174,12 +174,17 @@ avtMasterSlaveSLAlgorithm::Create(avtStreamlineFilter *slFilter,
//
// Dave Pugmire, Tue Mar 10 12:41:11 EDT 2009
// Generalized domain to include domain/time. Pathine cleanup.
//
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
avtMasterSlaveSLAlgorithm::avtMasterSlaveSLAlgorithm(avtStreamlineFilter *slFilter,
int maxCount)
: avtParSLAlgorithm(slFilter)
: avtParSLAlgorithm(slFilter),
SleepTime("sleepT"), LatencyTime("latT"), MaxLatencyTime("maxLatT"), SleepCnt("sleepC"),
LatencySavingCnt("latSaveCnt")
{
NUM_DOMAINS = numDomains * numTimeSteps;
maxCnt = maxCount;
......@@ -240,7 +245,6 @@ avtMasterSlaveSLAlgorithm::Sleep()
if (sleepMicroSec > 0)
{
//debug1<<"Sleep for "<<sleepMicroSec<<" microSec\n";
int sleepTimer = visitTimer->StartTimer();
struct timespec ts = {0, sleepMicroSec*1000};
nanosleep(&ts, 0);
......@@ -250,25 +254,46 @@ avtMasterSlaveSLAlgorithm::Sleep()
}
// ****************************************************************************
// Method: avtMasterSlaveSLAlgorithm::CalculateStatistics
// Method: avtMasterSlaveSLAlgorithm::CompileTimingStatistics
//
// Purpose:
// Calculate statistics.
//
// Programmer: Dave Pugmire
// Creation: January 27, 2009
//
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
void
avtMasterSlaveSLAlgorithm::CalculateStatistics()
avtMasterSlaveSLAlgorithm::CompileTimingStatistics()
{
avtParSLAlgorithm::CalculateStatistics();
avtParSLAlgorithm::CompileTimingStatistics();
ComputeStatistic(SleepTime);
ComputeStatistic(LatencyTime);
ComputeStatistic(MaxLatencyTime);
}
ComputeStatistics(SleepTime);
ComputeStatistics(LatencyTime);
ComputeStatistics(MaxLatencyTime);
ComputeStatistics(SleepCnt);
// ****************************************************************************
// Method: avtMasterSlaveSLAlgorithm::CompileCounterStatistics
//
// Purpose:
// Calculate statistics.
//
// Programmer: Dave Pugmire
// Creation: January 27, 2009
//
// ****************************************************************************
void
avtMasterSlaveSLAlgorithm::CompileCounterStatistics()
{
avtParSLAlgorithm::CompileCounterStatistics();
ComputeStatistic(SleepCnt);
ComputeStatistic(LatencySavingCnt);
}
// ****************************************************************************
......@@ -279,6 +304,9 @@ avtMasterSlaveSLAlgorithm::CalculateStatistics()
//
// Programmer: Dave Pugmire
// Creation: January 27, 2009
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
......@@ -286,8 +314,9 @@ void
avtMasterSlaveSLAlgorithm::CalculateExtraTime()
{
avtParSLAlgorithm::CalculateExtraTime();
ExtraTime.value -= SleepTime.value;
ExtraTime.value -= LatencyTime.value;
if (SleepTime.value > 0.0)
ExtraTime.value -= SleepTime.value;
}
// ****************************************************************************
......@@ -319,7 +348,10 @@ avtMasterSlaveSLAlgorithm::ReportTimings(ostream &os, bool totals)
//
// Programmer: Dave Pugmire
// Creation: January 27, 2009
//
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
void
......@@ -327,6 +359,7 @@ avtMasterSlaveSLAlgorithm::ReportCounters(ostream &os, bool totals)
{
avtParSLAlgorithm::ReportCounters(os, totals);
PrintCounter(os, "SleepCnt", SleepCnt, totals);
PrintCounter(os, "LSaveCnt", LatencySavingCnt, totals);
}
......@@ -448,7 +481,7 @@ avtMasterSLAlgorithm::Initialize(std::vector<avtStreamlineWrapper *> &seedPts)
}
// ****************************************************************************
// Method: avtMasterSLAlgorithm::CalculateStatistics
// Method: avtMasterSLAlgorithm::CompileTimingStatistics
//
// Purpose:
// Calculate statistics. For the master, we need to be a little tricky.
......@@ -458,22 +491,44 @@ avtMasterSLAlgorithm::Initialize(std::vector<avtStreamlineWrapper *> &seedPts)
//
// Programmer: Dave Pugmire
// Creation: January 27, 2009
//
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
void
avtMasterSLAlgorithm::CalculateStatistics()
avtMasterSLAlgorithm::CompileTimingStatistics()
{
LatencyTime.value = -1;
MaxLatencyTime.value = -1;
IOTime.value = -1;
IntegrateTime.value = -1;
SortTime.value = -1;
avtMasterSlaveSLAlgorithm::CompileTimingStatistics();
}
// ****************************************************************************
// Method: avtMasterSLAlgorithm::CompileCounterStatistics
//
// Purpose:
// Calculate statistics. For the master, we need to be a little tricky.
// The master doesn't do a lot of things, so we don't want to skew the
// statistics. So, set the timer/counters to -1 and then ignore these values
// when reporting statistics.
//
// Programmer: Dave Pugmire
// Creation: January 27, 2009
//
// ****************************************************************************
void
avtMasterSLAlgorithm::CompileCounterStatistics()
{
IntegrateCnt.value = -1;
DomLoadCnt.value = -1;
DomPurgeCnt.value = -1;
avtMasterSlaveSLAlgorithm::CalculateStatistics();
avtMasterSlaveSLAlgorithm::CompileCounterStatistics();
}
// ****************************************************************************
......@@ -966,10 +1021,15 @@ avtMasterSLAlgorithm::ManageWorkgroup()
PrintStatus();
if (slaveUpdate)
ManageSlaves();
if (masterUpdate)
ManageMasters();
if (!slaveUpdate && !masterUpdate)
Sleep();
else
{
if (slaveUpdate)
ManageSlaves();
if (masterUpdate)
ManageMasters();
}
debug1<<endl<<"Post-Mortem"<<endl;
PrintStatus();
......@@ -1659,23 +1719,10 @@ avtSlaveSLAlgorithm::UpdateStatus()
status[DomToIdx( (*s)->domain )] --;
}
//Start/stop the latency timer.
if (prevWorkToDo && !workToDo)
{
latencyTimer = visitTimer->StartTimer();
}
else if (!prevWorkToDo && workToDo && latencyTimer != -1)
{
double t = visitTimer->StopTimer(latencyTimer, "Latency");
LatencyTime.value += t;
if (t > MaxLatencyTime.value)
MaxLatencyTime.value = t;
}
if (!workToDo)
{
debug1<<"Nothing to do.....\n";
}
}
......@@ -1739,6 +1786,9 @@ avtSlaveSLAlgorithm::SendStatus(bool forceSend)
//
// Dave Pugmire, Wed Mar 18 17:17:40 EDT 2009
// Allow masters to share work loads.
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Changes related to trying to hide latency with slaves.
//
// ****************************************************************************
......@@ -1752,7 +1802,6 @@ avtSlaveSLAlgorithm::Execute()
SendStatus(true);
Barrier();
#if 1
while ( 1 )
{
//Fill oobSLs list.
......@@ -1767,12 +1816,30 @@ avtSlaveSLAlgorithm::Execute()
else
++si;
}
if (latencyTimer != -1 && !activeSLs.empty())
{
double t = visitTimer->StopTimer(latencyTimer, "Latency");
debug1<<"End latency: time= "<<t<<endl;
LatencyTime.value += t;
if (t > MaxLatencyTime.value)
MaxLatencyTime.value = t;
latencyTimer = -1;
}
bool done = false, newMsgs = false;
while (!activeSLs.empty())
while (!activeSLs.empty() && !done)
{
avtStreamlineWrapper *s = activeSLs.front();
activeSLs.pop_front();
bool doThis = false;
if (doThis && activeSLs.empty())
{
debug1<<"Latency saving sendStatus"<<endl;
LatencySavingCnt.value++;
SendStatus();
}
debug1<<"Integrate "<<s->domain<<endl;
IntegrateStreamline(s);
......@@ -1810,146 +1877,16 @@ avtSlaveSLAlgorithm::Execute()
SendStatus();
//Nothing to do, take a snooze....
if (activeSLs.empty())
Sleep();
}
#endif
#if 0
bool sendStatus = true;
while ( 1 )
{
//Fill oobSLs list.
list<avtStreamlineWrapper *>::iterator si = activeSLs.begin();
while (si != activeSLs.end())
{
if (!DomainLoaded((*si)->domain))
{
oobSLs.push_back(*si);
si = activeSLs.erase(si);
}
else
++si;
}
bool done = false, newMsgs = false;
while (!activeSLs.empty())
if (!workToDo)
{
avtStreamlineWrapper *s = activeSLs.front();
activeSLs.pop_front();
debug1<<"Actives= "<<activeSLs.size()<<endl;
sendStatus = true;
if (activeSLs.empty())
{
debug1<<"Latency saving status send\n";
SendStatus();
sendStatus = false;
}
IntegrateStreamline(s);
if (s->status == avtStreamlineWrapper::TERMINATE)
{
terminatedSLs.push_back(s);
numTerminated++;
}
else
if (latencyTimer == -1)
{
if (DomainLoaded(s->domain))
activeSLs.push_back(s);
else
oobSLs.push_back(s);
latencyTimer = visitTimer->StartTimer();
debug1<<"Begin latency!\n";
}
bool newM;
ProcessMessages(done, newM);
if (done)
break;
if (newM)
newMsgs = true;
}
activeSLs.splice(activeSLs.end(), oobSLs);
bool newM;
ProcessMessages(done, newM);
if (done)
break;
if (newM)
newMsgs = true;
//See if we have any SLs.
int earlyTerminations = 0;
bool newSLs = RecvSLs(activeSLs, earlyTerminations);
numTerminated += earlyTerminations;
if (newSLs)
debug1<<"Recv some SLs. sz= "<<activeSLs.size()<<endl;
if (sendStatus || newSLs || newMsgs || numTerminated > 0)
SendStatus();
else
{
if (!workToDo)
{
timeout++;
if (timeout > 1000)
{
debug1<<"TIMEOUT FORCE SEND STATUS\n";
SendStatus(true);
Sleep();
timeout = 0;
}
else if (timeout % 200 == 0)
Sleep();
}
}
CheckPendingSendRequests();
}
#endif
#if 0
while ( 1 )
{
// Integrate while I have work to do.
int cnt = 0;
while (cnt < maxCnt && !activeSLs.empty())
{
avtStreamlineWrapper *s = activeSLs.front();
activeSLs.pop_front();
if (DomainLoaded(s->domain))
{
IntegrateStreamline(s);
if (s->status == avtStreamlineWrapper::TERMINATE)
{
terminatedSLs.push_back(s);
numTerminated++;
}
else
activeSLs.push_back(s);
cnt++;
}
else
oobSLs.push_back(s);
Sleep();
}
activeSLs.splice(activeSLs.end(), oobSLs);
bool done, newMsgs;
ProcessMessages(done, newMsgs);
if (done)
break;
//See if we have any SLs.
int earlyTerminations = 0;
RecvSLs(activeSLs, earlyTerminations);
numTerminated += earlyTerminations;
SendStatus();
}
#endif
TotalTime.value += visitTimer->StopTimer(timer, "Execute");
}
......
......@@ -64,7 +64,10 @@ class SlaveInfo;
// Generalized domain to include domain/time. Pathine cleanup.
//
// Dave Pugmire, Wed Mar 18 17:17:40 EDT 2009
// Allow masters to share work loads.
// Allow masters to share work loads.
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
......@@ -113,8 +116,9 @@ class avtMasterSlaveSLAlgorithm : public avtParSLAlgorithm
//Statistics and coutners.
int latencyTimer;
SLStatistics SleepTime, LatencyTime, MaxLatencyTime;
SLStatistics SleepCnt;
virtual void CalculateStatistics();
SLStatistics SleepCnt, LatencySavingCnt;
virtual void CompileTimingStatistics();
virtual void CompileCounterStatistics();
virtual void CalculateExtraTime();
virtual void ReportTimings(ostream &os, bool totals);
virtual void ReportCounters(ostream &os, bool totals);
......@@ -160,7 +164,8 @@ class avtMasterSLAlgorithm : public avtMasterSlaveSLAlgorithm
virtual void ManageWorkgroup();
virtual void ManageSlaves();
virtual void ManageMasters();
virtual void CalculateStatistics();
virtual void CompileTimingStatistics();
virtual void CompileCounterStatistics();
int workGroupActiveSLs, workGroupSz;
bool done, slaveUpdate, masterUpdate;
......
......@@ -59,10 +59,14 @@ int avtParSLAlgorithm::STREAMLINE_TAG = 420001;
// Programmer: Dave Pugmire
// Creation: January 27, 2009
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
avtParSLAlgorithm::avtParSLAlgorithm(avtStreamlineFilter *slFilter)
: avtSLAlgorithm(slFilter)
: avtSLAlgorithm(slFilter),
CommTime("comT"), MsgCnt("msgC"), SLCommCnt("slcC"), BytesCnt("byteC")
{
nProcs = PAR_Size();
rank = PAR_Rank();
......@@ -212,12 +216,16 @@ avtParSLAlgorithm::CleanupAsynchronous()
// Dave Pugmire, Wed Mar 18 17:07:07 EDT 2009
// Delete entry from map after send is complete.
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
void
avtParSLAlgorithm::CheckPendingSendRequests()
{
debug5<<"avtParSLAlgorithm::CheckPendingSendRequests()\n";
int communicationTimer = visitTimer->StartTimer();
if (sendSLBufferMap.size() > 0)
{
vector<MPI_Request> req, copy;
......@@ -309,6 +317,8 @@ avtParSLAlgorithm::CheckPendingSendRequests()
}
}
CommTime.value += visitTimer->StopTimer(communicationTimer,
"CheckPending");
debug5 << "DONE CheckPendingSendRequests()\n";
}
......@@ -439,13 +449,16 @@ avtParSLAlgorithm::SendAllMsg(vector<int> &msg)
//
// Modifications:
//
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
void
avtParSLAlgorithm::RecvMsgs(std::vector<std::vector<int> > &msgs)
{
debug5<<"avtParSLAlgorithm::RecvMsgs()\n";
int communicationTimer = visitTimer->StartTimer();
msgs.resize(0);
while (true)
......@@ -502,6 +515,8 @@ avtParSLAlgorithm::RecvMsgs(std::vector<std::vector<int> > &msgs)
if (num == 0)
break;
}
CommTime.value += visitTimer->StopTimer(communicationTimer,
"RecvMsgs");
}
......@@ -567,7 +582,7 @@ avtParSLAlgorithm::SendSLs(int dst,
BytesCnt.value += sz;
CommTime.value += visitTimer->StopTimer(communicationTimer,
"StreamlineCommunication");
"SendSLs");
}
......@@ -584,11 +599,15 @@ avtParSLAlgorithm::SendSLs(int dst,
//
// Dave Pugmire, Wed Mar 18 17:17:40 EDT 2009
// RecvSLs broken into two methods.
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
int
avtParSLAlgorithm::RecvSLs(list<avtStreamlineWrapper *> &recvSLs)
{
int communicationTimer = visitTimer->StartTimer();
int slCount = 0;
while (true)
......@@ -644,7 +663,9 @@ avtParSLAlgorithm::RecvSLs(list<avtStreamlineWrapper *> &recvSLs)
if (num == 0)
break;
}
CommTime.value += visitTimer->StopTimer(communicationTimer,
"RecvSLs");
return slCount;
}
......@@ -743,25 +764,42 @@ avtParSLAlgorithm::ExchangeSLs(list<avtStreamlineWrapper *> &streamlines,
}
// ****************************************************************************
// Method: avtParSLAlgorithm::CalculateStatistics
// Method: avtParSLAlgorithm::CalculateTimingStatistics
//
// Purpose:
// Compute statistics over a value.
//
// Programmer: Dave Pugmire
// Creation: January 28, 2009
// Creation: March 23, 2009
//
// ****************************************************************************
void
avtParSLAlgorithm::CompileTimingStatistics()
{
avtSLAlgorithm::CompileTimingStatistics();
ComputeStatistic(CommTime);
}
// ****************************************************************************
// Method: avtParSLAlgorithm::CalculateCounterStatistics
//
// Purpose:
// Compute statistics over a value.
//
// Programmer: Dave Pugmire
// Creation: March 23, 2009
//
//
// ****************************************************************************
void
avtParSLAlgorithm::CalculateStatistics()
avtParSLAlgorithm::CompileCounterStatistics()
{
avtSLAlgorithm::CalculateStatistics();
ComputeStatistics(CommTime);
ComputeStatistics(MsgCnt);
ComputeStatistics(SLCommCnt);
ComputeStatistics(BytesCnt);
avtSLAlgorithm::CompileCounterStatistics();
ComputeStatistic(MsgCnt);
ComputeStatistic(SLCommCnt);
ComputeStatistic(BytesCnt);
}
// ****************************************************************************
......@@ -773,13 +811,17 @@ avtParSLAlgorithm::CalculateStatistics()
// Programmer: Dave Pugmire
// Creation: January 28, 2009
//
// Dave Pugmire, Mon Mar 23 12:48:12 EDT 2009
// Change how timings are reported/calculated.
//
// ****************************************************************************
void
avtParSLAlgorithm::CalculateExtraTime()
{
avtSLAlgorithm::CalculateExtraTime();
ExtraTime.value -= CommTime.value;
if (CommTime.value > 0.0)
ExtraTime.value -= CommTime.value;
}
// ****************************************************************************
......