Commit 13183666 authored by whitlocb's avatar whitlocb

This update enhances cumulative query selections so they can cache intermediate

results. When available (and appropriate) the cached results can be used to 
bypass the time loop stage that makes CQ selections costly, resulting in a 
large speedup. For example, CQ histogramming operations are able to reuse the
intermediate results.



git-svn-id: http://visit.ilight.com/svn/visit/trunk/src@16119 18c085ea-50e0-402c-830e-de6fd14e8384
parent 96b44d38
......@@ -51,9 +51,9 @@
class avtDataRepresentation;
class avtIntervalTree;
class avtNamedSelectionExtension;
class avtWebpage;
// ****************************************************************************
// Class: avtDataset
//
......@@ -156,7 +156,7 @@ class PIPELINE_API avtDataset : public avtDataObject
friend class avtLineScanQuery;
friend class avtLineSurfaceFilter;
friend class avtExecuteThenTimeLoopFilter;
friend class avtNamedSelectionManager;
friend class avtNamedSelectionExtension;
friend class avtTimeIteratorExpression;
friend class avtXRayImageQuery;
friend class avtResampleExpression;
......
......@@ -37,6 +37,23 @@
*****************************************************************************/
#include <avtNamedSelectionExtension.h>
#include <avtDataset.h>
#include <vtkCellData.h>
#include <vtkDataArray.h>
#include <vtkDataSet.h>
#include <DebugStream.h>
avtNamedSelectionCacheItem::avtNamedSelectionCacheItem() : properties()
{
}
avtNamedSelectionCacheItem::~avtNamedSelectionCacheItem()
{
}
avtNamedSelectionExtension::avtNamedSelectionExtension()
{
}
......@@ -45,7 +62,181 @@ avtNamedSelectionExtension::~avtNamedSelectionExtension()
{
}
// ****************************************************************************
// Method: avtNamedSelectionExtension::GetContract
//
// Purpose:
// Return the contract that we'll use for pipeline execution.
//
// Arguments:
//
// Returns:
//
// Note: We turn on zone numbers in the returned contract. If the prior
// contract did not have zone numbers then we'll need to execute
// the pipeline.
//
// Programmer: Brad Whitlock
// Creation: Tue Sep 6 15:44:42 PDT 2011
//
// Modifications:
//
// ****************************************************************************
avtContract_p
avtNamedSelectionExtension::GetContract(avtDataObject_p dob, bool &needsUpdate)
{
avtContract_p c1 = dob->GetContractFromPreviousExecution();
avtContract_p contract;
if (c1->GetDataRequest()->NeedZoneNumbers() == false)
{
// If we don't have zone numbers, then get them, even if we have
// to re-execute the whole darn pipeline.
contract = new avtContract(c1);
contract->GetDataRequest()->TurnZoneNumbersOn();
needsUpdate = true;
}
else
{
contract = c1;
needsUpdate = false;
}
return contract;
}
// ****************************************************************************
// Method: avtNamedSelectionExtension::GetSelection
//
// Purpose:
// Return the selection as a set of domain and zone numbers.
//
// Arguments:
// dob : The input to the selection's pipeline.
// props : The selection properties (ignored for default case)
// cache : The selection manager's cache (ignored)
// doms : The return vector of domains.
// zones : The return vector of cells.
//
// Returns: The cells that make up the selection are returned in doms/zones
// vectors.
//
// Note:
//
// Programmer: Brad Whitlock
// Creation: Tue Sep 6 16:02:40 PDT 2011
//
// Modifications:
//
// ****************************************************************************
void
avtNamedSelectionExtension::GetSelection(avtDataObject_p dob,
const SelectionProperties &/*props*/,
avtNamedSelectionCache &/*cache*/,
std::vector<int> &doms, std::vector<int> &zones)
{
const char *mName = "avtNamedSelectionExtension::GetSelection: ";
bool needsUpdate = false;
avtContract_p contract = GetContract(dob, needsUpdate);
if (needsUpdate)
{
debug1 << mName << "Must re-execute pipeline to create named selection" << endl;
dob->Update(contract);
debug1 << mName << "Done re-executing pipeline to create named selection" << endl;
}
avtDataset_p ds;
CopyTo(ds, dob);
GetSelectionFromDataset(ds, doms, zones);
}
// ****************************************************************************
// Method: avtNamedSelectionExtension::GetSelectionFromDataset
//
// Purpose:
// Convert the data in the data tree into a selection stored in int vectors.
//
// Arguments:
// ds : The dataset.
// doms : The return vector containing the domains.
// zones : The return vector containing the zones.
//
// Returns:
//
// Note:
//
// Programmer: Brad Whitlock
// Creation: Tue Sep 6 15:45:54 PDT 2011
//
// Modifications:
// Brad Whitlock, Tue Sep 6 15:54:01 PDT 2011
// I moved it from the named selection manager and changed it to a 2 pass
// scheme to remove successive calls to resize.
//
// ****************************************************************************
void
avtNamedSelectionExtension::FreeUpResources()
avtNamedSelectionExtension::GetSelectionFromDataset(avtDataset_p ds,
std::vector<int> &doms, std::vector<int> &zones)
{
const char *mName = "avtNamedSelectionExtension::GetSelectionFromDataTree: ";
int nleaves = 0;
avtDataTree_p tree = ds->GetDataTree();
vtkDataSet **leaves = tree->GetAllLeaves(nleaves);
unsigned int idx = 0, maxSize = 0;
for(int pass = 0; pass < 2; ++pass)
{
if(pass == 1)
{
doms.resize(maxSize);
zones.resize(maxSize);
}
for (int i = 0 ; i < nleaves ; i++)
{
if (leaves[i]->GetNumberOfCells() == 0)
continue;
vtkDataArray *ocn = leaves[i]->GetCellData()->
GetArray("avtOriginalCellNumbers");
if (ocn == NULL)
{
// Write an error to the logs but don't fail out since we have
// a collective communication coming up.
debug5 << mName
<< "This dataset has no original cell numbers so it cannot "
"contribute to the selection." << endl;
}
else
{
unsigned int *ptr = (unsigned int *) ocn->GetVoidPointer(0);
if (ptr == NULL)
{
// Write an error to the logs but don't fail out since we have
// a collective communication coming up.
debug5 << mName
<< "This dataset has no original cell numbers so it "
"cannot contribute to the selection." << endl;
}
else
{
unsigned int ncells = leaves[i]->GetNumberOfCells();
if(pass == 0)
maxSize += leaves[i]->GetNumberOfCells();
else
{
// We have original cell numbers so add them to the selection.
for (int j = 0 ; j < ncells ; j++)
{
doms[idx] = ptr[2*j];
zones[idx] = ptr[2*j+1];
idx++;
}
}
}
}
}
}
delete [] leaves;
}
......@@ -39,10 +39,29 @@
#define AVT_NAMED_SELECTION_EXTENSION_H
#include <pipeline_exports.h>
#include <vector>
#include <avtContract.h>
#include <avtDataset.h>
#include <avtDataObject.h>
#include <MRUCache.h>
#include <SelectionProperties.h>
// Base class for things we can stick in the cache.
class PIPELINE_API avtNamedSelectionCacheItem
{
public:
avtNamedSelectionCacheItem();
virtual ~avtNamedSelectionCacheItem();
SelectionProperties properties;
};
typedef MRUCache<std::string,
avtNamedSelectionCacheItem *,
MRUCache_Delete,
10> avtNamedSelectionCache;
// ****************************************************************************
// Class: avtNamedSelectionExtension
//
......@@ -57,7 +76,9 @@
// Creation: Mon Dec 13 16:09:53 PST 2010
//
// Modifications:
//
// Brad Whitlock, Tue Sep 6 14:47:35 PDT 2011
// I changed the API.
//
// ****************************************************************************
class PIPELINE_API avtNamedSelectionExtension
......@@ -65,10 +86,16 @@ class PIPELINE_API avtNamedSelectionExtension
public:
avtNamedSelectionExtension();
virtual ~avtNamedSelectionExtension();
virtual avtDataObject_p GetSelectedData(avtDataObject_p dob,
avtContract_p contract,
const SelectionProperties &props) = 0;
virtual void FreeUpResources();
virtual void GetSelection(avtDataObject_p dob, const SelectionProperties &props,
avtNamedSelectionCache &cache,
std::vector<int> &doms, std::vector<int> &zones);
protected:
avtContract_p GetContract(avtDataObject_p dob, bool &needsUpdate);
void GetSelectionFromDataset(avtDataset_p tree,
std::vector<int> &doms, std::vector<int> &zones);
};
#endif
......@@ -44,10 +44,6 @@
#include <avtNamedSelectionManager.h>
#include <avtNamedSelectionExtension.h>
#include <vtkCellData.h>
#include <vtkDataArray.h>
#include <vtkDataSet.h>
#include <avtDataObjectSource.h>
#include <avtDataset.h>
#include <avtNamedSelection.h>
......@@ -56,6 +52,7 @@
#include <DebugStream.h>
#include <snprintf.h>
#include <InstallationFunctions.h>
#include <StackTimer.h>
#include <visitstream.h>
#include <VisItException.h>
......@@ -82,10 +79,15 @@ avtNamedSelectionManager::avtNamedSelectionManager(void)
// Programmer: Hank Childs
// Creation: January 30, 2009
//
// Modifications:
// Brad Whitlock, Tue Sep 6 16:00:38 PDT 2011
// Clear the cache.
//
// ****************************************************************************
avtNamedSelectionManager::~avtNamedSelectionManager()
{
cache.clear();
}
......@@ -166,6 +168,9 @@ avtNamedSelectionManager::MaximumSelectionSize()
// I fixed a memory corruption problem that caused bad selections to be
// generated in parallel.
//
// Brad Whitlock, Tue Sep 6 16:01:52 PDT 2011
// I moved most of the code to do the selection into the extension.
//
// ****************************************************************************
void
......@@ -174,6 +179,7 @@ avtNamedSelectionManager::CreateNamedSelection(avtDataObject_p dob,
{
int i;
const char *mName = "avtNamedSelectionManager::CreateNamedSelection: ";
StackTimer t0("CreateNamedSelection");
if (strcmp(dob->GetType(), "avtDataset") != 0)
{
......@@ -218,81 +224,13 @@ avtNamedSelectionManager::CreateNamedSelection(avtDataObject_p dob,
return;
}
bool needZoneNumbers = c1->GetDataRequest()->NeedZoneNumbers() == false;
avtDataset_p ds;
if(ext != 0)
{
// Perform additional setup using the extension.
avtDataObject_p newdob = ext->GetSelectedData(dob, contract, selProps);
debug5 << mName << "Must execute the pipeline to create the named selection" << endl;
newdob->Update(contract);
debug5 << mName << "Done executing the pipeline to create the named selection" << endl;
CopyTo(ds, newdob);
}
else
{
if (needZoneNumbers)
{
debug1 << mName << "Must re-execute pipeline to create named selection" << endl;
dob->Update(contract);
debug1 << mName << "Done re-executing pipeline to create named selection" << endl;
}
CopyTo(ds, dob);
}
avtDataTree_p tree = ds->GetDataTree();
std::vector<int> doms;
std::vector<int> zones;
int nleaves = 0;
vtkDataSet **leaves = tree->GetAllLeaves(nleaves);
for (i = 0 ; i < nleaves ; i++)
{
if (leaves[i]->GetNumberOfCells() == 0)
continue;
vtkDataArray *ocn = leaves[i]->GetCellData()->
GetArray("avtOriginalCellNumbers");
if (ocn == NULL)
{
// Write an error to the logs but don't fail out since we have
// a collective communication coming up.
debug5 << mName
<< "This dataset has no original cell numbers so it cannot "
"contribute to the selection." << endl;
}
else
{
unsigned int *ptr = (unsigned int *) ocn->GetVoidPointer(0);
if (ptr == NULL)
{
// Write an error to the logs but don't fail out since we have
// a collective communication coming up.
debug5 << mName
<< "This dataset has no original cell numbers so it "
"cannot contribute to the selection." << endl;
}
else
{
// We have original cell numbers so add them to the selection.
int ncells = leaves[i]->GetNumberOfCells();
int curSize = doms.size();
doms.resize(curSize+ncells);
zones.resize(curSize+ncells);
for (int j = 0 ; j < ncells ; j++)
{
doms[curSize+j] = ptr[2*j];
zones[curSize+j] = ptr[2*j+1];
}
}
}
}
delete [] leaves;
// Let the extension free its resources. (We could just do this later...)
if(ext != 0)
ext->FreeUpResources();
//
// Call into the extension to get the domains and zones that make up the selection.
//
std::vector<int> doms, zones;
TimedCodeBlock("Creating selection in extension",
ext->GetSelection(dob, selProps, cache, doms, zones);
);
// Note the poor use of MPI below, coded for expediency, as I believe all
// of the named selections will be small.
......@@ -661,3 +599,37 @@ avtNamedSelectionManager::AddSelectionProperties(const SelectionProperties &srcp
else
properties.push_back(srcp);
}
// ****************************************************************************
// Method: avtNamedSelectionManager::ClearCache
//
// Purpose:
// Clear the cache.
//
// Programmer: Brad Whitlock
// Creation: Wed Sep 7 13:32:46 PDT 2011
//
// Modifications:
//
// ****************************************************************************
void
avtNamedSelectionManager::ClearCache(const std::string &selName)
{
if(selName.empty())
cache.clear();
else
{
avtNamedSelectionCache::iterator it = cache.begin();
while(it != cache.end())
{
if(it->second->properties.GetName() == selName)
{
cache.remove(it->first);
it = cache.begin();
}
else
it++;
}
}
}
......@@ -44,12 +44,13 @@
#include <vector>
#include <avtDataObject.h>
#include <avtNamedSelection.h>
#include <avtNamedSelectionExtension.h>
#include <MRUCache.h>
#include <SelectionProperties.h>
#include <visitstream.h>
class avtNamedSelection;
class avtNamedSelectionExtension;
// ****************************************************************************
// Class: avtNamedSelectionManager
//
......@@ -78,6 +79,9 @@ class avtNamedSelectionExtension;
// defined. These properties can be queried in the event that we need to
// create a selection using some other method.
//
// Brad Whitlock, Tue Sep 6 15:15:53 PDT 2011
// I added a cache.
//
// ****************************************************************************
class PIPELINE_API avtNamedSelectionManager
......@@ -97,10 +101,12 @@ class PIPELINE_API avtNamedSelectionManager
avtNamedSelectionExtension *);
void DeleteNamedSelection(const std::string &,
bool expectThisSelToBeThere = true);
bool expectThisSelToBeThere);
bool LoadNamedSelection(const std::string &, bool = false);
void SaveNamedSelection(const std::string &, bool = false);
void ClearCache(const std::string &selName = std::string());
const SelectionProperties *GetSelectionProperties(const std::string &selName) const;
static int MaximumSelectionSize();
......@@ -112,6 +118,8 @@ class PIPELINE_API avtNamedSelectionManager
void AddSelectionProperties(const SelectionProperties &);
std::vector<SelectionProperties> properties;
avtNamedSelectionCache cache;
private:
// These methods are defined to prevent accidental use of bitwise copy
// implementations. If you want to re-define them to do something
......
......@@ -60,7 +60,10 @@ class CQFilter;
// Creation: Mon Dec 13 15:56:56 PST 2010
//
// Modifications:
//
// Brad Whitlock, Tue Sep 6 15:27:34 PDT 2011
// I changed the API so we could add caching support that lets setting
// query attributes sometimes bypass filter execution.
//
// ****************************************************************************
class CumulativeQueryNamedSelectionExtension : public avtNamedSelectionExtension
......@@ -68,13 +71,19 @@ class CumulativeQueryNamedSelectionExtension : public avtNamedSelectionExtension
public:
CumulativeQueryNamedSelectionExtension();
virtual ~CumulativeQueryNamedSelectionExtension();
virtual avtDataObject_p GetSelectedData(avtDataObject_p dob,
avtContract_p contract,
const SelectionProperties &props);
virtual void FreeUpResources();
virtual void GetSelection(avtDataObject_p dob, const SelectionProperties &props,
avtNamedSelectionCache &cache,
std::vector<int> &doms, std::vector<int> &zones);
const SelectionSummary &GetSelectionSummary() const;
private:
std::string CreateSelectionKey(const SelectionProperties &props) const;
bool CheckProperties(const SelectionProperties &newProps,
const SelectionProperties &oldProps) const;
avtDataObject_p AddFilters(avtDataObject_p dob, const SelectionProperties &props);
SelectionSummary BuildSummary();
CQHistogramCalculationFilter *hist;
CQFilter *cqFilter;
avtThresholdFilter *threshold;
......
......@@ -59,6 +59,7 @@
#include <avtColorTables.h>
#include <avtDatabaseFactory.h>
#include <avtDataObjectQuery.h>
#include <avtNamedSelectionManager.h>
#include <avtNullData.h>
......@@ -1604,6 +1605,9 @@ RPCExecutor<SimulationCommandRPC>::Execute(SimulationCommandRPC *rpc)
// Brad Whitlock, Mon Aug 22 10:21:17 PDT 2011
// I changed how named selections get applied.
//
// Brad Whitlock, Wed Sep 7 14:33:01 PDT 2011
// I added NS_UPDATE and code to clear the NSM's cache.
//
// ****************************************************************************
template<>
void
......@@ -1631,10 +1635,19 @@ RPCExecutor<NamedSelectionRPC>::Execute(NamedSelectionRPC *rpc)
switch (rpc->GetNamedSelectionOperation())
{
case NamedSelectionRPC::NS_CREATE:
avtNamedSelectionManager::GetInstance()->ClearCache(rpc->GetSelectionName());
summary = netmgr->CreateNamedSelection(
rpc->GetPlotID(), rpc->GetSelectionProperties());
break;
case NamedSelectionRPC::NS_UPDATE:
if(!rpc->GetAllowCache())
avtNamedSelectionManager::GetInstance()->ClearCache(rpc->GetSelectionName());
netmgr->DeleteNamedSelection(rpc->GetSelectionName());
summary = netmgr->CreateNamedSelection(
rpc->GetPlotID(), rpc->GetSelectionProperties());
break;
case NamedSelectionRPC::NS_DELETE:
avtNamedSelectionManager::GetInstance()->ClearCache(rpc->GetSelectionName());
netmgr->DeleteNamedSelection(rpc->GetSelectionName());
break;
case NamedSelectionRPC::NS_LOAD:
......
......@@ -391,7 +391,11 @@ NetworkManager::GetPlotPluginManager() const
// Hank Childs, Thu Mar 2 10:06:33 PST 2006
// Added support for image based plots.
//
// Brad Whitlock, Wed Sep 7 13:31:27 PDT 2011
// Tell the NSM to clear its cache.
//
// ****************************************************************************
void
NetworkManager::ClearAllNetworks(void)
{
......@@ -426,6 +430,8 @@ NetworkManager::ClearAllNetworks(void)
}
viswinMap.clear();
NewVisWindow(0);
avtNamedSelectionManager::GetInstance()->Clea