avtGTCFileFormat.C 28 KB
Newer Older
hrchilds's avatar
hrchilds committed
1 2
/*****************************************************************************
*
3
* Copyright (c) 2000 - 2011, Lawrence Livermore National Security, LLC
hrchilds's avatar
hrchilds committed
4
* Produced at the Lawrence Livermore National Laboratory
5
* LLNL-CODE-442911
hrchilds's avatar
hrchilds committed
6 7
* All rights reserved.
*
8
* This file is  part of VisIt. For  details, see https://visit.llnl.gov/.  The
hrchilds's avatar
hrchilds committed
9 10 11 12 13 14 15 16 17 18
* full copyright notice is contained in the file COPYRIGHT located at the root
* of the VisIt distribution or at http://www.llnl.gov/visit/copyright.html.
*
* Redistribution  and  use  in  source  and  binary  forms,  with  or  without
* modification, are permitted provided that the following conditions are met:
*
*  - Redistributions of  source code must  retain the above  copyright notice,
*    this list of conditions and the disclaimer below.
*  - Redistributions in binary form must reproduce the above copyright notice,
*    this  list of  conditions  and  the  disclaimer (as noted below)  in  the
19 20 21
*    documentation and/or other materials provided with the distribution.
*  - Neither the name of  the LLNS/LLNL nor the names of  its contributors may
*    be used to endorse or promote products derived from this software without
hrchilds's avatar
hrchilds committed
22 23 24 25 26
*    specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT  HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR  IMPLIED WARRANTIES, INCLUDING,  BUT NOT  LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND  FITNESS FOR A PARTICULAR  PURPOSE
27 28 29
* ARE  DISCLAIMED. IN  NO EVENT  SHALL LAWRENCE  LIVERMORE NATIONAL  SECURITY,
* LLC, THE  U.S.  DEPARTMENT OF  ENERGY  OR  CONTRIBUTORS BE  LIABLE  FOR  ANY
* DIRECT,  INDIRECT,   INCIDENTAL,   SPECIAL,   EXEMPLARY,  OR   CONSEQUENTIAL
hrchilds's avatar
hrchilds committed
30 31 32 33 34 35 36 37 38
* DAMAGES (INCLUDING, BUT NOT  LIMITED TO, PROCUREMENT OF  SUBSTITUTE GOODS OR
* SERVICES; LOSS OF  USE, DATA, OR PROFITS; OR  BUSINESS INTERRUPTION) HOWEVER
* CAUSED  AND  ON  ANY  THEORY  OF  LIABILITY,  WHETHER  IN  CONTRACT,  STRICT
* LIABILITY, OR TORT  (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY  WAY
* OUT OF THE  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*
*****************************************************************************/

hrchilds's avatar
hrchilds committed
39
// ************************************************************************* //
40
//                             avtGTCFileFormat.C                            //
hrchilds's avatar
hrchilds committed
41 42 43 44 45 46 47 48 49 50
// ************************************************************************* //

#include <avtGTCFileFormat.h>

#include <snprintf.h>

#include <vtkCellType.h>
#include <vtkFloatArray.h>
#include <vtkUnstructuredGrid.h>
#include <avtDatabaseMetaData.h>
51

52
#include <InvalidDBTypeException.h>
53
#include <InvalidVariableException.h>
54 55
#include <InvalidFilesException.h>
#include <NonCompliantException.h>
56

57 58 59 60 61 62 63
// Define this symbol BEFORE including hdf5.h to indicate the HDF5 code
// in this file uses version 1.6 of the HDF5 API. This is harmless for
// versions of HDF5 before 1.8 and ensures correct compilation with
// version 1.8 and thereafter. When, and if, the HDF5 code in this file
// is explicitly upgraded to the 1.8 API, this symbol should be removed.
#define H5_USE_16_API
#include <hdf5.h>
64
#include <visit-hdf5.h>
hrchilds's avatar
hrchilds committed
65 66 67

#include <DebugStream.h>

pugmire's avatar
pugmire committed
68 69 70 71
#ifdef PARALLEL
#include <mpi.h>
#include <avtParallel.h>
#endif
72

73 74 75
#include <string>
#include <vector>

hrchilds's avatar
hrchilds committed
76
// ****************************************************************************
77
//  Method: avtGTCFileFormat constructor
hrchilds's avatar
hrchilds committed
78
//
79 80
//  Programmer: pugmire -- generated by xml2avt
//  Creation:   Tue Nov 20 14:08:56 PST 2007
hrchilds's avatar
hrchilds committed
81 82 83 84
//
// ****************************************************************************

avtGTCFileFormat::avtGTCFileFormat(const char *filename)
85
    : avtSTMDFileFormat(&filename, 1)
hrchilds's avatar
hrchilds committed
86
{
pugmire's avatar
pugmire committed
87 88
    fileHandle = -1;
    particleHandle = -1;
hrchilds's avatar
hrchilds committed
89
    nVars = 0;
pugmire's avatar
pugmire committed
90 91
    nTotalPoints = 0;
    nPoints = 0;
hrchilds's avatar
hrchilds committed
92 93 94
    initialized = false;

    // Make sure that the file is in fact GTC.
pugmire's avatar
pugmire committed
95
    Initialize();
hrchilds's avatar
hrchilds committed
96 97 98 99 100 101 102 103 104 105 106
}

// ****************************************************************************
//  Method: avtGTCFileFormat::FreeUpResources
//
//  Purpose:
//      When VisIt is done focusing on a particular timestep, it asks that
//      timestep to free up any resources (memory, file descriptors) that
//      it has associated with it.  This method is the mechanism for doing
//      that.
//
107 108
//  Programmer: pugmire -- generated by xml2avt
//  Creation:   Tue Nov 20 14:08:56 PST 2007
hrchilds's avatar
hrchilds committed
109 110 111 112 113 114
//
// ****************************************************************************

void
avtGTCFileFormat::FreeUpResources(void)
{
pugmire's avatar
pugmire committed
115
    if(fileHandle >= 0)
hrchilds's avatar
hrchilds committed
116
    {
pugmire's avatar
pugmire committed
117 118 119 120 121
        H5Fclose(fileHandle);
        fileHandle = -1;
    }
    if (particleHandle >= 0)
    {
ahern's avatar
ahern committed
122 123
        H5Dclose(particleHandle);
        particleHandle = -1;
hrchilds's avatar
hrchilds committed
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
    }

    initialized = false;
}

// ****************************************************************************
// Method: avtGTCFileFormat::Initialize
//
// Purpose: 
//   Opens the file and makes some other checks.
//
// Programmer: Brad Whitlock
// Creation:   Thu Apr 27 13:48:11 PST 2006
//
// Modifications:
//   
140 141 142
//    Hank Childs, Wed Jul  9 06:34:05 PDT 2008
//    Uncomment code disabling error printing.
//
143 144 145
//    Jeremy Meredith, Thu Jan  7 15:36:19 EST 2010
//    Close all open ids when returning an exception.  
//
hrchilds's avatar
hrchilds committed
146 147 148
// ****************************************************************************

bool
pugmire's avatar
pugmire committed
149
avtGTCFileFormat::Initialize()
hrchilds's avatar
hrchilds committed
150 151 152 153 154 155
{
    const char *mName = "avtGTCFileFormat::Initialize: ";

    if(initialized)
        return true;

allens's avatar
allens committed
156 157 158
    // Init HDF5 and turn off error message printing.
    H5open();
    H5Eset_auto( NULL, NULL );
159 160 161 162

    bool err = false;

    // Check for a valid GTC file
163 164 165
    if( H5Fis_hdf5( GetFilename() ) < 0 )
      EXCEPTION1( InvalidFilesException, GetFilename() );

166 167
    if ((fileHandle = H5Fopen(GetFilename(), H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
      EXCEPTION1( InvalidFilesException, GetFilename() );
168 169
    
    if ((particleHandle = H5Dopen(fileHandle, "particle_data")) < 0)
170 171 172 173 174
    {
      H5Fclose(fileHandle);
      EXCEPTION1( InvalidFilesException, GetFilename() );
    }

175 176
    // At this point consider the file to truly be a GTC file. If
    // some other file NonCompliantExceptions will be thrown.
177

178
    // Continue as normal reporting NonCompliantExceptions
179

pugmire's avatar
pugmire committed
180 181 182 183 184 185
    //Check variable's size.
    hid_t dataspace = H5Dget_space(particleHandle);
    hsize_t dims[3];
    hid_t sid = H5Dget_space(particleHandle);
    int ndims = H5Sget_simple_extent_dims(dataspace, dims, NULL);
    if(ndims < 0 || ndims > 2)
hrchilds's avatar
hrchilds committed
186
    {
ahern's avatar
ahern committed
187 188 189 190
        debug4 << mName << "Could not determine number of dimensions" << endl;
        H5Sclose(sid);
        H5Dclose(particleHandle);
        H5Fclose(fileHandle);
allens's avatar
allens committed
191
        EXCEPTION1( InvalidVariableException, "GTC Dataset Extents - Dataset 'particle_data' has an invalid extents");
hrchilds's avatar
hrchilds committed
192
    }
pugmire's avatar
pugmire committed
193 194 195 196 197 198
    
    debug4 << mName << "Determining variable size" << endl;
    int val = H5Sget_simple_extent_dims(sid, dims, NULL);

    if(val < 0 || dims[1] < 3)
    {
ahern's avatar
ahern committed
199 200 201 202
        debug4 << mName << "Could not determine variable size" << endl;
        H5Sclose(sid);
        H5Dclose(particleHandle);
        H5Fclose(fileHandle);
allens's avatar
allens committed
203
        EXCEPTION1( InvalidVariableException, "GTC Dataset Extents - Dataset 'particle_data' has an insufficient number of variables");
pugmire's avatar
pugmire committed
204 205
    }
    H5Sclose(dataspace);
hrchilds's avatar
hrchilds committed
206

pugmire's avatar
pugmire committed
207
    debug4 << mName << "variable size (" << dims[0] << ", " << dims[1] << ")" << endl;
hrchilds's avatar
hrchilds committed
208

pugmire's avatar
pugmire committed
209 210 211 212 213 214 215 216 217 218 219
    nTotalPoints = dims[0];
    nVars = dims[1];

#ifdef PARALLEL
    nProcs = PAR_Size();
    rank = PAR_Rank();
    nPoints = nTotalPoints / nProcs;
    int remainder = nTotalPoints % nProcs;

    startOffset = rank * nPoints;
    if ( rank < remainder )
ahern's avatar
ahern committed
220
        startOffset += rank;
pugmire's avatar
pugmire committed
221
    else
ahern's avatar
ahern committed
222
        startOffset += remainder;
pugmire's avatar
pugmire committed
223 224

    if ( rank < remainder )
ahern's avatar
ahern committed
225
        nPoints++;
pugmire's avatar
pugmire committed
226 227 228 229 230 231 232
#else
    nPoints = nTotalPoints;
    startOffset = 0;
#endif
    
    initialized = true;
    
hrchilds's avatar
hrchilds committed
233 234 235 236 237 238 239 240 241 242 243
    return initialized;
}

// ****************************************************************************
//  Method: avtGTCFileFormat::PopulateDatabaseMetaData
//
//  Purpose:
//      This database meta-data object is like a table of contents for the
//      file.  By populating it, you are telling the rest of VisIt what
//      information it can request from you.
//
244 245
//  Programmer: pugmire -- generated by xml2avt
//  Creation:   Tue Nov 20 14:08:56 PST 2007
hrchilds's avatar
hrchilds committed
246 247 248 249 250 251 252 253 254 255 256 257 258
//
// ****************************************************************************

void
avtGTCFileFormat::PopulateDatabaseMetaData(avtDatabaseMetaData *md)
{
    // Add a point mesh
    std::string meshname = "particles";
    avtMeshMetaData *mmd = new avtMeshMetaData;
    mmd->name = meshname;
    mmd->spatialDimension = 3;
    mmd->topologicalDimension = 0;
    mmd->meshType = AVT_POINT_MESH;
pugmire's avatar
pugmire committed
259 260 261
#ifdef PARALLEL
    mmd->numBlocks = PAR_Size();
#endif
hrchilds's avatar
hrchilds committed
262 263 264
    md->Add(mmd);

    // Add scalar variables.
265
    for ( int i = 3; i < nVars; i++ )
hrchilds's avatar
hrchilds committed
266
    {
267
        std::string var = IndexToVarName( i );
ahern's avatar
ahern committed
268 269
        if ( var != "" )
            AddScalarVarToMetaData(md, var, meshname, AVT_NODECENT);        
hrchilds's avatar
hrchilds committed
270 271 272 273 274 275 276 277 278 279 280 281 282
    }
}


// ****************************************************************************
//  Method: avtGTCFileFormat::GetMesh
//
//  Purpose:
//      Gets the mesh associated with this file.  The mesh is returned as a
//      derived type of vtkDataSet (ie vtkRectilinearGrid, vtkStructuredGrid,
//      vtkUnstructuredGrid, etc).
//
//  Arguments:
283 284 285
//      domain      The index of the domain.  If there are NDomains, this
//                  value is guaranteed to be between 0 and NDomains-1,
//                  regardless of block origin.
hrchilds's avatar
hrchilds committed
286 287 288
//      meshname    The name of the mesh of interest.  This can be ignored if
//                  there is only one mesh.
//
289 290
//  Programmer: pugmire -- generated by xml2avt
//  Creation:   Tue Nov 20 14:08:56 PST 2007
hrchilds's avatar
hrchilds committed
291 292 293 294
//
// ****************************************************************************

vtkDataSet *
295
avtGTCFileFormat::GetMesh(int domain, const char *meshname)
hrchilds's avatar
hrchilds committed
296
{
pugmire's avatar
pugmire committed
297 298 299 300 301 302 303 304
    Initialize();
    // Allocate mesh.
    vtkPoints *points  = vtkPoints::New();
    points->SetNumberOfPoints(nPoints);
    
    vtkUnstructuredGrid  *grid = vtkUnstructuredGrid::New();
    grid->SetPoints(points);
    grid->Allocate(nPoints);
hrchilds's avatar
hrchilds committed
305

pugmire's avatar
pugmire committed
306 307 308 309 310 311 312
    //Read in particles.
    float *ptrXYZ = (float *) points->GetVoidPointer(0);
    ReadVariable( domain, 0, 3, &ptrXYZ );    

    // Set the IDs in the grid.
    vtkIdType vertID;
    for ( int i = 0; i < nPoints; i++ )
hrchilds's avatar
hrchilds committed
313
    {
ahern's avatar
ahern committed
314 315
        vertID = i;
        grid->InsertNextCell( VTK_VERTEX, 1, &vertID );
hrchilds's avatar
hrchilds committed
316 317
    }

pugmire's avatar
pugmire committed
318 319 320
    points->Delete();
    return grid;
}
hrchilds's avatar
hrchilds committed
321

pugmire's avatar
pugmire committed
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
// ****************************************************************************
// Method: avtGTCFileFormat::ReadVariable
//
// Purpose: 
//   Reads in the variables from the file. If parallel, then do data exchange.
//
//  Arguments:
//      domain      The index of the domain.
//      varIdx      Variable of index to be read.
//      varDim      Dimension of varible to be read.
//      ptrVar      pointer to buffer for variable.
//
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
//   
// ****************************************************************************
hrchilds's avatar
hrchilds committed
340

pugmire's avatar
pugmire committed
341 342 343 344 345
void
avtGTCFileFormat::ReadVariable( int domain, int varIdx, int varDim, float **ptrVar )
{
    debug5 << "Reading Variable: " << startOffset << " " << nPoints << endl;
    hid_t dataspace = H5Dget_space(particleHandle);
346

pugmire's avatar
pugmire committed
347 348 349
    //Select the Var.
    hsize_t start[2] = { startOffset, varIdx };
    hsize_t count[2] = { nPoints, varDim };
350 351
    H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, NULL, count, NULL);

pugmire's avatar
pugmire committed
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
    hsize_t dataDim[1] = {nPoints*varDim};
    hid_t memspace = H5Screate_simple(1, dataDim, NULL);
    H5Sselect_all(memspace);

    //Read the variable from file.
    float *var = new float[nPoints*varDim];
    H5Dread(particleHandle, H5T_NATIVE_FLOAT, memspace, dataspace, H5P_DEFAULT, var );
    H5Sclose(memspace);

    //Select ID
    start[0] = startOffset;
    start[1] = VarNameToIndex( "id" );
    
    count[0] = nPoints;
    count[1] = 1;    
    H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, start, NULL, count, NULL);
    
    // Read in ID.
    dataDim[0] = nPoints;
    memspace = H5Screate_simple(1, dataDim, NULL);
372 373
    H5Sselect_all(memspace);
    float *ids = new float[nPoints];
pugmire's avatar
pugmire committed
374 375
    H5Dread(particleHandle, H5T_NATIVE_FLOAT, memspace, dataspace, H5P_DEFAULT, ids );
    
376 377 378
    H5Sclose(memspace);
    H5Sclose(dataspace);

pugmire's avatar
pugmire committed
379 380 381
#ifdef PARALLEL
    ParallelReadVariable( domain, varDim, var, ids );
#endif
hrchilds's avatar
hrchilds committed
382

pugmire's avatar
pugmire committed
383 384
    //Put the variables into the right order.
    for ( int i = 0; i < nPoints; i++ )
hrchilds's avatar
hrchilds committed
385
    {
ahern's avatar
ahern committed
386 387
        int id = (int)ids[i] - startOffset - 1;
        memcpy( (void *)&((*ptrVar)[i*varDim]), (void*)&var[id*varDim], varDim*sizeof(float) );
hrchilds's avatar
hrchilds committed
388 389
    }

390
    delete [] ids;
pugmire's avatar
pugmire committed
391
    delete [] var;
hrchilds's avatar
hrchilds committed
392 393 394 395 396 397 398 399 400 401 402
}

// ****************************************************************************
//  Method: avtGTCFileFormat::GetVar
//
//  Purpose:
//      Gets a scalar variable associated with this file.  Although VTK has
//      support for many different types, the best bet is vtkFloatArray, since
//      that is supported everywhere through VisIt.
//
//  Arguments:
403 404 405
//      domain     The index of the domain.  If there are NDomains, this
//                 value is guaranteed to be between 0 and NDomains-1,
//                 regardless of block origin.
hrchilds's avatar
hrchilds committed
406 407
//      varname    The name of the variable requested.
//
408 409
//  Programmer: pugmire -- generated by xml2avt
//  Creation:   Tue Nov 20 14:08:56 PST 2007
hrchilds's avatar
hrchilds committed
410 411 412 413
//
// ****************************************************************************

vtkDataArray *
414
avtGTCFileFormat::GetVar(int domain, const char *varname)
hrchilds's avatar
hrchilds committed
415 416
{
    const char *mName = "avtGTCFileFormat::GetVar: ";
pugmire's avatar
pugmire committed
417 418

    Initialize();
hrchilds's avatar
hrchilds committed
419 420

    // Determine the variable index from the varname
421 422
    int varIdx = VarNameToIndex( varname );
    if ( varIdx == -1 )
hrchilds's avatar
hrchilds committed
423
        EXCEPTION1(InvalidVariableException, varname);
pugmire's avatar
pugmire committed
424 425
    
    debug4 << mName << "We want var " << varIdx << " for " << varname << endl;
426

pugmire's avatar
pugmire committed
427
    // Allocate the variable.
hrchilds's avatar
hrchilds committed
428
    vtkFloatArray *rv = vtkFloatArray::New();
pugmire's avatar
pugmire committed
429 430
    rv->SetNumberOfTuples(nPoints);
    float *ptrVar = (float *) rv->GetVoidPointer(0);
431
    
pugmire's avatar
pugmire committed
432 433
    ReadVariable( domain, varIdx, 1, &ptrVar );

hrchilds's avatar
hrchilds committed
434 435 436 437 438 439 440 441 442 443 444 445 446
    return rv;
}


// ****************************************************************************
//  Method: avtGTCFileFormat::GetVectorVar
//
//  Purpose:
//      Gets a vector variable associated with this file.  Although VTK has
//      support for many different types, the best bet is vtkFloatArray, since
//      that is supported everywhere through VisIt.
//
//  Arguments:
447 448 449
//      domain     The index of the domain.  If there are NDomains, this
//                 value is guaranteed to be between 0 and NDomains-1,
//                 regardless of block origin.
hrchilds's avatar
hrchilds committed
450 451
//      varname    The name of the variable requested.
//
452 453
//  Programmer: pugmire -- generated by xml2avt
//  Creation:   Tue Nov 20 14:08:56 PST 2007
hrchilds's avatar
hrchilds committed
454 455 456 457
//
// ****************************************************************************

vtkDataArray *
458
avtGTCFileFormat::GetVectorVar(int domain,const char *varname)
hrchilds's avatar
hrchilds committed
459 460 461
{
    EXCEPTION1(InvalidVariableException, varname);
}
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476

// ****************************************************************************
//  Method: IndexToVarName
//
//  Purpose:
//      Given an index, return the variable name.
//
//  Arguments:
//      idx        Index of variable.
//
//  Programmer: Dave Pugmire
//  Creation:   Tue Nov 20 14:08:56 PST 2007
//
// ****************************************************************************

477
std::string
pugmire's avatar
pugmire committed
478
avtGTCFileFormat::IndexToVarName( int idx ) const
479
{
480
    std::string var = "";
481
    if ( idx == 3 )
ahern's avatar
ahern committed
482
        var = "v_par";
483
    else if ( idx == 4 )
ahern's avatar
ahern committed
484
        var = "v_perp";
485
    else if ( idx == 5 )
ahern's avatar
ahern committed
486
        var = "weight";
487
    else if ( idx == 6 )
ahern's avatar
ahern committed
488
        var = "id";
489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
    return var;
}

// ****************************************************************************
//  Method: VarNameToIndex
//
//  Purpose:
//      Given a variable name, return the index.
//
//  Arguments:
//      idx        Index of variable.
//
//  Programmer: Dave Pugmire
//  Creation:   Tue Nov 20 14:08:56 PST 2007
//
// ****************************************************************************

pugmire's avatar
pugmire committed
506
int
507
avtGTCFileFormat::VarNameToIndex( const std::string &var ) const
508 509
{
    if ( var == "v_par" )
ahern's avatar
ahern committed
510
        return 3;
511
    else if ( var == "v_perp" )
ahern's avatar
ahern committed
512
        return 4;
513
    else if ( var == "weight" )
ahern's avatar
ahern committed
514
        return 5;
515
    else if ( var == "id" )
ahern's avatar
ahern committed
516
        return 6;
517 518 519
    
    return -1;
}
pugmire's avatar
pugmire committed
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552

#if PARALLEL

// ****************************************************************************
// Method: avtGTCFileFormat::ParallelReadVariable
//
// Purpose: 
//   Sorts the variable into bins by ID and exchanges data with other processes.
//
//  Arguments:
//      domain        Domain
//      varDim        Dimension of the variable
//      var           Variable read by this process.
//      ids           IDs for the variables.
//
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
//   
// ****************************************************************************

void
avtGTCFileFormat::ParallelReadVariable( int domain, int varDim, float *var, float *ids )
{
    // Sort the particles into bins. We need to build a package for each processor.
    int ptCnt = nTotalPoints / nProcs;
    int remainder = nTotalPoints % nProcs;

    float *myVar = var, *myIds = ids;

    parallelBuffer **particleArr = new parallelBuffer*[nProcs];
    for ( int i = 0; i < nProcs; i++ )
ahern's avatar
ahern committed
553
        particleArr[i] = new parallelBuffer( varDim+1 );
pugmire's avatar
pugmire committed
554 555 556 557 558 559 560

    BinData( varDim, particleArr, var, ids, &myVar, &myIds );
    int *dataShareMatrix = GetDataShareMatrix( particleArr );
    CommunicateData( varDim, dataShareMatrix, particleArr, &myVar, &myIds );

    //Cleanup.
    for ( int i = 0; i < nProcs; i++ )
ahern's avatar
ahern committed
561
        delete particleArr[i];
pugmire's avatar
pugmire committed
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
    delete [] particleArr;
    delete [] dataShareMatrix;    
}



// ****************************************************************************
// Method: avtGTCFileFormat::BinData
//
// Purpose: 
//   Bin the data based on ID. Data that belong on other processors is placed into
//   the parallelBuffer 'array'. Data that remains on "this" process, will be copied
//   towards the front of the var/ids array. myVarsPtr and myIdsPtr track the end of
//   this array.
//
//  Arguments:
//      dim           Dimension of the variable
//      array         Array of dynamic arrays for storing bined data.
//      var           Input data variables.
//      ids           IDs for the variables.
//      myVarsPtr     Upon exit, this points to the next location in vars where data
//                    can be stored.
//      myIdsPtr      Upon exit, this points to the next location in vars where data
//                    can be stored.      
//
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
//   
// ****************************************************************************

void
avtGTCFileFormat::BinData( int dim, parallelBuffer **array, float *vars, float *ids,
ahern's avatar
ahern committed
596
                            float **myVarsPtr, float **myIdsPtr )
pugmire's avatar
pugmire committed
597 598 599 600 601 602 603 604 605
{
    int ptCnt = nTotalPoints / nProcs;
    int remainder = nTotalPoints % nProcs;

    float *ptrIDs = ids, *ptrVars = vars, *myVars = *myVarsPtr, *myIds = *myIdsPtr;
    float *data = new float[dim+1];
    
    for ( int i = 0; i < nPoints; i++ )
    {
ahern's avatar
ahern committed
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
        int id = ((int) *ptrIDs) - 1; //Make the id 0...N-1 for the math below.
        int whichProc;

        // Determine which processor this ID goes to.
        if ( id < (remainder * (ptCnt+1)) )
            whichProc = id / (ptCnt+1);
        else
        {
            int id2 = id - (remainder * (ptCnt+1));
            whichProc = remainder + id2/ptCnt;
        }

        if ( whichProc < 0 || whichProc >= nProcs )
        {
            char str[512];
            sprintf( str, "Bad Id mapping: %d ==> %d\n", id, whichProc );
            EXCEPTION1( InvalidDBTypeException, str );
        }

        // This is our data, so copy it to the front of the var, ids arrays.
        if ( whichProc == rank )
        {
            // ID
            *myIds++ = *ptrIDs++;
            for ( int j = 0; j < dim; j++ )
                *myVars++ = *ptrVars++;
        }
        // This data isn't ours. Stash it in the buffer to be sent.        
        else   
        {
            // ID
            data[0] = *ptrIDs++;
            for ( int j = 0; j < dim; j++ )
                data[j+1] = *ptrVars++;
            array[whichProc]->Add( data );
        }
pugmire's avatar
pugmire committed
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
    }
    
    delete [] data;

    *myIdsPtr = myIds;
    *myVarsPtr = myVars;
}

// ****************************************************************************
// Method: avtGTCFileFormat::GetDataShareMatrix
//
// Purpose: 
//   This computes an NxN matrix, where M[i,j] contains the number of values that
//   processor i has that need to be sent to j.
//
//  Arguments:
//      array           Array of dynamic arrays. This contains data that must be
//                      communicated.
//
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
//   
// ****************************************************************************

int *
avtGTCFileFormat::GetDataShareMatrix( parallelBuffer **array )
{
    // Binning done, now tell everyone else how many particles we have for them.
    int *particleCnts = new int[nProcs], *gatherCnts = new int[nProcs*nProcs];
    
    for ( int i = 0; i < nProcs; i++ )
ahern's avatar
ahern committed
675
        particleCnts[i] = array[i]->Size();
pugmire's avatar
pugmire committed
676 677 678

    // Allgather is a little overkill, but there isn't much data.
    int err = MPI_Allgather( particleCnts, nProcs, MPI_INT,
ahern's avatar
ahern committed
679
                             gatherCnts, nProcs*nProcs, MPI_INT, VISIT_MPI_COMM );
pugmire's avatar
pugmire committed
680
    if ( err != MPI_SUCCESS )
ahern's avatar
ahern committed
681
        EXCEPTION1(InvalidDBTypeException, "GTC Reader: MPI_Allgather() failure." );
pugmire's avatar
pugmire committed
682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
    
    return gatherCnts;
}

// ****************************************************************************
// Method: avtGTCFileFormat::CommunicateData
//
// Purpose: 
//   Communicate the data to the owning processors via non blocking send/recv.
//
//  Arguments:
//      dim           Dimension of the variable
//      shareMatrix   Matrix with share data.
//      array         Array of dynamic arrays for storing bined data.
//      var           Input data variables.
//      ids           IDs for the variables.
//      myVarsPtr     Upon exit, this points to the next location in vars where data
//                    can be stored.
//      myIdsPtr      Upon exit, this points to the next location in vars where data
//                    can be stored.         
//
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
707 708 709
//
//  Dave Pugmire, Thu Dec 20 16:23:48 EST 2007
//  Use MPI_Waitsome instead of MPI_Waitall. Process things as they become ready.
pugmire's avatar
pugmire committed
710
//   
711 712 713 714
//    Jeremy Meredith, Thu Aug  7 13:58:44 EDT 2008
//    MPI_Request does not have a well-specified type by the MPI spec.
//    I removed its use from within a printf, since it was debugging code.
//
pugmire's avatar
pugmire committed
715 716 717
// ****************************************************************************
void
avtGTCFileFormat::CommunicateData( int dim, int *shareMatrix, parallelBuffer **array,
ahern's avatar
ahern committed
718
                                   float **myVarsPtr, float **myIdsPtr )
pugmire's avatar
pugmire committed
719 720 721 722 723
{
    int err;
    
    // Everyone knows what to expect now. Do the send/recvs via nonblocking send/recv.
    std::vector<MPI_Request> requests;
724 725
    std::vector<int> requestRank;
    int numSends = 0;
pugmire's avatar
pugmire committed
726 727
    for ( int i = 0; i < nProcs; i++ )
    {
ahern's avatar
ahern committed
728 729 730 731 732 733 734 735 736 737 738 739 740 741
        MPI_Request req;
        int sz = (dim+1) * array[i]->Size();
        if ( sz == 0 )
            continue;

        err = MPI_Isend( array[i]->Get(0), sz, MPI_FLOAT, i, rank, VISIT_MPI_COMM, &req );
        if ( err != MPI_SUCCESS )
            EXCEPTION1(InvalidDBTypeException, "GTC Reader: MPI_Isend() failure." );
        char str[512]; sprintf( str, "%d: sending to %d [%d]\n", rank, i, sz );
        debug5 << str;
        
        requests.push_back( req );
        requestRank.push_back( i );
        numSends++;
pugmire's avatar
pugmire committed
742 743 744 745 746
    }

    //Do the recvs.
    float **bufs = new float*[nProcs];
    float *myVars = *myVarsPtr, *myIds = *myIdsPtr;
747
    int numRecvs = 0;
pugmire's avatar
pugmire committed
748 749 750
    
    for ( int i = 0; i < nProcs; i++ )
    {
ahern's avatar
ahern committed
751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
        int sz = (dim+1) * shareMatrix[i*nProcs + rank];
        bufs[i] = NULL;
        if ( i == rank || sz == 0 )
            continue;

        bufs[i] = new float[sz];
        MPI_Request req;
        err = MPI_Irecv( bufs[i], sz, MPI_FLOAT, i, i, VISIT_MPI_COMM, &req );
        if ( err != MPI_SUCCESS )
            EXCEPTION1(InvalidDBTypeException, "GTC Reader: MPI_Irecv() failure." );
        
        char str[512]; sprintf( str, "%d: receiving from %d [%d]\n", rank, i, sz );
        debug5 << str;
        
        requests.push_back( req );
        requestRank.push_back( i );
        numRecvs++;
pugmire's avatar
pugmire committed
768 769
    }

770 771 772
    // Process the send/recvs as they complete.
    int numRequests = numSends+numRecvs;
    if ( numRequests > 0 )
pugmire's avatar
pugmire committed
773
    {
ahern's avatar
ahern committed
774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814
        int num, nTotalReq = numRequests, *idxArray = new int[numRequests];
        MPI_Status *statusArray = new MPI_Status[numRequests];
        
        while ( numRequests > 0 )
        {
            err = MPI_Waitsome( nTotalReq, &requests[0], &num, idxArray, statusArray );
            if ( err != MPI_SUCCESS || num == MPI_UNDEFINED )
                EXCEPTION1(InvalidDBTypeException, "GTC Reader: MPI_Waitany() failure." );

            debug5 << "Waitsome=: " << num << endl;
            for ( int i = 0; i < num; i++ )
            {
                int idx = idxArray[i];
                if ( idx < numSends )
                {
                    //Nothing to do for send.
                }
                else
                {
                    int src = requestRank[idx];
                    int cnt = shareMatrix[src*nProcs + rank];

                    //Copy ID, var.
                    float *bufPtr = bufs[src];
                    for ( int j = 0; j < cnt; j++ )
                    {
                        *myIds++ = *bufPtr++; // ID
                        for ( int k = 0; k < dim; k++ )
                            *myVars++ = *bufPtr++;
                    }
                    delete [] bufs[src];
                    bufs[src] = NULL;
                }
                requests[idx] = MPI_REQUEST_NULL;
            }
            
            numRequests -= num;
        }

        delete [] idxArray;
        delete [] statusArray;
pugmire's avatar
pugmire committed
815 816
    }

817

pugmire's avatar
pugmire committed
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
    //Cleanup.
    delete [] bufs;

    *myIdsPtr = myIds;
    *myVarsPtr = myVars;
}

// ****************************************************************************
// Method: parallelBuffer::parallelBuffer
//
// Purpose: 
//   Constructor for dynamic arrays for holding data shared across processors.
//
//  Arguments:
//      elemSz        The dimension of the tuples being held.
//  
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
//   
// ****************************************************************************

parallelBuffer::parallelBuffer( int elemSz )
{
    pArray = NULL;
    buffSize = size = 0;
    elemSize = elemSz;
}


// ****************************************************************************
// Method: parallelBuffer::~parallelBuffer
//
// Purpose: 
//   Desctructor
//
//  Arguments:
//      data     data to add.
//  
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
//   
// ****************************************************************************

parallelBuffer::~parallelBuffer()
{
    if ( pArray )
ahern's avatar
ahern committed
868
        free( pArray );
pugmire's avatar
pugmire committed
869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
    
    pArray = NULL;
    buffSize = size = 0;
}

// ****************************************************************************
// Method: parallelBuffer::AddElement
//
// Purpose: 
//   Add an element to the array, growing the array size if needed.
//
//  Arguments:
//      data     data to add.
//  
// Programmer: Dave Pugmire
// Creation:   Mon Dec  3 16:17:45 EST 2007
//
// Modifications:
//   
// ****************************************************************************

void
parallelBuffer::AddElement( float *data )
{
    // Allocate array if needed.
    if ( buffSize == 0 )
    {
ahern's avatar
ahern committed
896 897
        buffSize = 64; //Experiment to see if this is too big/too small.
        pArray = (float *)malloc( buffSize*elemSize*sizeof(float) );
pugmire's avatar
pugmire committed
898 899 900 901
    }
    // Grow array if needed.
    else if ( size == buffSize )
    {
ahern's avatar
ahern committed
902 903
        buffSize = buffSize*2;
        pArray = (float *)realloc( pArray, buffSize*elemSize*sizeof(float) );
pugmire's avatar
pugmire committed
904 905 906 907 908 909 910 911
    }

    //Copy into array.
    memcpy( &pArray[size*elemSize], data, elemSize*sizeof(float) );
    size++;
}

#endif