Commit e699987e authored by miller86's avatar miller86

2.11RC->Trunk: adding gzstream compression in several places

git-svn-id: http://visit.ilight.com/svn/visit/trunk/src@29534 18c085ea-50e0-402c-830e-de6fd14e8384
parent 258997d0
......@@ -1293,6 +1293,9 @@ INCLUDE(${VISIT_SOURCE_DIR}/CMake/FindTCMALLOC.cmake)
# zlib
INCLUDE(${VISIT_SOURCE_DIR}/CMake/FindZlib.cmake)
IF(ZLIB_FOUND)
SET(HAVE_LIBZ 1)
ENDIF(ZLIB_FOUND)
# szip and jpeg
IF(WIN32)
......
......@@ -44,6 +44,7 @@
#include <snprintf.h>
#include <visitstream.h>
#include <visit_gzstream.h>
#include <vtkAppendFilter.h>
#include <vtkAppendPolyData.h>
......@@ -79,6 +80,7 @@
#include <NoInputException.h>
#include <float.h>
#include <math.h>
#include <visit-config.h>
......@@ -95,6 +97,18 @@ static vtkScalarsToColors * GetColorTableFromEnv();
using std::string;
using std::vector;
#define SETUP_OFSTREAM(OFH, FNAME, QUAL, COMP) \
std::string _ext = ""; \
std::string _mode = "w"; \
if (COMP != 0) \
{ \
char levelchar = '0' + QUAL; \
char levelstr[2] = {levelchar, '\0'}; \
_ext = ".gz"; \
_mode = std::string("zwb")+std::string(levelstr); \
} \
std::string _fname = std::string(FNAME)+std::string(_ext); \
visit_ofstream ofile(_fname.c_str(), _mode.c_str())
// ****************************************************************************
// Method: avtDatasetFileWriter constructor
......@@ -174,12 +188,24 @@ avtDatasetFileWriter::~avtDatasetFileWriter()
void
avtDatasetFileWriter::Write(DatasetFileFormat format, const char *filename,
bool binary)
int quality, int compression, bool binary)
{
if (quality == 80) // GUI default
{
quality = 6; // Deflate default
}
else
{
double q = (double) quality / 100.0 * 9.0;
quality = (int) round(q);
if (quality < 1) quality = 1;
if (quality > 9) quality = 9;
}
switch (format)
{
case CURVE:
WriteCurveFile(filename);
WriteCurveFile(filename, quality, compression);
break;
case OBJ:
WriteOBJFamily(filename);
......@@ -191,7 +217,7 @@ avtDatasetFileWriter::Write(DatasetFileFormat format, const char *filename,
WriteSTLFile(filename, binary);
break;
case ULTRA:
WriteCurveFile(filename);
WriteCurveFile(filename, quality, compression);
break;
case VTK:
WriteVTKFamily(filename, binary);
......@@ -729,7 +755,7 @@ avtDatasetFileWriter::WritePLYFile(const char *filename, bool binary)
// ****************************************************************************
void
avtDatasetFileWriter::WriteCurveFile(const char *filename)
avtDatasetFileWriter::WriteCurveFile(const char *filename, int quality, int compression)
{
// We want it all in a single output file
vtkDataSet *ds = GetSingleDataset();
......@@ -755,20 +781,21 @@ avtDatasetFileWriter::WriteCurveFile(const char *filename)
std::vector< std::vector<int> > line_segments;
SortLineSegments(pd, line_segments);
ofstream ofile(filename, ios::out);
SETUP_OFSTREAM(ofile, filename, quality, compression);
vtkPoints *pts = pd->GetPoints();
for (size_t i = 0 ; i < line_segments.size() ; i++)
{
if (line_segments.size() <= 1)
ofile << "# curve" << endl;
ofile() << "# curve" << endl;
else
ofile << "# curve" << i << endl;
ofile() << "# curve" << i << endl;
ofile() << std::setprecision(16);
for (size_t j = 0 ; j < line_segments[i].size() ; j++)
{
double pt[3];
pts->GetPoint(line_segments[i][j], pt);
ofile << std::setprecision(16) << pt[0] << " " << pt[1] << endl;
ofile() << pt[0] << " " << pt[1] << endl;
}
}
......
......@@ -106,7 +106,7 @@ class AVTFILEWRITER_API avtDatasetFileWriter : public avtTerminatingDatasetSink
avtDatasetFileWriter();
virtual ~avtDatasetFileWriter();
void Write(DatasetFileFormat, const char *filename, bool);
void Write(DatasetFileFormat, const char *filename, int, int, bool);
char *CreateFilename(const char *base, bool family,
DatasetFileFormat);
......@@ -120,7 +120,7 @@ class AVTFILEWRITER_API avtDatasetFileWriter : public avtTerminatingDatasetSink
void WritePLYFile(const char *, bool);
void WriteCurveFile(const char *);
void WriteCurveFile(const char *, int, int);
void WritePOVRayFamily(const char *);
int WritePOVRayTree(avtDataTree_p, int, const char *,
......
......@@ -47,6 +47,10 @@
#include <avtCallback.h>
#include <ImproperUseException.h>
#include <FileFunctions.h>
#include <cerrno>
// ****************************************************************************
// Method: avtFileWriter constructor
//
......@@ -276,7 +280,7 @@ avtFileWriter::Write(const char *filename, avtDataObject_p dob, int quality,
else
{
dsWriter->SetInput(dob);
dsWriter->Write(dsFormat, filename, binary);
dsWriter->Write(dsFormat, filename, quality, compression, binary);
}
}
}
......@@ -357,13 +361,42 @@ avtFileWriter::CreateFilename(const char *base, bool family, bool fileChecks)
if(fileChecks)
{
bool fileExists = false;
ifstream ifile(rv);
if (!ifile.fail())
bool fileExists = true;
bool isWriteable = false;
FileFunctions::VisItStat_t statbuf;
if (FileFunctions::VisItStat(rv, &statbuf) != 0)
{
int errnum = errno;
if (errno == ENOENT)
{
fileExists = false;
}
else
{
char statmsg[512];
SNPRINTF(statmsg, sizeof(statmsg),
"VisIt encountered error \"%s\\n"
"attempting to stat file \"%s\"\n"
"prior to writing", strerror(errnum), rv);
msg = statmsg;
}
}
if (statbuf.st_mode & S_IWUSR)
{
fileExists = true;
isWriteable = true;
}
if (fileExists && family)
else
{
char wrtmsg[512];
SNPRINTF(wrtmsg, sizeof(wrtmsg),
"The file \"%s\" is not writeable\n", rv);
msg = wrtmsg;
}
if (fileExists && isWriteable && family)
{
//
// We are saving a family, so reject this one and keep going.
......@@ -373,17 +406,6 @@ avtFileWriter::CreateFilename(const char *base, bool family, bool fileChecks)
"file to avoid overwriting previous saves.";
keepGoing = true;
}
else
{
ofstream ofile(rv);
if (ofile.fail())
{
rv = NULL;
msg = "VisIt cannot write a file in the directory specified.\n"
"Note: If you are running client/server, VisIt can only "
"save files onto the local client.";
}
}
}
}
......
......@@ -324,6 +324,7 @@ utility/GeometricHelpers.C
utility/Namescheme.C
utility/StringHelpers.C
utility/Utility.C
utility/visit_gzstream.C
utility/void_ref_ptr.C
)
IF(WIN32)
......@@ -351,18 +352,18 @@ ADD_LIBRARY(visitcommon
IF(WIN32)
TARGET_LINK_LIBRARIES(visitcommon ws2_32 shlwapi userenv)
ELSE(WIN32)
TARGET_LINK_LIBRARIES(visitcommon ${CMAKE_THREAD_LIBS} ${DL_LIB})
TARGET_LINK_LIBRARIES(visitcommon ${ZLIB_LIB} ${CMAKE_THREAD_LIBS} ${DL_LIB})
ADD_EXECUTABLE(StringHelpers_test utility/StringHelpers_test.C)
TARGET_LINK_LIBRARIES(StringHelpers_test visitcommon ${CMAKE_THREAD_LIBS} ${DL_LIB})
TARGET_LINK_LIBRARIES(StringHelpers_test visitcommon ${ZLIB_LIB} ${CMAKE_THREAD_LIBS} ${DL_LIB})
ADD_EXECUTABLE(Utility_test utility/Utility_test.C)
TARGET_LINK_LIBRARIES(Utility_test visitcommon ${CMAKE_THREAD_LIBS} ${DL_LIB})
TARGET_LINK_LIBRARIES(Utility_test visitcommon ${ZLIB_LIB} ${CMAKE_THREAD_LIBS} ${DL_LIB})
ADD_EXECUTABLE(Namescheme_test utility/Namescheme_test.C)
TARGET_LINK_LIBRARIES(Namescheme_test visitcommon ${CMAKE_THREAD_LIBS} ${DL_LIB})
TARGET_LINK_LIBRARIES(Namescheme_test visitcommon ${ZLIB_LIB} ${CMAKE_THREAD_LIBS} ${DL_LIB})
ADD_EXECUTABLE(MRUCache_test utility/MRUCache_test.C)
ADD_EXECUTABLE(exprconfig expr/ExprConfig.C)
TARGET_LINK_LIBRARIES(exprconfig visitcommon ${CMAKE_THREAD_LIBS} ${DL_LIB})
TARGET_LINK_LIBRARIES(exprconfig visitcommon ${ZLIB_LIB} ${CMAKE_THREAD_LIBS} ${DL_LIB})
ADD_EXECUTABLE(exprtest expr/test.C)
TARGET_LINK_LIBRARIES(exprtest visitcommon ${CMAKE_THREAD_LIBS} ${DL_LIB})
TARGET_LINK_LIBRARIES(exprtest visitcommon ${ZLIB_LIB} ${CMAKE_THREAD_LIBS} ${DL_LIB})
ADD_CUSTOM_TARGET(init)
ADD_DEPENDENCIES(init exprconfig)
ADD_CUSTOM_COMMAND(TARGET init POST_BUILD COMMAND exprconfig ExprInit.C
......
// ============================================================================
// gzstream, C++ iostream classes wrapping the zlib compression library.
// Copyright (C) 2001 Deepak Bandyopadhyay, Lutz Kettner
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
// ============================================================================
//
// File : gzstream.C
// Revision : $Revision: 1.7 $
// Revision_date : $Date: 2003/01/08 14:41:27 $
// Author(s) : Deepak Bandyopadhyay, Lutz Kettner
//
// Standard streambuf implementation following Nicolai Josuttis, "The
// Standard C++ Library".
// ============================================================================
#include "visit_gzstream.h"
#ifdef HAVE_LIBZ
#include <iostream>
#include <string.h> // for memcpy
// ----------------------------------------------------------------------------
// Internal classes to implement gzstream. See header file for user classes.
// ----------------------------------------------------------------------------
// --------------------------------------
// class gzstreambuf:
// --------------------------------------
gzstreambuf* gzstreambuf::open(char const *name, char const *_mode) {
if (is_open())
return (gzstreambuf*)0;
file = gzopen(name, _mode);
if (file == 0)
return (gzstreambuf*)0;
strncpy(mode, _mode, sizeof(mode)-1);
opened = 1;
return this;
}
gzstreambuf * gzstreambuf::close() {
if ( is_open()) {
sync();
opened = 0;
if ( gzclose( file) == Z_OK)
return this;
}
return (gzstreambuf*)0;
}
int gzstreambuf::underflow() { // used for input buffer only
if ( gptr() && ( gptr() < egptr()))
return * reinterpret_cast<unsigned char *>( gptr());
if ( ! strchr(mode,'r') || ! opened)
return EOF;
// Josuttis' implementation of inbuf
int n_putback = gptr() - eback();
if ( n_putback > 4)
n_putback = 4;
memcpy( buffer + (4 - n_putback), gptr() - n_putback, n_putback);
int num = gzread( file, buffer+4, bufferSize-4);
if (num <= 0) // ERROR or EOF
return EOF;
// reset buffer pointers
setg( buffer + (4 - n_putback), // beginning of putback area
buffer + 4, // read position
buffer + 4 + num); // end of buffer
// return next character
return * reinterpret_cast<unsigned char *>( gptr());
}
int gzstreambuf::flush_buffer() {
// Separate the writing of the buffer from overflow() and
// sync() operation.
int w = pptr() - pbase();
if ( gzwrite( file, pbase(), w) != w)
return EOF;
pbump( -w);
return w;
}
int gzstreambuf::overflow( int c) { // used for output buffer only
if ( ! (strchr(mode,'w') || strchr(mode,'a')) || ! opened)
return EOF;
if (c != EOF) {
*pptr() = c;
pbump(1);
}
if ( flush_buffer() == EOF)
return EOF;
return c;
}
int gzstreambuf::sync() {
// Changed to use flush_buffer() instead of overflow( EOF)
// which caused improper behavior with std::endl and flush(),
// bug reported by Vincent Ricard.
if ( pptr() && pptr() > pbase()) {
if ( flush_buffer() == EOF)
return -1;
}
return 0;
}
// --------------------------------------
// class gzstreambase:
// --------------------------------------
gzstreambase::gzstreambase(char const *name, char const *_mode) {
init( &buf);
open(name, _mode);
}
gzstreambase::~gzstreambase() {
buf.close();
}
void gzstreambase::open(char const *name, char const *_mode) {
if ( ! buf.open( name, _mode))
clear( rdstate() | std::ios::badbit);
}
void gzstreambase::close() {
if ( buf.is_open())
if ( ! buf.close())
clear( rdstate() | std::ios::badbit);
}
#endif // HAVE_LIBZ
// ============================================================================
// EOF //
This diff is collapsed.
......@@ -5,6 +5,10 @@
*.ultra
*.ult
*.u
*.curve.gz
*.ultra.gz
*.ult.gz
*.u.gz
</FilePatterns>
<Attribute name="" purpose="" persistent="false">
</Attribute>
......
......@@ -92,3 +92,19 @@ Curve2DCommonPluginInfo::SetupDatabase(const char *const *list,
= new avtSTSDFileFormatInterface(ffl, nTimestep, nBlock);
return new avtGenericDatabase(inter);
}
// ****************************************************************************
// Method: Curve2DCommonPluginInfo::GetLicense
//
// Purpose:
// Gets the write options.
//
// Programmer: generated by xml2info
// Creation: omitted
//
// ****************************************************************************
std::string
Curve2DCommonPluginInfo::GetLicense() const
{
return std::string();
}
......@@ -157,6 +157,10 @@ Curve2DGeneralPluginInfo::GetDefaultFilePatterns() const
defaultPatterns.push_back("*.ultra");
defaultPatterns.push_back("*.ult");
defaultPatterns.push_back("*.u");
defaultPatterns.push_back("*.curve.gz");
defaultPatterns.push_back("*.ultra.gz");
defaultPatterns.push_back("*.ult.gz");
defaultPatterns.push_back("*.u.gz");
return defaultPatterns;
}
......
......@@ -82,6 +82,7 @@ class Curve2DCommonPluginInfo : public virtual CommonDatabasePluginInfo, public
virtual DatabaseType GetDatabaseType();
virtual avtDatabase *SetupDatabase(const char * const *list,
int nList, int nBlock);
virtual std::string GetLicense() const;
};
class Curve2DMDServerPluginInfo : public virtual MDServerDatabasePluginInfo, public virtual Curve2DCommonPluginInfo
......
......@@ -60,6 +60,8 @@
#include <InvalidFilesException.h>
#include <InvalidVariableException.h>
#include "visit_gzstream.h"
#include <errno.h>
#include <float.h>
#include <stdlib.h>
......@@ -355,9 +357,9 @@ avtCurve2DFileFormat::ReadFile(void)
{
int invalidPointCount = 0;
int lineCount = 1;
ifstream ifile(filename.c_str());
visit_ifstream ifile(filename.c_str());
if (ifile.fail())
if (ifile().fail())
{
debug1 << "Unable to open file " << filename.c_str() << endl;
return;
......@@ -380,11 +382,11 @@ avtCurve2DFileFormat::ReadFile(void)
bool justStartedNewCurve = false;
xl.reserve(1000);
yl.reserve(1000);
while (!ifile.eof())
while (!ifile().eof())
{
double x, y;
string lineName;
CurveToken t = GetPoint(ifile, x, y, lineName);
CurveToken t = GetPoint(ifile(), x, y, lineName);
switch (t)
{
case VALID_POINT:
......@@ -670,7 +672,7 @@ avtCurve2DFileFormat::ReadFile(void)
// ****************************************************************************
CurveToken
avtCurve2DFileFormat::GetPoint(ifstream &ifile, double &x, double &y, string &ln)
avtCurve2DFileFormat::GetPoint(istream &ifile, double &x, double &y, string &ln)
{
char line[256];
ifile.getline(line, 256, '\n');
......
......@@ -125,7 +125,7 @@ class avtCurve2DFileFormat : public avtSTSDFileFormat
int curveCycle;
void ReadFile(void);
CurveToken GetPoint(ifstream &, double &, double &,
CurveToken GetPoint(istream &, double &, double &,
std::string &);
};
......
......@@ -2,5 +2,6 @@
<Plugin name="Lines" label="Lines" type="database" dbtype="STMD" version="1.0">
<FilePatterns>
*.lines
*.lines.gz
</FilePatterns>
</Plugin>
......@@ -87,3 +87,19 @@ LinesCommonPluginInfo::SetupDatabase(const char *const *list,
= new avtSTMDFileFormatInterface(ffl, nList);
return new avtGenericDatabase(inter);
}
// ****************************************************************************
// Method: LinesCommonPluginInfo::GetLicense
//
// Purpose:
// Gets the write options.
//
// Programmer: generated by xml2info
// Creation: omitted
//
// ****************************************************************************
std::string
LinesCommonPluginInfo::GetLicense() const
{
return std::string();
}
......@@ -154,6 +154,7 @@ LinesGeneralPluginInfo::GetDefaultFilePatterns() const
{
std::vector<std::string> defaultPatterns;
defaultPatterns.push_back("*.lines");
defaultPatterns.push_back("*.lines.gz");
return defaultPatterns;
}
......
......@@ -82,6 +82,7 @@ class LinesCommonPluginInfo : public virtual CommonDatabasePluginInfo, public vi
virtual DatabaseType GetDatabaseType();
virtual avtDatabase *SetupDatabase(const char * const *list,
int nList, int nBlock);
virtual std::string GetLicense() const;
};
class LinesMDServerPluginInfo : public virtual MDServerDatabasePluginInfo, public virtual LinesCommonPluginInfo
......
......@@ -56,6 +56,8 @@
#include <StringHelpers.h>
#include <InvalidFilesException.h>
#include "visit_gzstream.h"
using std::vector;
using std::string;
......@@ -265,9 +267,9 @@ avtLinesFileFormat::PopulateDatabaseMetaData(avtDatabaseMetaData *md)
void
avtLinesFileFormat::ReadFile(void)
{
ifstream ifile(filename.c_str());
visit_ifstream ifile(filename.c_str());
if (ifile.fail())
if (ifile().fail())
{
debug1 << "Unable to open file " << filename.c_str() << endl;
return;
......@@ -282,11 +284,11 @@ avtLinesFileFormat::ReadFile(void)
vector<float> zl;
vector<int> cutoff;
string headerName = "";
while (!ifile.eof())
while (!ifile().eof())
{
float x, y, z;
string lineName;
if (GetPoint(ifile, x, y, z, lineName))
if (GetPoint(ifile(), x, y, z, lineName))
{
if (headerName.find_first_not_of(" ") != string::npos)
{
......@@ -393,7 +395,7 @@ avtLinesFileFormat::ReadFile(void)
// ****************************************************************************
bool
avtLinesFileFormat::GetPoint(ifstream &ifile, float &x, float &y, float &z,
avtLinesFileFormat::GetPoint(istream &ifile, float &x, float &y, float &z,
string &ln)
{
char line[256];
......
......@@ -89,7 +89,7 @@ class avtLinesFileFormat : public avtSTMDFileFormat
std::vector<std::string> lineNames;
void ReadFile(void);
bool GetPoint(ifstream &, float &, float &, float &,
bool GetPoint(istream &, float &, float &, float &,
std::string &);
};
......
......@@ -4,6 +4,9 @@
*.csv
*.tsv
*.txt
*.csv.gz
*.tsv.gz
*.txt.gz
</FilePatterns>
<Attribute name="" purpose="" persistent="true" keyframe="true" exportAPI="" exportInclude="">
</Attribute>
......
......@@ -126,3 +126,19 @@ PlainTextCommonPluginInfo::GetWriteOptions() const
{
return GetPlainTextWriteOptions();
}
// ****************************************************************************
// Method: PlainTextCommonPluginInfo::GetLicense
//
// Purpose:
// Gets the write options.
//
// Programmer: generated by xml2info
// Creation: omitted
//
// ****************************************************************************
std::string
PlainTextCommonPluginInfo::GetLicense() const
{
return std::string();
}
......@@ -156,6 +156,9 @@ PlainTextGeneralPluginInfo::GetDefaultFilePatterns() const
defaultPatterns.push_back("*.csv");
defaultPatterns.push_back("*.tsv");
defaultPatterns.push_back("*.txt");
defaultPatterns.push_back("*.csv.gz");
defaultPatterns.push_back("*.tsv.gz");
defaultPatterns.push_back("*.txt.gz");
return defaultPatterns;
}
......
......@@ -84,6 +84,7 @@ class PlainTextCommonPluginInfo : public virtual CommonDatabasePluginInfo, publi
int nList, int nBlock);
virtual DBOptionsAttributes *GetReadOptions() const;
virtual DBOptionsAttributes *GetWriteOptions() const;
virtual std::string GetLicense() const;
};
class PlainTextMDServerPluginInfo : public virtual MDServerDatabasePluginInfo, public virtual PlainTextCommonPluginInfo
......
......@@ -59,6 +59,8 @@
#include <Expression.h>
#include <StringHelpers.h>
#include "visit_gzstream.h"
#include <InvalidVariableException.h>
#include <InvalidFilesException.h>
......@@ -489,8 +491,8 @@ avtPlainTextFileFormat::ReadFile()
if (fileRead)
return;
ifstream in(filename.c_str());
if (!in)
visit_ifstream in(filename.c_str());
if (!in())
EXCEPTION1(InvalidFilesException, filename.c_str());
// skip the first lines if asked
......@@ -499,7 +501,7 @@ avtPlainTextFileFormat::ReadFile()
memset(buff, 0, sizeof(char) * linelen);
for (int l=0; l<skipLines; l++)
{
in.getline(buff, linelen);
in().getline(buff, linelen);
if (!StringHelpers::IsPureASCII(buff, linelen))
EXCEPTION2(InvalidFilesException, filename.c_str(), "Not ASCII.");
}
......@@ -509,7 +511,7 @@ avtPlainTextFileFormat::ReadFile()
ncolumns = 0;
nrows = 0;
bool firstRow = true;
in.getline(buff, linelen);
in().getline(buff, linelen);
if (!StringHelpers::IsPureASCII(buff, linelen))
EXCEPTION2(InvalidFilesException, filename.c_str(), "Not ASCII.");
......@@ -518,7 +520,7 @@ avtPlainTextFileFormat::ReadFile()
if (*p == ',')
comma = true;
while (!!in)
while (in())
{