Commit 29be94ac authored by Patrick Avery's avatar Patrick Avery Committed by Kitware Robot
Browse files

Merge topic 'json-scene-textures'

1699ed76 For poly LODs, limit quadric clustering memory use
7d84d50e Add comments for #include's
0a0771a4 Add poly LOD writing to vtkJSONSceneExporter
23a15753 Do not re-write textures in vtkJSONSceneExporter
a3052474

 Added texture writing to vtkJSONSceneExporter

Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Acked-by: default avatarKen Martin <ken.martin@kitware.com>
Merge-request: !6313
parents 91eec535 1699ed76
Pipeline #155908 failed with stage
in 0 seconds
......@@ -21,6 +21,9 @@
#include "vtkDataSet.h"
#include "vtkDiscretizableColorTransferFunction.h"
#include "vtkExporter.h"
#include "vtkImageData.h"
#include "vtkImageResize.h"
#include "vtkJPEGWriter.h"
#include "vtkJSONDataSetWriter.h"
#include "vtkMapper.h"
#include "vtkNew.h"
......@@ -28,11 +31,12 @@
#include "vtkProp.h"
#include "vtkPropCollection.h"
#include "vtkProperty.h"
#include "vtkQuadricClustering.h"
#include "vtkRenderWindow.h"
#include "vtkRenderer.h"
#include "vtkRendererCollection.h"
#include "vtkScalarsToColors.h"
#include "vtkSmartPointer.h"
#include "vtkTexture.h"
#include "vtksys/FStream.hxx"
#include "vtksys/SystemTools.hxx"
......@@ -47,6 +51,13 @@ vtkStandardNewMacro(vtkJSONSceneExporter);
vtkJSONSceneExporter::vtkJSONSceneExporter()
{
this->FileName = nullptr;
this->WriteTextures = false;
this->WriteTextureLODs = false;
this->TextureLODsBaseSize = 100000;
this->TextureLODsBaseUrl = nullptr;
this->WritePolyLODs = false;
this->PolyLODsBaseSize = 100000;
this->PolyLODsBaseUrl = nullptr;
}
// ----------------------------------------------------------------------------
......@@ -69,9 +80,23 @@ void vtkJSONSceneExporter::WriteDataObject(ostream& os, vtkDataObject* dataObjec
// Handle Dataset
if (dataObject->IsA("vtkDataSet"))
{
std::string texturesString;
if (this->WriteTextures && actor->GetTexture())
{
// Write out the textures, add it to the textures string
texturesString += this->WriteTexture(actor->GetTexture());
}
if (this->WriteTextureLODs && actor->GetTexture())
{
// Write out the texture LODs, add it to the textures string
texturesString += this->WriteTextureLODSeries(actor->GetTexture());
}
std::string renderingSetup = this->ExtractRenderingSetup(actor);
std::string addOnMeta = renderingSetup + texturesString + "\n";
std::string dsMeta =
this->WriteDataSet(vtkDataSet::SafeDownCast(dataObject), renderingSetup.c_str());
this->WriteDataSet(vtkDataSet::SafeDownCast(dataObject), addOnMeta.c_str());
if (!dsMeta.empty())
{
os << dsMeta;
......@@ -146,13 +171,22 @@ std::string vtkJSONSceneExporter::ExtractRenderingSetup(vtkActor* actor)
<< ", " << colorToUse[2] << "],\n"
<< INDENT << " \"pointSize\": " << pointSize << ",\n"
<< INDENT << " \"opacity\": " << opacity << "\n"
<< INDENT << "}\n";
<< INDENT << "}";
return renderingConfig.str();
}
// ----------------------------------------------------------------------------
std::string vtkJSONSceneExporter::CurrentDataSetPath() const
{
std::stringstream path;
path << this->FileName << "/" << this->DatasetCount + 1;
return vtksys::SystemTools::ConvertToOutputPath(path.str());
}
// ----------------------------------------------------------------------------
std::string vtkJSONSceneExporter::WriteDataSet(vtkDataSet* dataset, const char* addOnMeta = nullptr)
{
if (!dataset)
......@@ -160,12 +194,22 @@ std::string vtkJSONSceneExporter::WriteDataSet(vtkDataSet* dataset, const char*
return "";
}
std::stringstream dsPath;
dsPath << this->FileName << "/" << (++this->DatasetCount);
std::string dsPath = this->CurrentDataSetPath();
++this->DatasetCount;
auto* polyData = vtkPolyData::SafeDownCast(dataset);
vtkSmartPointer<vtkPolyData> newDataset;
std::string polyLODsConfig;
if (this->WritePolyLODs && polyData)
{
newDataset = this->WritePolyLODSeries(polyData, polyLODsConfig);
// Write the smallest poly LOD to the vtkjs file
dataset = newDataset.Get();
}
vtkNew<vtkJSONDataSetWriter> dsWriter;
dsWriter->SetInputData(dataset);
dsWriter->SetFileName(dsPath.str().c_str());
dsWriter->SetFileName(dsPath.c_str());
dsWriter->Write();
if (!dsWriter->IsDataSetValid())
......@@ -194,6 +238,7 @@ std::string vtkJSONSceneExporter::WriteDataSet(vtkDataSet* dataset, const char*
meta << addOnMeta;
}
meta << polyLODsConfig;
meta << INDENT << "}";
return meta.str();
......@@ -254,6 +299,9 @@ void vtkJSONSceneExporter::WriteLookupTable(const char* name, vtkScalarsToColors
void vtkJSONSceneExporter::WriteData()
{
this->DatasetCount = 0;
this->TextureStrings.clear();
this->TextureLODStrings.clear();
this->FilesToZip.clear();
// make sure the user specified a FileName or FilePointer
if (this->FileName == nullptr)
......@@ -340,6 +388,354 @@ void vtkJSONSceneExporter::WriteData()
file.close();
}
namespace
{
// ----------------------------------------------------------------------------
size_t getFileSize(const std::string& path)
{
// TODO: This function gives me slightly different sizes than what my
// filesystem gives me. Find out why.
// For instance, I get 240MB for a 230MB file.
// Maybe we can say "it's close enough" for now, though...
vtksys::SystemTools::Stat_t stat_buf;
int res = vtksys::SystemTools::Stat(path, &stat_buf);
if (res < 0)
{
std::cerr << "Failed to get size of file " << path.c_str() << std::endl;
return 0;
}
return stat_buf.st_size;
}
} // end anon namespace
// ----------------------------------------------------------------------------
std::string vtkJSONSceneExporter::WriteTexture(vtkTexture* texture)
{
// If this texture has already been written, just re-use the one
// we have.
if (this->TextureStrings.find(texture) != this->TextureStrings.end())
{
return this->TextureStrings[texture];
}
std::string path = this->CurrentDataSetPath();
// Make sure it exists
if (!vtksys::SystemTools::MakeDirectory(path))
{
vtkErrorMacro(<< "Cannot create directory " << path);
return "";
}
path += "/texture.jpg";
path = vtksys::SystemTools::ConvertToOutputPath(path);
vtkSmartPointer<vtkImageData> image = texture->GetInput();
vtkNew<vtkJPEGWriter> writer;
writer->SetFileName(path.c_str());
writer->SetInputDataObject(image);
writer->Write();
const char* INDENT = " ";
std::stringstream config;
config << ",\n" << INDENT << "\"texture\": \"" << this->DatasetCount + 1 << "/texture.jpg\"";
this->TextureStrings[texture] = config.str();
return config.str();
}
// ----------------------------------------------------------------------------
std::string vtkJSONSceneExporter::WriteTextureLODSeries(vtkTexture* texture)
{
// If this texture has already been written, just re-use the one
// we have.
if (this->TextureLODStrings.find(texture) != this->TextureLODStrings.end())
{
return this->TextureLODStrings[texture];
}
std::vector<std::string> files;
std::string name = "texture";
std::string ext = ".jpg";
vtkSmartPointer<vtkImageData> image = texture->GetInput();
int dims[3];
image->GetDimensions(dims);
// Write these into the parent directory of our file.
// This next line also converts the path to unix slashes.
std::string path = vtksys::SystemTools::GetParentDirectory(this->FileName);
path += "/";
path = vtksys::SystemTools::ConvertToOutputPath(path);
while (true)
{
// Name is "<name>_<dataset_number>-<width>x<height><ext>"
// For example, "texture_1-256x256.jpg"
std::stringstream full_name;
full_name << name << "_" << std::to_string(this->DatasetCount + 1) << "-"
<< std::to_string(dims[0]) << "x" << std::to_string(dims[1]) << ext;
std::string full_path = path + full_name.str();
vtkNew<vtkJPEGWriter> writer;
writer->SetFileName(full_path.c_str());
writer->SetInputDataObject(image);
writer->Write();
files.push_back(full_name.str());
size_t size = getFileSize(full_path);
if (size <= this->TextureLODsBaseSize || (dims[0] == 1 && dims[1] == 1))
{
// We are done...
break;
}
// Shrink the image and go again
vtkNew<vtkImageResize> shrink;
shrink->SetInputData(image);
dims[0] = dims[0] > 1 ? dims[0] / 2 : 1;
dims[1] = dims[1] > 1 ? dims[1] / 2 : 1;
shrink->SetOutputDimensions(dims[0], dims[1], 1);
shrink->Update();
image = shrink->GetOutput();
}
const char* url = this->TextureLODsBaseUrl;
std::string baseUrl = url ? url : "";
// Now, write out the config
const char* INDENT = " ";
std::stringstream config;
config << ",\n"
<< INDENT << "\"textureLODs\": {\n"
<< INDENT << " \"baseUrl\": \"" << baseUrl << "\",\n"
<< INDENT << " \"files\": [\n";
// Reverse the order of the files so the smallest comes first
std::reverse(files.begin(), files.end());
for (size_t i = 0; i < files.size(); ++i)
{
config << INDENT << " \"" << files[i] << "\"";
if (i != files.size() - 1)
{
config << ",\n";
}
else
{
config << "\n";
}
}
config << INDENT << " ]\n" << INDENT << "}";
this->TextureLODStrings[texture] = config.str();
return config.str();
}
// ----------------------------------------------------------------------------
vtkSmartPointer<vtkPolyData> vtkJSONSceneExporter::WritePolyLODSeries(
vtkPolyData* dataset, std::string& polyLODsConfig)
{
vtkSmartPointer<vtkPolyData> polyData = dataset;
std::vector<std::string> files;
// Write these into the parent directory of our file.
// This next line also converts the path to unix slashes.
vtkNew<vtkJSONDataSetWriter> dsWriter;
std::string path = vtksys::SystemTools::GetParentDirectory(this->FileName) + "/";
path = vtksys::SystemTools::ConvertToOutputPath(path);
// If the new size is not at least 5% different from the old size,
// stop writing out the LODs, because the difference is too small.
size_t previousDataSize = 0;
double minDiffFraction = 0.05;
const size_t& baseSize = this->PolyLODsBaseSize;
int count = 0;
while (true)
{
// Squeeze the data, or we won't get an accurate memory size
polyData->Squeeze();
auto dataSize = polyData->GetActualMemorySize();
bool tooSimilar = false;
if (previousDataSize != 0)
{
double fraction = (static_cast<double>(previousDataSize) - dataSize) / previousDataSize;
if (fabs(fraction) < minDiffFraction)
{
tooSimilar = true;
}
}
previousDataSize = dataSize;
if (static_cast<size_t>(dataSize) * 1000 <= baseSize || tooSimilar)
{
// It is either now below the base size, or the size isn't
// changing much anymore.
// The latest "polyData" will be written into the .vtkjs directory
break;
}
// Write out the source LOD
// They are not zipped yet, but they should be zipped by subclasses
std::string name =
"sourceLOD_" + std::to_string(this->DatasetCount) + "_" + std::to_string(++count) + ".zip";
std::string full_path = path + name;
dsWriter->SetInputData(polyData);
dsWriter->SetFileName(full_path.c_str());
dsWriter->Write();
files.push_back(name);
this->FilesToZip.push_back(full_path);
// Now reduce the size of the data
double bounds[6];
polyData->GetBounds(bounds);
double length = polyData->GetLength();
double factors[3] = { (bounds[1] - bounds[0]) / length + 0.01,
(bounds[3] - bounds[2]) / length + 0.01, (bounds[5] - bounds[4]) / length + 0.01 };
double factorsCube = factors[0] * factors[1] * factors[2];
// Try to make a good first guess for B
// TODO: this first guess can probably be improved a lot.
double B = pow(100 * dataSize / factorsCube, 0.3333);
vtkNew<vtkQuadricClustering> cc;
cc->UseInputPointsOn();
cc->CopyCellDataOn();
cc->SetInputDataObject(polyData);
cc->SetAutoAdjustNumberOfDivisions(false);
// We will try to get the next size to be between 1/3 and 1/5
// of the original. The goal is to be approximately 1/4.
auto targetSize = dataSize / 4;
auto targetMin = dataSize / 5;
auto targetMax = dataSize / 3;
int maxAttempts = 100;
size_t previousSize = 0;
// If we fail to get to ~1/4 the size for some reason, just use
// the default divisions. Sometimes, a failure is caused by
// one of the factors being too big.
bool useDefaultDivisions = false;
for (int numAttempts = 0; numAttempts < maxAttempts; ++numAttempts)
{
int divs[3];
for (int i = 0; i < 3; ++i)
{
divs[i] = B * factors[i] + 1;
}
// The allocated memory is proportional to divs[0] * divs[1] * divs[2].
// Make sure this product is not too big, or we may run out of memory,
// or, worse, have an integer overflow.
// At the time of testing, a product of 1e8 requires more than 10 GB of
// memory to run. Let's set a limit here for now.
size_t maxProduct = 1e8;
if (static_cast<size_t>(divs[0]) * divs[1] * divs[2] > maxProduct)
{
// Too big. Just use the defaults.
useDefaultDivisions = true;
break;
}
cc->SetNumberOfXDivisions(divs[0]);
cc->SetNumberOfYDivisions(divs[1]);
cc->SetNumberOfZDivisions(divs[2]);
try
{
cc->Update();
}
catch (const std::bad_alloc&)
{
// Too many divisions, probably. Just use the defaults.
useDefaultDivisions = true;
break;
}
// Squeeze the data, or we won't get an accurate memory size
cc->GetOutput()->Squeeze();
auto newSize = cc->GetOutput()->GetActualMemorySize();
if (newSize == previousSize)
{
// This is not changing. Just use the default divisions.
useDefaultDivisions = true;
break;
}
previousSize = newSize;
if (newSize >= targetMin && newSize <= targetMax)
{
// We are within the tolerance!
break;
}
else
{
// Figure out the fraction that we are off, and change B
// accordingly
double fraction = newSize / static_cast<double>(targetSize);
B /= pow(fraction, 0.333);
}
}
if (useDefaultDivisions)
{
vtkNew<vtkQuadricClustering> defaultCC;
defaultCC->UseInputPointsOn();
defaultCC->CopyCellDataOn();
defaultCC->SetInputDataObject(polyData);
defaultCC->Update();
polyData = defaultCC->GetOutput();
}
else
{
polyData = cc->GetOutput();
}
}
// Write out the config
const char* url = this->PolyLODsBaseUrl;
std::string baseUrl = url ? url : "";
const char* INDENT = " ";
std::stringstream config;
config << ",\n"
<< INDENT << "\"sourceLODs\": {\n"
<< INDENT << " \"baseUrl\": \"" << baseUrl << "\",\n"
<< INDENT << " \"files\": [\n";
// Reverse the order of the files so the smallest comes first
std::reverse(files.begin(), files.end());
for (size_t i = 0; i < files.size(); ++i)
{
config << INDENT << " \"" << files[i] << "\"";
if (i != files.size() - 1)
{
config << ",\n";
}
else
{
config << "\n";
}
}
config << INDENT << " ]\n" << INDENT << "}";
polyLODsConfig = config.str();
return polyData;
}
// ----------------------------------------------------------------------------
void vtkJSONSceneExporter::PrintSelf(ostream& os, vtkIndent indent)
......
......@@ -27,14 +27,18 @@
#include "vtkExporter.h"
#include "vtkIOExportModule.h" // For export macro
#include "vtkSmartPointer.h" // For vtkSmartPointer
#include <map> // For string parameter
#include <map> // For member variables
#include <string> // For string parameter
#include <vector> // For member variables
class vtkActor;
class vtkDataObject;
class vtkDataSet;
class vtkPolyData;
class vtkScalarsToColors;
class vtkTexture;
class VTKIOEXPORT_EXPORT vtkJSONSceneExporter : public vtkExporter
{
......@@ -52,6 +56,89 @@ public:
vtkGetStringMacro(FileName);
//@}
//@{
/**
* Whether or not to write textures.
* Textures will be written in JPEG format.
* Default is false.
*/
vtkSetMacro(WriteTextures, bool);
vtkGetMacro(WriteTextures, bool);
//@}
//@{
/**
* Whether or not to write texture LODs.
* This will write out the textures in a series of decreasing
* resolution JPEG files, which are intended to be uploaded to the
* web. Each file will be 1/4 the size of the previous one. The files
* will stop being written out when one is smaller than the
* TextureLODsBaseSize.
* Default is false.
*/
vtkSetMacro(WriteTextureLODs, bool);
vtkGetMacro(WriteTextureLODs, bool);
//@}
//@{
/**
* The base size to be used for texture LODs. The texture LODs will
* stop being written out when one is smaller than this size.
* Default is 100 KB. Units are in bytes.
*/
vtkSetMacro(TextureLODsBaseSize, size_t);
vtkGetMacro(TextureLODsBaseSize, size_t);
//@}
//@{
/**
* The base URL to be used for texture LODs.
* Default is nullptr.
*/
vtkSetStringMacro(TextureLODsBaseUrl);
vtkGetStringMacro(TextureLODsBaseUrl);
//@}
//@{
/**
* Whether or not to write poly LODs.
* This will write out the poly LOD sources in a series of decreasing
* resolution data sets, which are intended to be uploaded to the
* web. vtkQuadricCluster is used to decrease the resolution of the
* poly data. Each will be approximately 1/4 the size of the previous
* one (unless certain errors occur, and then the defaults for quadric
* clustering will be used, which will produce an unknown size). The
* files will stop being written out when one is smaller than the
* PolyLODsBaseSize, or if the difference in the sizes of the two
* most recent LODs is less than 5%. The smallest LOD will be written
* into the vtkjs file, rather than with the rest of the LODs.
* Default is false.
*/
vtkSetMacro(WritePolyLODs, bool);
vtkGetMacro(WritePolyLODs, bool);
//@}
//@{
/**
* The base size to be used for poly LODs. The poly LODs will stop
* being written out when one is smaller than this size, or if the
* difference in the sizes of the two most recent LODs is less
* than 5%.
* Default is 100 KB. Units are in bytes.
*/
vtkSetMacro(PolyLODsBaseSize, size_t);
vtkGetMacro(PolyLODsBaseSize, size_t);
//@}
//@{
/**
* The base URL to be used for poly LODs.
* Default is nullptr.
*/
vtkSetStringMacro(PolyLODsBaseUrl);
vtkGetStringMacro(PolyLODsBaseUrl);
//@}
protected:
vtkJSONSceneExporter();
~vtkJSONSceneExporter() override;
......@@ -63,9 +150,30 @@ protected:
void WriteData() override;
std::string CurrentDataSetPath() const;
std::string WriteTexture(vtkTexture* texture);
std::string WriteTextureLODSeries(vtkTexture* texture);
// The returned pointer is the smallest poly LOD, intended to be
// written out in the vtkjs file.
vtkSmartPointer<vtkPolyData> WritePolyLODSeries(vtkPolyData* polys, std::string& config);
char* FileName;
bool WriteTextures;
bool WriteTextureLODs;
size_t TextureLODsBaseSize;
char* TextureLODsBaseUrl;
bool WritePolyLODs;