Skip to content
GitLab
Commit 177edc5e authored by Zach's avatar Zach
Browse files

Fixed ctest -N segfault issue. Further refactored ctest. Enabled failover for ctest

parent fdc0d977
Hide whitespace changes
Inline Side-by-side
Source/CTest/cmCTestGenericHandler.cxx
View file @ 177edc5e
......@@ -166,12 +166,6 @@ bool cmCTestGenericHandler::StartLogFile(const char* name,
ostr << "_" << this->CTest->GetCurrentTag();
}
ostr << ".log";
// if this is a parallel subprocess then add the id to the
// file so they don't clobber each other
if(this->CTest->GetParallelSubprocess())
{
ostr << "." << this->CTest->GetParallelSubprocessId();
}
if( !this->CTest->OpenOutputFile("Temporary", ostr.str().c_str(), xofs) )
{
cmCTestLog(this->CTest, ERROR_MESSAGE, "Cannot create log file: "
......
Source/CTest/cmCTestMultiProcessHandler.cxx
View file @ 177edc5e
......@@ -18,7 +18,8 @@
#include "cmProcess.h"
#include "cmStandardIncludes.h"
#include "cmCTest.h"
#include "cmSystemTools.h"
#include <stdlib.h>
cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
{
......@@ -39,6 +40,7 @@ cmCTestMultiProcessHandler::SetTests(TestMap& tests,
this->Tests = tests;
this->Properties = properties;
}
// Set the max number of tests that can be run at the same time.
void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
{
......@@ -47,6 +49,7 @@ void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
void cmCTestMultiProcessHandler::RunTests()
{
this->CheckResume();
this->StartNextTests();
while(this->Tests.size() != 0)
{
......@@ -57,6 +60,7 @@ void cmCTestMultiProcessHandler::RunTests()
while(this->CheckOutput())
{
}
this->MarkFinished();
}
void cmCTestMultiProcessHandler::StartTestProcess(int test)
......@@ -176,7 +180,6 @@ bool cmCTestMultiProcessHandler::CheckOutput()
finished.push_back(p);
}
}
for( std::vector<cmCTestRunTest*>::iterator i = finished.begin();
i != finished.end(); ++i)
{
......@@ -199,12 +202,30 @@ bool cmCTestMultiProcessHandler::CheckOutput()
this->TestFinishMap[test] = true;
this->TestRunningMap[test] = false;
this->RunningTests.erase(p);
this->WriteCheckpoint(test);
delete p;
}
return true;
}
void cmCTestMultiProcessHandler::WriteCheckpoint(int index)
{
std::string fname = this->CTest->GetBinaryDir()
+ "/Testing/Temporary/CTestCheckpoint.txt";
std::fstream fout;
fout.open(fname.c_str(), std::ios::app);
fout << index << "\n";
fout.close();
}
void cmCTestMultiProcessHandler::MarkFinished()
{
std::string fname = this->CTest->GetBinaryDir()
+ "/Testing/Temporary/CTestCheckpoint.txt";
cmSystemTools::RemoveFile(fname.c_str());
}
void cmCTestMultiProcessHandler::PrintTests()
{
#undef cout
......@@ -221,6 +242,48 @@ void cmCTestMultiProcessHandler::PrintTests()
}
}
//----------------------------------------------------------------
void cmCTestMultiProcessHandler::CheckResume()
{
std::string fname = this->CTest->GetBinaryDir()
+ "/Testing/Temporary/CTestCheckpoint.txt";
if(this->CTest->GetFailover())
{
if(cmSystemTools::FileExists(fname.c_str(), true))
{
*this->TestHandler->LogFile << "Resuming previously interrupted test set"
<< std::endl
<< "----------------------------------------------------------"
<< std::endl;
std::ifstream fin;
fin.open(fname.c_str());
std::string line;
while(std::getline(fin, line))
{
int index = atoi(line.c_str());
this->RemoveTest(index);
}
fin.close();
}
}
else
{
if(cmSystemTools::FileExists(fname.c_str(), true))
{
cmSystemTools::RemoveFile(fname.c_str());
}
}
}
void cmCTestMultiProcessHandler::RemoveTest(int index)
{
this->Tests.erase(index);
this->Properties.erase(index);
this->TestRunningMap[index] = false;
this->TestFinishMap[index] = true;
}
#if 0
int main()
{
......
Source/CTest/cmCTestMultiProcessHandler.h
View file @ 177edc5e
......@@ -68,10 +68,16 @@ protected:
void StartNextTests();
void StartTestProcess(int test);
bool StartTest(int test);
//void EndTest(cmProcess*);
// Mark the checkpoint for the given test
void WriteCheckpoint(int index);
// Removes the checkpoint file
void MarkFinished();
// Return true if there are still tests running
// check all running processes for output and exit case
bool CheckOutput();
void RemoveTest(int index);
//Check if we need to resume an interrupted test set
void CheckResume();
// map from test number to set of depend tests
TestMap Tests;
//list of test properties (indices concurrent to the test map)
......@@ -79,8 +85,6 @@ protected:
std::map<int, bool> TestRunningMap;
std::map<int, bool> TestFinishMap;
std::map<int, cmStdString> TestOutput;
//std::string CTestCommand;
//std::string CTestCacheFile;
std::vector<cmStdString>* Passed;
std::vector<cmStdString>* Failed;
std::vector<cmCTestTestHandler::cmCTestTestResult>* TestResults;
......
Source/CTest/cmCTestRunTest.cxx
View file @ 177edc5e
......@@ -67,120 +67,118 @@ bool cmCTestRunTest::EndTest()
bool passed = true;
int res = this->TestProcess->GetProcessStatus();
int retVal = this->TestProcess->GetExitValue();
if ( !this->CTest->GetShowOnly() )
std::vector<std::pair<cmsys::RegularExpression,
std::string> >::iterator passIt;
bool forceFail = false;
if ( this->TestProperties->RequiredRegularExpressions.size() > 0 )
{
std::vector<std::pair<cmsys::RegularExpression,
std::string> >::iterator passIt;
bool forceFail = false;
if ( this->TestProperties->RequiredRegularExpressions.size() > 0 )
bool found = false;
for ( passIt = this->TestProperties->RequiredRegularExpressions.begin();
passIt != this->TestProperties->RequiredRegularExpressions.end();
++ passIt )
{
bool found = false;
for ( passIt = this->TestProperties->RequiredRegularExpressions.begin();
passIt != this->TestProperties->RequiredRegularExpressions.end();
++ passIt )
{
if ( passIt->first.find(this->ProcessOutput.c_str()) )
{
found = true;
reason = "Required regular expression found.";
}
}
if ( !found )
{
reason = "Required regular expression not found.";
forceFail = true;
}
reason += "Regex=[";
for ( passIt = this->TestProperties->RequiredRegularExpressions.begin();
passIt != this->TestProperties->RequiredRegularExpressions.end();
++ passIt )
if ( passIt->first.find(this->ProcessOutput.c_str()) )
{
reason += passIt->second;
reason += "\n";
found = true;
reason = "Required regular expression found.";
}
reason += "]";
}
if ( this->TestProperties->ErrorRegularExpressions.size() > 0 )
if ( !found )
{
reason = "Required regular expression not found.";
forceFail = true;
}
reason += "Regex=[";
for ( passIt = this->TestProperties->RequiredRegularExpressions.begin();
passIt != this->TestProperties->RequiredRegularExpressions.end();
++ passIt )
{
for ( passIt = this->TestProperties->ErrorRegularExpressions.begin();
passIt != this->TestProperties->ErrorRegularExpressions.end();
++ passIt )
{
if ( passIt->first.find(this->ProcessOutput.c_str()) )
{
reason = "Error regular expression found in output.";
reason += " Regex=[";
reason += passIt->second;
reason += "]";
forceFail = true;
}
}
reason += passIt->second;
reason += "\n";
}
if (res == cmsysProcess_State_Exited)
reason += "]";
}
if ( this->TestProperties->ErrorRegularExpressions.size() > 0 )
{
for ( passIt = this->TestProperties->ErrorRegularExpressions.begin();
passIt != this->TestProperties->ErrorRegularExpressions.end();
++ passIt )
{
bool success =
!forceFail && (retVal == 0 ||
this->TestProperties->RequiredRegularExpressions.size());
if((success && !this->TestProperties->WillFail)
|| (!success && this->TestProperties->WillFail))
if ( passIt->first.find(this->ProcessOutput.c_str()) )
{
this->TestResult.Status = cmCTestTestHandler::COMPLETED;
cmCTestLog(this->CTest, HANDLER_OUTPUT, " Passed " );
}
else
{
this->TestResult.Status = cmCTestTestHandler::FAILED;
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Failed " << reason );
reason = "Error regular expression found in output.";
reason += " Regex=[";
reason += passIt->second;
reason += "]";
forceFail = true;
}
}
else if ( res == cmsysProcess_State_Expired )
}
if (res == cmsysProcess_State_Exited)
{
bool success =
!forceFail && (retVal == 0 ||
this->TestProperties->RequiredRegularExpressions.size());
if((success && !this->TestProperties->WillFail)
|| (!success && this->TestProperties->WillFail))
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Timeout");
this->TestResult.Status = cmCTestTestHandler::TIMEOUT;
this->TestResult.Status = cmCTestTestHandler::COMPLETED;
cmCTestLog(this->CTest, HANDLER_OUTPUT, " Passed " );
}
else if ( res == cmsysProcess_State_Exception )
else
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Exception: ");
switch ( retVal )
{
case cmsysProcess_Exception_Fault:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "SegFault");
this->TestResult.Status = cmCTestTestHandler::SEGFAULT;
break;
case cmsysProcess_Exception_Illegal:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Illegal");
this->TestResult.Status = cmCTestTestHandler::ILLEGAL;
break;
case cmsysProcess_Exception_Interrupt:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Interrupt");
this->TestResult.Status = cmCTestTestHandler::INTERRUPT;
break;
case cmsysProcess_Exception_Numerical:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Numerical");
this->TestResult.Status = cmCTestTestHandler::NUMERICAL;
break;
default:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Other");
this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT;
}
this->TestResult.Status = cmCTestTestHandler::FAILED;
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Failed " << reason );
}
else // if ( res == cmsysProcess_State_Error )
}
else if ( res == cmsysProcess_State_Expired )
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Timeout");
this->TestResult.Status = cmCTestTestHandler::TIMEOUT;
}
else if ( res == cmsysProcess_State_Exception )
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Exception: ");
switch ( retVal )
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Bad command " << res );
this->TestResult.Status = cmCTestTestHandler::BAD_COMMAND;
case cmsysProcess_Exception_Fault:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "SegFault");
this->TestResult.Status = cmCTestTestHandler::SEGFAULT;
break;
case cmsysProcess_Exception_Illegal:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Illegal");
this->TestResult.Status = cmCTestTestHandler::ILLEGAL;
break;
case cmsysProcess_Exception_Interrupt:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Interrupt");
this->TestResult.Status = cmCTestTestHandler::INTERRUPT;
break;
case cmsysProcess_Exception_Numerical:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Numerical");
this->TestResult.Status = cmCTestTestHandler::NUMERICAL;
break;
default:
cmCTestLog(this->CTest, HANDLER_OUTPUT, "Other");
this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT;
}
}
else // if ( res == cmsysProcess_State_Error )
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Bad command " << res );
this->TestResult.Status = cmCTestTestHandler::BAD_COMMAND;
}
passed = this->TestResult.Status == cmCTestTestHandler::COMPLETED;
passed = this->TestResult.Status == cmCTestTestHandler::COMPLETED;
char buf[1024];
sprintf(buf, "%6.2f sec", this->TestProcess->GetTotalTime());
cmCTestLog(this->CTest, HANDLER_OUTPUT, buf << "\n" );
if ( this->TestHandler->LogFile )
{
*this->TestHandler->LogFile << "Test time = " << buf << std::endl;
}
this->DartProcessing();
}
char buf[1024];
sprintf(buf, "%6.2f sec", this->TestProcess->GetTotalTime());
cmCTestLog(this->CTest, HANDLER_OUTPUT, buf << "\n" );
if ( this->TestHandler->LogFile )
{
*this->TestHandler->LogFile << "Test time = " << buf << std::endl;
}
this->DartProcessing();
// if this is doing MemCheck then all the output needs to be put into
// Output since that is what is parsed by cmCTestMemCheckHandler
if(!this->TestHandler->MemCheck)
......@@ -303,14 +301,10 @@ bool cmCTestRunTest::StartTest()
this->StartTime = this->CTest->CurrentTime();
if ( !this->CTest->GetShowOnly() )
{
return this->CreateProcess(this->ActualCommand,
this->TestProperties->Args,
this->TestProperties->Timeout,
&this->TestProperties->Environment);
}
return true;
return this->CreateProcess(this->ActualCommand,
this->TestProperties->Args,
this->TestProperties->Timeout,
&this->TestProperties->Environment);
}
//----------------------------------------------------------------------
......@@ -449,15 +443,7 @@ void cmCTestRunTest::WriteLogOutputTop()
*this->TestHandler->LogFile
<< this->ProcessOutput.c_str() << "<end of output>" << std::endl;
if ( this->CTest->GetShowOnly() )
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, outname.c_str() << std::endl);
}
else
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, outname.c_str());
}
cmCTestLog(this->CTest, HANDLER_OUTPUT, outname.c_str());
cmCTestLog(this->CTest, DEBUG, "Testing "
<< this->TestProperties->Name.c_str() << " ... ");
}
Source/CTest/cmCTestRunTest.h
View file @ 177edc5e
......@@ -63,7 +63,8 @@ protected:
std::vector<std::string> args,
double testTimeOut,
std::vector<std::string>* environment);
private:
void WriteLogOutputTop();
cmCTestTestHandler::cmCTestTestProperties * TestProperties;
//Pointer back to the "parent"; the handler that invoked this test run
cmCTestTestHandler * TestHandler;
......@@ -85,7 +86,6 @@ private:
std::string StartTime;
std::string TestCommand;
std::string ActualCommand;
void WriteLogOutputTop();
};
#endif
......
Source/CTest/cmCTestTestHandler.cxx
View file @ 177edc5e
......@@ -523,14 +523,11 @@ int cmCTestTestHandler::ProcessHandler()
}
this->TestResults.clear();
// do not output startup if this is a sub-process for parallel tests
if(!this->CTest->GetParallelSubprocess())
{
cmCTestLog(this->CTest, HANDLER_OUTPUT,
(this->MemCheck ? "Memory check" : "Test")
<< " project " << cmSystemTools::GetCurrentWorkingDirectory()
<< std::endl);
}
cmCTestLog(this->CTest, HANDLER_OUTPUT,
(this->MemCheck ? "Memory check" : "Test")
<< " project " << cmSystemTools::GetCurrentWorkingDirectory()
<< std::endl);
if ( ! this->PreProcessHandler() )
{
return -1;
......@@ -583,57 +580,49 @@ int cmCTestTestHandler::ProcessHandler()
percent = 99;
}
if(!this->CTest->GetParallelSubprocess())
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
<< static_cast<int>(percent + .5) << "% tests passed, "
<< failed.size() << " tests failed out of "
<< total << std::endl);
double totalTestTime = 0;
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
<< static_cast<int>(percent + .5) << "% tests passed, "
<< failed.size() << " tests failed out of "
<< total << std::endl);
double totalTestTime = 0;
for(cmCTestTestHandler::TestResultsVector::size_type cc = 0;
cc < this->TestResults.size(); cc ++ )
{
cmCTestTestResult *result = &this->TestResults[cc];
totalTestTime += result->ExecutionTime;
}
char realBuf[1024];
sprintf(realBuf, "%6.2f sec", (double)(clock_finish - clock_start));
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\nTotal Test time (real) = "
<< realBuf << "\n" );
char totalBuf[1024];
sprintf(totalBuf, "%6.2f sec", totalTestTime);
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\nTotal Test time (parallel) = "
<< totalBuf << "\n" );
for(cmCTestTestHandler::TestResultsVector::size_type cc = 0;
cc < this->TestResults.size(); cc ++ )
{
cmCTestTestResult *result = &this->TestResults[cc];
totalTestTime += result->ExecutionTime;
}
char realBuf[1024];
sprintf(realBuf, "%6.2f sec", (double)(clock_finish - clock_start));
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\nTotal Test time (real) = "
<< realBuf << "\n" );
char totalBuf[1024];
sprintf(totalBuf, "%6.2f sec", totalTestTime);
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\nTotal Test time (parallel) = "
<< totalBuf << "\n" );
if (failed.size())
{
cmGeneratedFileStream ofs;
if(!this->CTest->GetParallelSubprocess())
cmCTestLog(this->CTest, ERROR_MESSAGE, std::endl
<< "The following tests FAILED:" << std::endl);
this->StartLogFile("TestsFailed", ofs);
std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator ftit;
for(ftit = this->TestResults.begin();
ftit != this->TestResults.end(); ++ftit)
{
cmCTestLog(this->CTest, ERROR_MESSAGE, std::endl
<< "The following tests FAILED:" << std::endl);
this->StartLogFile("TestsFailed", ofs);
std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator ftit;
for(ftit = this->TestResults.begin();
ftit != this->TestResults.end(); ++ftit)
if ( ftit->Status != cmCTestTestHandler::COMPLETED )
{
if ( ftit->Status != cmCTestTestHandler::COMPLETED )
{
ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" << std::setw(3)
<< ftit->TestCount << " - "
<< ftit->Name.c_str() << " ("
<< this->GetTestStatus(ftit->Status) << ")"
<< std::endl);
}
ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" << std::setw(3)
<< ftit->TestCount << " - "
<< ftit->Name.c_str() << " ("
<< this->GetTestStatus(ftit->Status) << ")"
<< std::endl);
}
}
}
}
......@@ -808,7 +797,6 @@ void cmCTestTestHandler::ProcessOneTest(cmCTestTestProperties *it,
std::string output;
int retVal = 0;
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl
<< (this->MemCheck?"MemCheck":"Test")
<< " command: " << testCommand
......@@ -839,9 +827,7 @@ void cmCTestTestHandler::ProcessOneTest(cmCTestTestProperties *it,
it->Timeout, &it->Environment);
}
clock_finish = cmSystemTools::GetTime();
clock_finish = cmSystemTools::GetTime();
cres.ExecutionTime = (double)(clock_finish - clock_start);
cres.FullCommandLine = testCommand;
......@@ -1118,15 +1104,7 @@ void cmCTestTestHandler::CheckLabelFilter(cmCTestTestProperties& it)
void cmCTestTestHandler::ComputeTestList()
{
this->TestList.clear(); // clear list of test
if(this->CTest->GetParallelSubprocess())
{
this->LoadTestList();
return;
}
else
{
this->GetListOfTests();
}
this->GetListOfTests();
cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
// how many tests are in based on RegExp?
int inREcnt = 0;
......@@ -1325,225 +1303,42 @@ bool cmCTestTestHandler::GetValue(const char* tag,
return ret;
}
// This should load only one test and is used in -j N mode.
// it is used by the sub-process ctest runs which should have
// only one -I N test to run.
void cmCTestTestHandler::LoadTestList()
//---------------------------------------------------------------------
void cmCTestTestHandler::PrintTestList()
{
this->TestList.clear();
std::string fname = this->CTest->GetBinaryDir()
+ "/Testing/Temporary/PCache.txt";
std::ifstream fin(fname.c_str());
std::string line;
if(!fin)
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Could not load PCache.txt file: "
<< fname.c_str() << std::endl);
return;
}