Commit bb7b27e4 authored by Bill Hoffman's avatar Bill Hoffman
Browse files

ENH: add initial ctest -j feature

parent ea71721b
......@@ -314,6 +314,7 @@ INCLUDE_DIRECTORIES(
# Sources for CTestLib
#
SET(CTEST_SRCS cmCTest.cxx
CTest/cmProcess.cxx
CTest/cmCTestBuildAndTestHandler.cxx
CTest/cmCTestBuildCommand.cxx
CTest/cmCTestBuildHandler.cxx
......@@ -326,6 +327,7 @@ SET(CTEST_SRCS cmCTest.cxx
CTest/cmCTestHandlerCommand.cxx
CTest/cmCTestMemCheckCommand.cxx
CTest/cmCTestMemCheckHandler.cxx
CTest/cmCTestMultiProcessHandler.cxx
CTest/cmCTestReadCustomFilesCommand.cxx
CTest/cmCTestRunScriptCommand.cxx
CTest/cmCTestScriptHandler.cxx
......@@ -458,3 +460,4 @@ IF(APPLE)
ENDIF(APPLE)
INSTALL_FILES(${CMAKE_DATA_DIR}/include cmCPluginAPI.h)
......@@ -161,6 +161,12 @@ bool cmCTestGenericHandler::StartLogFile(const char* name,
ostr << "_" << this->CTest->GetCurrentTag();
}
ostr << ".log";
// if this is a parallel subprocess then add the id to the
// file so they don't clobber each other
if(this->CTest->GetParallelSubprocess())
{
ostr << "." << this->CTest->GetParallelSubprocessId();
}
if( !this->CTest->OpenOutputFile("Temporary", ostr.str().c_str(), xofs) )
{
cmCTestLog(this->CTest, ERROR_MESSAGE, "Cannot create log file: "
......
/*=========================================================================
Program: CMake - Cross-Platform Makefile Generator
Module: $RCSfile$
Language: C++
Date: $Date$
Version: $Revision$
Copyright (c) 2002 Kitware, Inc., Insight Consortium. All rights reserved.
See Copyright.txt or http://www.cmake.org/HTML/Copyright.html for details.
This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the above copyright notices for more information.
=========================================================================*/
#include "cmCTestMultiProcessHandler.h"
#include "cmProcess.h"
#include "cmStandardIncludes.h"
#include "cmCTest.h"
cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
{
this->ParallelLevel = 1;
this->ProcessId = 0;
}
// Set the tests
void
cmCTestMultiProcessHandler::SetTests(std::map<int, std::set<int> >& tests,
std::map<int,cmStdString>& testNames)
{
// set test run map to false for all
for(std::map<int, std::set<int> >::iterator i = this->Tests.begin();
i != this->Tests.end(); ++i)
{
this->TestRunningMap[i->first] = false;
this->TestFinishMap[i->first] = false;
}
this->Tests = tests;
this->TestNames = testNames;
}
// Set the max number of tests that can be run at the same time.
void cmCTestMultiProcessHandler::SetParallelLevel(size_t l)
{
this->ParallelLevel = l;
}
void cmCTestMultiProcessHandler::RunTests()
{
this->StartNextTests();
while(this->Tests.size() != 0)
{
this->CheckOutput();
this->StartNextTests();
}
// let all running tests finish
while(this->CheckOutput())
{
}
for(std::map<int, cmStdString>::iterator i =
this->TestOutput.begin();
i != this->TestOutput.end(); ++i)
{
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
i->second << std::endl);
}
}
void cmCTestMultiProcessHandler::StartTestProcess(int test)
{
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
" test " << test << "\n");
this->TestRunningMap[test] = true; // mark the test as running
// now remove the test itself
this->Tests.erase(test);
// now run the test
cmProcess* newp = new cmProcess;
newp->SetId(this->ProcessId);
newp->SetId(test);
newp->SetCommand(this->CTestCommand.c_str());
std::vector<std::string> args;
args.push_back("-I");
cmOStringStream strm;
strm << test << "," << test;
args.push_back(strm.str());
args.push_back("--parallel-cache");
args.push_back(this->CTestCacheFile.c_str());
args.push_back("--internal-ctest-parallel");
cmOStringStream strm2;
strm2 << test;
args.push_back(strm2.str());
if(this->CTest->GetExtraVerbose())
{
args.push_back("-VV");
}
newp->SetCommandArguments(args);
if(!newp->StartProcess())
{
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Error starting " << newp->GetCommand() << "\n");
this->EndTest(newp);
}
else
{
this->RunningTests.insert(newp);
}
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"ctest -I " << test << "\n");
this->ProcessId++;
}
bool cmCTestMultiProcessHandler::StartTest(int test)
{
// copy the depend tests locally because when
// a test is finished it will be removed from the depend list
// and we don't want to be iterating a list while removing from it
std::set<int> depends = this->Tests[test];
size_t totalDepends = depends.size();
if(totalDepends)
{
for(std::set<int>::const_iterator i = depends.begin();
i != depends.end(); ++i)
{
// if the test is not already running then start it
if(!this->TestRunningMap[*i])
{
// this test might be finished, but since
// this is a copy of the depend map we might
// still have it
if(!this->TestFinishMap[*i])
{
// only start one test in this function
return this->StartTest(*i);
}
else
{
// the depend has been and finished
totalDepends--;
}
}
}
}
// if there are no depends left then run this test
if(totalDepends == 0)
{
// Start this test it has no depends
this->StartTestProcess(test);
return true;
}
// This test was not able to start because it is waiting
// on depends to run
return false;
}
void cmCTestMultiProcessHandler::StartNextTests()
{
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl
<< "Number of running tests : " << this->RunningTests.size()
<< "\n");
size_t numToStart = this->ParallelLevel - this->RunningTests.size();
if(numToStart == 0)
{
return;
}
std::map<int, std::set<int> > tests = this->Tests;
for(std::map<int, std::set<int> >::iterator i = tests.begin();
i != tests.end(); ++i)
{
// start test should start only one test
if(this->StartTest(i->first))
{
numToStart--;
}
else
{
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl
<< "Test did not start waiting on depends to finish: "
<< i->first << "\n");
}
if(numToStart == 0 )
{
return;
}
}
}
bool cmCTestMultiProcessHandler::CheckOutput()
{
// no more output we are done
if(this->RunningTests.size() == 0)
{
return false;
}
std::vector<cmProcess*> finished;
std::string out, err;
for(std::set<cmProcess*>::const_iterator i = this->RunningTests.begin();
i != this->RunningTests.end(); ++i)
{
cmProcess* p = *i;
int pipe = p->CheckOutput(.1, out, err);
if(pipe == cmsysProcess_Pipe_STDOUT)
{
cmCTestLog(this->CTest, HANDLER_OUTPUT,
p->GetId() << ": " << out << std::endl);
this->TestOutput[ p->GetId() ] += out;
this->TestOutput[ p->GetId() ] += "\n";
}
else if(pipe == cmsysProcess_Pipe_STDERR)
{
cmCTestLog(this->CTest, HANDLER_OUTPUT,
p->GetId() << ": " << err << std::endl);
this->TestOutput[ p->GetId() ] += err;
this->TestOutput[ p->GetId() ] += "\n";
}
if(!p->IsRunning())
{
finished.push_back(p);
}
}
for( std::vector<cmProcess*>::iterator i = finished.begin();
i != finished.end(); ++i)
{
cmProcess* p = *i;
this->EndTest(p);
}
return true;
}
void cmCTestMultiProcessHandler::EndTest(cmProcess* p)
{
int test = p->GetId();
int exitVal = p->GetExitValue();
cmCTestTestHandler::cmCTestTestResult cres;
cres.Properties = 0;
cres.ExecutionTime = 0;// ???
cres.ReturnValue = exitVal;
cres.Status = cmCTestTestHandler::COMPLETED;
cres.TestCount = test;
cres.Name = this->TestNames[test];
cres.Path.clear();
if(exitVal)
{
cres.Status = cmCTestTestHandler::FAILED;
this->Failed->push_back(this->TestNames[test]);
}
else
{
this->Passed->push_back(this->TestNames[test]);
}
this->TestResults->push_back(cres);
// remove test from depend of all other tests
for( std::map<int, std::set<int> >::iterator i = this->Tests.begin();
i!= this->Tests.end(); ++i)
{
i->second.erase(test);
}
this->TestFinishMap[test] = true;
this->TestRunningMap[test] = false;
this->RunningTests.erase(p);
delete p;
cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"finish test " << test << "\n");
}
void cmCTestMultiProcessHandler::PrintTests()
{
#undef cout
for( std::map<int, std::set<int> >::iterator i = this->Tests.begin();
i!= this->Tests.end(); ++i)
{
std::cout << "Test " << i->first << " (";
for(std::set<int>::iterator j = i->second.begin();
j != i->second.end(); ++j)
{
std::cout << *j << " ";
}
std::cout << ")\n";
}
}
#if 0
int main()
{
cmCTestMultiProcessHandler h;
h.SetParallelLevel(4);
std::map<int, std::set<int> > tests;
std::set<int> depends;
for(int i =1; i < 92; i++)
{
tests[i] = depends;
}
depends.clear();
depends.insert(45); subprject
tests[46] = depends; subproject-stage2
depends.clear();
depends.insert(55); simpleinstall simpleinstall-s2
tests[56] = depends;
depends.clear();
depends.insert(70); wrapping
tests[71] = depends; qtwrapping
depends.clear();
depends.insert(71); qtwrapping
tests[72] = depends; testdriver1
depends.clear();
depends.insert(72) testdriver1
tests[73] = depends; testdriver2
depends.clear();
depends.insert(73); testdriver2
tests[74] = depends; testdriver3
depends.clear();
depends.insert(79); linkorder1
tests[80] = depends; linkorder2
h.SetTests(tests);
h.PrintTests();
h.RunTests();
}
#endif
/*=========================================================================
Program: CMake - Cross-Platform Makefile Generator
Module: $RCSfile$
Language: C++
Date: $Date$
Version: $Revision$
Copyright (c) 2002 Kitware, Inc., Insight Consortium. All rights reserved.
See Copyright.txt or http://www.cmake.org/HTML/Copyright.html for details.
This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the above copyright notices for more information.
=========================================================================*/
#ifndef cmCTestMultiProcessHandler_h
#define cmCTestMultiProcessHandler_h
#include <set>
#include <map>
#include <string>
#include <vector>
class cmProcess;
#include <cmStandardIncludes.h>
#include <cmCTestTestHandler.h>
/** \class cmCTestMultiProcessHandler
* \brief run parallel ctest
*
* cmCTestMultiProcessHandler
*/
class cmCTestMultiProcessHandler
{
public:
cmCTestMultiProcessHandler();
// Set the tests
void SetTests(std::map<int, std::set<int> >& tests,
std::map<int, cmStdString>& testNames);
// Set the max number of tests that can be run at the same time.
void SetParallelLevel(size_t);
void RunTests();
void PrintTests();
void SetCTestCommand(const char* c) { this->CTestCommand = c;}
void SetTestCacheFile(const char* c) { this->CTestCacheFile = c;}
void SetPassFailVectors(std::vector<cmStdString>* passed,
std::vector<cmStdString>* failed)
{
this->Passed = passed;
this->Failed = failed;
}
void SetTestResults(std::vector<cmCTestTestHandler::cmCTestTestResult>* r)
{
this->TestResults = r;
}
void SetCTest(cmCTest* ctest) { this->CTest = ctest;}
protected:
cmCTest* CTest;
// Start the next test or tests as many as are allowed by
// ParallelLevel
void StartNextTests();
void StartTestProcess(int test);
bool StartTest(int test);
void EndTest(cmProcess*);
// Return true if there are still tests running
// check all running processes for output and exit case
bool CheckOutput();
// map from test number to set of depend tests
std::map<int, std::set<int> > Tests;
std::map<int, cmStdString> TestNames;
std::map<int, bool> TestRunningMap;
std::map<int, bool> TestFinishMap;
std::map<int, cmStdString> TestOutput;
std::string CTestCommand;
std::string CTestCacheFile;
std::vector<cmStdString>* Passed;
std::vector<cmStdString>* Failed;
std::vector<cmCTestTestHandler::cmCTestTestResult>* TestResults;
int ProcessId;
size_t ParallelLevel; // max number of process that can be run at once
std::set<cmProcess*> RunningTests; // current running tests
};
#endif
......@@ -16,7 +16,7 @@
=========================================================================*/
#include "cmCTestTestHandler.h"
#include "cmCTestMultiProcessHandler.h"
#include "cmCTest.h"
#include "cmake.h"
#include "cmGeneratedFileStream.h"
......@@ -500,11 +500,14 @@ int cmCTestTestHandler::ProcessHandler()
}
this->TestResults.clear();
cmCTestLog(this->CTest, HANDLER_OUTPUT,
(this->MemCheck ? "Memory check" : "Test")
<< " project " << cmSystemTools::GetCurrentWorkingDirectory()
<< std::endl);
// do not output startup if this is a sub-process for parallel tests
if(!this->CTest->GetParallelSubprocess())
{
cmCTestLog(this->CTest, HANDLER_OUTPUT,
(this->MemCheck ? "Memory check" : "Test")
<< " project " << cmSystemTools::GetCurrentWorkingDirectory()
<< std::endl);
}
if ( ! this->PreProcessHandler() )
{
return -1;
......@@ -517,7 +520,6 @@ int cmCTestTestHandler::ProcessHandler()
std::vector<cmStdString> passed;
std::vector<cmStdString> failed;
int total;
this->ProcessDirectory(passed, failed);
total = int(passed.size()) + int(failed.size());
......@@ -550,33 +552,37 @@ int cmCTestTestHandler::ProcessHandler()
{
percent = 99;
}
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
<< static_cast<int>(percent + .5) << "% tests passed, "
<< failed.size() << " tests failed out of " << total << std::endl);
//fprintf(stderr,"\n%.0f%% tests passed, %i tests failed out of %i\n",
// percent, int(failed.size()), total);
if(!this->CTest->GetParallelSubprocess())
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
<< static_cast<int>(percent + .5) << "% tests passed, "
<< failed.size() << " tests failed out of "
<< total << std::endl);
}
if (failed.size())
{
cmGeneratedFileStream ofs;
cmCTestLog(this->CTest, ERROR_MESSAGE, std::endl
<< "The following tests FAILED:" << std::endl);
this->StartLogFile("TestsFailed", ofs);
std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator ftit;
for(ftit = this->TestResults.begin();
ftit != this->TestResults.end(); ++ftit)
if(!this->CTest->GetParallelSubprocess())
{
if ( ftit->Status != cmCTestTestHandler::COMPLETED )
cmCTestLog(this->CTest, ERROR_MESSAGE, std::endl
<< "The following tests FAILED:" << std::endl);
this->StartLogFile("TestsFailed", ofs);
std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator ftit;
for(ftit = this->TestResults.begin();
ftit != this->TestResults.end(); ++ftit)
{
ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" << std::setw(3)
<< ftit->TestCount << " - " << ftit->Name.c_str() << " ("
<< this->GetTestStatus(ftit->Status) << ")" << std::endl);
if ( ftit->Status != cmCTestTestHandler::COMPLETED )
{
ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" << std::setw(3)
<< ftit->TestCount << " - " << ftit->Name.c_str() << " ("
<< this->GetTestStatus(ftit->Status) << ")" << std::endl);
}
}
}
}
}
......@@ -677,8 +683,8 @@ void cmCTestTestHandler::ProcessOneTest(cmCTestTestProperties *it,
// add the arguments
std::vector<std::string>::const_iterator j = args.begin();
++j;
++j;
++j; // skip test name
++j; // skip command as it is in actualCommand
std::vector<const char*> arguments;
this->GenerateTestCommand(arguments);
arguments.push_back(actualCommand.c_str());
......@@ -917,23 +923,20 @@ void cmCTestTestHandler::ProcessOneTest(cmCTestTestProperties *it,
}
//----------------------------------------------------------------------
void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
std::vector<cmStdString> &failed)
void cmCTestTestHandler::ComputeTestList()
{
std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
this->TestList.clear();
this->GetListOfTests();
this->TestList.clear(); // clear list of test
if(this->CTest->GetParallelSubprocess())
{
this->LoadTestList();
return;
}
else
{
this->GetListOfTests();
}
cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
this->StartTest = this->CTest->CurrentTime();
this->StartTestTime = static_cast<unsigned int>(cmSystemTools::GetTime());
double elapsed_time_start = cmSystemTools::GetTime();
*this->LogFile << "Start testing: " << this->StartTest << std::endl
<< "----------------------------------------------------------"
<< std::endl;
// how many tests are in based on RegExp?
int inREcnt = 0;
cmCTestTestHandler::ListOfTests::iterator it;
......@@ -953,10 +956,11 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
{
this->ExpandTestsToRunInformation(inREcnt);
}
// Now create a final list of tests to run
int cnt = 0;
inREcnt = 0;
std::string last_directory = "";
std::string last_directory = "";
ListOfTests finalList;
for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
{
cnt ++;
......@@ -965,23 +969,6 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
inREcnt++;
}
// if we are out of time then skip this test, we leave two minutes
// to submit results
if (this->CTest->GetRemainingTimeAllowed() - 120 <= 0)
{