ENH: add initial ctest -j feature

Source/CMakeLists.txt

@@ -314,6 +314,7 @@ INCLUDE_DIRECTORIES(
 # Sources for CTestLib
 #
 SET(CTEST_SRCS cmCTest.cxx
+  CTest/cmProcess.cxx
   CTest/cmCTestBuildAndTestHandler.cxx
   CTest/cmCTestBuildCommand.cxx
   CTest/cmCTestBuildHandler.cxx
@@ -326,6 +327,7 @@ SET(CTEST_SRCS cmCTest.cxx
   CTest/cmCTestHandlerCommand.cxx
   CTest/cmCTestMemCheckCommand.cxx
   CTest/cmCTestMemCheckHandler.cxx
+  CTest/cmCTestMultiProcessHandler.cxx
   CTest/cmCTestReadCustomFilesCommand.cxx
   CTest/cmCTestRunScriptCommand.cxx
   CTest/cmCTestScriptHandler.cxx
@@ -458,3 +460,4 @@ IF(APPLE)
 ENDIF(APPLE)
 
 INSTALL_FILES(${CMAKE_DATA_DIR}/include cmCPluginAPI.h)
+

Source/CTest/cmCTestGenericHandler.cxx

@@ -161,6 +161,12 @@ bool cmCTestGenericHandler::StartLogFile(const char* name,
     ostr << "_" << this->CTest->GetCurrentTag();
     }
   ostr << ".log";
+  // if this is a parallel subprocess then add the id to the
+  // file so they don't clobber each other
+  if(this->CTest->GetParallelSubprocess())
+    {
+    ostr << "." << this->CTest->GetParallelSubprocessId();
+    }
   if( !this->CTest->OpenOutputFile("Temporary", ostr.str().c_str(), xofs) )
     {
     cmCTestLog(this->CTest, ERROR_MESSAGE, "Cannot create log file: "

Source/CTest/cmCTestMultiProcessHandler.cxx

+/*=========================================================================
+
+  Program:   CMake - Cross-Platform Makefile Generator
+  Module:    $RCSfile$
+  Language:  C++
+  Date:      $Date$
+  Version:   $Revision$
+
+  Copyright (c) 2002 Kitware, Inc., Insight Consortium.  All rights reserved.
+  See Copyright.txt or http://www.cmake.org/HTML/Copyright.html for details.
+
+     This software is distributed WITHOUT ANY WARRANTY; without even
+     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+     PURPOSE.  See the above copyright notices for more information.
+
+=========================================================================*/
+#include "cmCTestMultiProcessHandler.h"
+#include "cmProcess.h"
+#include "cmStandardIncludes.h"
+#include "cmCTest.h"
+
+
+cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
+{
+  this->ParallelLevel = 1;
+  this->ProcessId = 0;
+}
+  // Set the tests
+void 
+cmCTestMultiProcessHandler::SetTests(std::map<int, std::set<int> >& tests,
+                                          std::map<int,cmStdString>& testNames)
+{
+  // set test run map to false for all
+  for(std::map<int, std::set<int> >::iterator i = this->Tests.begin();
+      i != this->Tests.end(); ++i)
+    {
+    this->TestRunningMap[i->first] = false;
+    this->TestFinishMap[i->first] = false;
+    }
+  this->Tests = tests;
+  this->TestNames = testNames;
+}
+  // Set the max number of tests that can be run at the same time.
+void cmCTestMultiProcessHandler::SetParallelLevel(size_t l)
+{
+  this->ParallelLevel = l;
+}
+
+
+void cmCTestMultiProcessHandler::RunTests()
+{
+  this->StartNextTests();
+  while(this->Tests.size() != 0)
+    {
+    this->CheckOutput();
+    this->StartNextTests();
+    }
+  // let all running tests finish
+  while(this->CheckOutput())
+    {
+    }
+  
+  for(std::map<int, cmStdString>::iterator i =
+        this->TestOutput.begin();
+      i != this->TestOutput.end(); ++i)
+    {
+    cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, 
+               i->second << std::endl);
+    }
+      
+}
+
+void cmCTestMultiProcessHandler::StartTestProcess(int test)
+{
+  cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, 
+            " test " << test << "\n");
+  this->TestRunningMap[test] = true; // mark the test as running
+  // now remove the test itself
+  this->Tests.erase(test);
+  // now run the test
+  cmProcess* newp = new cmProcess;
+  newp->SetId(this->ProcessId);
+  newp->SetId(test);
+  newp->SetCommand(this->CTestCommand.c_str());
+  std::vector<std::string> args;
+  args.push_back("-I");
+  cmOStringStream strm;
+  strm << test << "," << test;
+  args.push_back(strm.str());
+  args.push_back("--parallel-cache");
+  args.push_back(this->CTestCacheFile.c_str());
+  args.push_back("--internal-ctest-parallel"); 
+  cmOStringStream strm2;
+  strm2 << test;
+  args.push_back(strm2.str());
+  if(this->CTest->GetExtraVerbose())
+    {
+    args.push_back("-VV");
+    }
+  newp->SetCommandArguments(args);
+  if(!newp->StartProcess())
+    {
+     cmCTestLog(this->CTest, ERROR_MESSAGE, 
+                "Error starting " << newp->GetCommand() << "\n");
+    this->EndTest(newp);
+    }
+  else
+    {
+    this->RunningTests.insert(newp);
+    }
+ cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, 
+            "ctest -I " << test << "\n");
+ this->ProcessId++;
+}
+
+bool cmCTestMultiProcessHandler::StartTest(int test)
+{
+  // copy the depend tests locally because when 
+  // a test is finished it will be removed from the depend list
+  // and we don't want to be iterating a list while removing from it
+  std::set<int> depends = this->Tests[test];
+  size_t totalDepends = depends.size();
+  if(totalDepends)
+    {
+    for(std::set<int>::const_iterator i = depends.begin();
+        i != depends.end(); ++i)
+      {
+      // if the test is not already running then start it
+      if(!this->TestRunningMap[*i])
+        {
+        // this test might be finished, but since
+        // this is a copy of the depend map we might
+        // still have it
+        if(!this->TestFinishMap[*i])
+          {
+          // only start one test in this function
+          return this->StartTest(*i);
+          }
+        else
+          {
+          // the depend has been and finished
+          totalDepends--;
+          }
+        }
+      }
+    }
+  // if there are no depends left then run this test
+  if(totalDepends == 0)
+    {
+    // Start this test it has no depends 
+    this->StartTestProcess(test);
+    return true;
+    }
+  // This test was not able to start because it is waiting 
+  // on depends to run
+  return false;  
+}
+
+void cmCTestMultiProcessHandler::StartNextTests()
+{
+  cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl
+             << "Number of running tests : " << this->RunningTests.size()
+             << "\n");
+  size_t numToStart = this->ParallelLevel - this->RunningTests.size();
+  if(numToStart == 0)
+    {
+    return;
+    }
+  std::map<int, std::set<int> > tests = this->Tests;
+  for(std::map<int, std::set<int> >::iterator i = tests.begin();
+      i !=  tests.end(); ++i)
+    {
+    // start test should start only one test
+    if(this->StartTest(i->first))
+      {
+      numToStart--;
+      }
+    else
+      {
+      cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl
+             << "Test did not start waiting on depends to finish: " 
+                 << i->first << "\n");
+      }
+    if(numToStart == 0 )
+      {
+      return;
+      }
+    }
+}
+
+
+bool cmCTestMultiProcessHandler::CheckOutput()
+{
+  // no more output we are done
+  if(this->RunningTests.size() == 0)
+    {
+    return false;
+    }
+  std::vector<cmProcess*> finished;
+  std::string out, err;
+  for(std::set<cmProcess*>::const_iterator i = this->RunningTests.begin();
+      i != this->RunningTests.end(); ++i)
+    {
+    cmProcess* p = *i;
+    int pipe = p->CheckOutput(.1, out, err);
+    if(pipe == cmsysProcess_Pipe_STDOUT)
+      {
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, 
+                 p->GetId() << ": " << out << std::endl);
+      this->TestOutput[ p->GetId() ] += out;
+      this->TestOutput[ p->GetId() ] += "\n";
+      }
+    else if(pipe == cmsysProcess_Pipe_STDERR)
+      {
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, 
+                 p->GetId() << ": " << err << std::endl);
+      this->TestOutput[ p->GetId() ] += err;
+      this->TestOutput[ p->GetId() ] += "\n";
+      }
+    if(!p->IsRunning())
+      {
+      finished.push_back(p);
+      }
+    }
+  for( std::vector<cmProcess*>::iterator i = finished.begin();
+       i != finished.end(); ++i)
+    {
+    cmProcess* p = *i;
+    this->EndTest(p);
+    }
+  return true;
+}
+
+void cmCTestMultiProcessHandler::EndTest(cmProcess* p)
+{
+  int test = p->GetId();
+  int exitVal = p->GetExitValue();
+  cmCTestTestHandler::cmCTestTestResult cres;
+  cres.Properties = 0;
+  cres.ExecutionTime = 0;// ???
+  cres.ReturnValue = exitVal;
+  cres.Status = cmCTestTestHandler::COMPLETED;
+  cres.TestCount = test;  
+  cres.Name = this->TestNames[test];
+  cres.Path.clear();
+  if(exitVal)
+    {
+    cres.Status = cmCTestTestHandler::FAILED;
+    this->Failed->push_back(this->TestNames[test]);
+    }
+  else
+    {
+    this->Passed->push_back(this->TestNames[test]);
+    }
+  this->TestResults->push_back(cres);
+  // remove test from depend of all other tests
+  for( std::map<int, std::set<int> >::iterator i = this->Tests.begin();
+       i!=  this->Tests.end(); ++i)
+    {
+    i->second.erase(test);
+    }
+  this->TestFinishMap[test] = true;
+  this->TestRunningMap[test] = false;
+  this->RunningTests.erase(p);
+  delete p;
+  cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, 
+             "finish test " << test << "\n");
+}
+
+
+void cmCTestMultiProcessHandler::PrintTests()
+{
+#undef cout
+  for( std::map<int, std::set<int> >::iterator i = this->Tests.begin();
+       i!=  this->Tests.end(); ++i)
+    {
+    std::cout << "Test " << i->first << "  (";
+    for(std::set<int>::iterator j = i->second.begin(); 
+        j != i->second.end(); ++j)
+      {
+      std::cout << *j << " ";
+      }
+    std::cout << ")\n";
+    }
+}
+
+#if 0
+int main()
+{
+  cmCTestMultiProcessHandler h;
+  h.SetParallelLevel(4);
+  std::map<int, std::set<int> > tests;
+  std::set<int> depends;
+  for(int i =1; i < 92; i++)
+    {
+    tests[i] = depends;
+    }
+  depends.clear();
+  depends.insert(45);  subprject
+  tests[46] = depends;  subproject-stage2
+  depends.clear();
+  depends.insert(55);  simpleinstall simpleinstall-s2
+  tests[56] = depends;
+  depends.clear();
+  depends.insert(70);  wrapping
+  tests[71] = depends; qtwrapping
+  depends.clear();
+  depends.insert(71);  qtwrapping
+  tests[72] = depends;  testdriver1
+  depends.clear();
+  depends.insert(72)    testdriver1
+  tests[73] = depends;  testdriver2
+  depends.clear();
+  depends.insert(73);   testdriver2
+  tests[74] = depends;  testdriver3
+  depends.clear();
+  depends.insert(79);   linkorder1
+  tests[80] = depends;  linkorder2
+  h.SetTests(tests);
+  h.PrintTests();
+  h.RunTests();
+}
+#endif

Source/CTest/cmCTestMultiProcessHandler.h

+/*=========================================================================
+
+  Program:   CMake - Cross-Platform Makefile Generator
+  Module:    $RCSfile$
+  Language:  C++
+  Date:      $Date$
+  Version:   $Revision$
+
+  Copyright (c) 2002 Kitware, Inc., Insight Consortium.  All rights reserved.
+  See Copyright.txt or http://www.cmake.org/HTML/Copyright.html for details.
+
+     This software is distributed WITHOUT ANY WARRANTY; without even
+     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+     PURPOSE.  See the above copyright notices for more information.
+
+=========================================================================*/
+#ifndef cmCTestMultiProcessHandler_h
+#define cmCTestMultiProcessHandler_h
+
+#include <set>
+#include <map>
+#include <string>
+#include <vector>
+class cmProcess;
+#include <cmStandardIncludes.h>
+#include <cmCTestTestHandler.h>
+
+/** \class cmCTestMultiProcessHandler
+ * \brief run parallel ctest
+ *
+ * cmCTestMultiProcessHandler 
+ */
+class cmCTestMultiProcessHandler 
+{
+public:
+  cmCTestMultiProcessHandler();
+  // Set the tests
+  void SetTests(std::map<int, std::set<int> >& tests,
+                std::map<int, cmStdString>& testNames);
+  // Set the max number of tests that can be run at the same time.
+  void SetParallelLevel(size_t);
+  void RunTests();
+  void PrintTests();
+  void SetCTestCommand(const char* c) { this->CTestCommand = c;}
+  void SetTestCacheFile(const char* c) { this->CTestCacheFile = c;}
+  void SetPassFailVectors(std::vector<cmStdString>* passed,
+                          std::vector<cmStdString>* failed)
+    {
+      this->Passed = passed;
+      this->Failed = failed;
+    }
+  void SetTestResults(std::vector<cmCTestTestHandler::cmCTestTestResult>* r)
+    {
+      this->TestResults = r;
+    }
+  void SetCTest(cmCTest* ctest) { this->CTest = ctest;}
+protected:  
+  cmCTest* CTest;
+  // Start the next test or tests as many as are allowed by
+  // ParallelLevel
+  void StartNextTests();
+  void StartTestProcess(int test);
+  bool StartTest(int test);
+  void EndTest(cmProcess*);
+  // Return true if there are still tests running
+  // check all running processes for output and exit case
+  bool CheckOutput();
+  // map from test number to set of depend tests
+  std::map<int, std::set<int> > Tests;
+  std::map<int, cmStdString> TestNames;
+  std::map<int, bool> TestRunningMap;
+  std::map<int, bool> TestFinishMap;
+  std::map<int, cmStdString> TestOutput;
+  std::string CTestCommand;
+  std::string CTestCacheFile;
+  std::vector<cmStdString>* Passed;
+  std::vector<cmStdString>* Failed;
+  std::vector<cmCTestTestHandler::cmCTestTestResult>* TestResults;
+  int ProcessId;
+  size_t ParallelLevel; // max number of process that can be run at once
+  std::set<cmProcess*> RunningTests;  // current running tests
+};
+
+#endif

Source/CTest/cmCTestTestHandler.cxx

@@ -16,7 +16,7 @@
 =========================================================================*/
 
 #include "cmCTestTestHandler.h"
-
+#include "cmCTestMultiProcessHandler.h"
 #include "cmCTest.h"
 #include "cmake.h"
 #include "cmGeneratedFileStream.h"
@@ -500,11 +500,14 @@ int cmCTestTestHandler::ProcessHandler()
     }
   
   this->TestResults.clear();
-
-  cmCTestLog(this->CTest, HANDLER_OUTPUT,
-    (this->MemCheck ? "Memory check" : "Test")
-             << " project " << cmSystemTools::GetCurrentWorkingDirectory()
-             << std::endl);
+ // do not output startup if this is a sub-process for parallel tests
+  if(!this->CTest->GetParallelSubprocess())
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT,
+               (this->MemCheck ? "Memory check" : "Test")
+               << " project " << cmSystemTools::GetCurrentWorkingDirectory()
+               << std::endl);
+    }
   if ( ! this->PreProcessHandler() )
     {
     return -1;
@@ -517,7 +520,6 @@ int cmCTestTestHandler::ProcessHandler()
   std::vector<cmStdString> passed;
   std::vector<cmStdString> failed;
   int total;
-
   this->ProcessDirectory(passed, failed);
 
   total = int(passed.size()) + int(failed.size());
@@ -550,33 +552,37 @@ int cmCTestTestHandler::ProcessHandler()
       {
       percent = 99;
       }
-    cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
-      << static_cast<int>(percent + .5) << "% tests passed, "
-      << failed.size() << " tests failed out of " << total << std::endl);
-    //fprintf(stderr,"\n%.0f%% tests passed, %i tests failed out of %i\n",
-    //  percent, int(failed.size()), total);
+    if(!this->CTest->GetParallelSubprocess())
+      {
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
+                 << static_cast<int>(percent + .5) << "% tests passed, "
+                 << failed.size() << " tests failed out of " 
+                 << total << std::endl);
+      }
 
     if (failed.size())
       {
       cmGeneratedFileStream ofs;
-
-      cmCTestLog(this->CTest, ERROR_MESSAGE, std::endl
-        << "The following tests FAILED:" << std::endl);
-      this->StartLogFile("TestsFailed", ofs);
-
-      std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator ftit;
-      for(ftit = this->TestResults.begin();
-        ftit != this->TestResults.end(); ++ftit)
+      if(!this->CTest->GetParallelSubprocess())
         {
-        if ( ftit->Status != cmCTestTestHandler::COMPLETED )
+        cmCTestLog(this->CTest, ERROR_MESSAGE, std::endl
+                   << "The following tests FAILED:" << std::endl);
+        this->StartLogFile("TestsFailed", ofs);
+        
+        std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator ftit;
+        for(ftit = this->TestResults.begin();
+            ftit != this->TestResults.end(); ++ftit)
           {
-          ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
-          cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" << std::setw(3)
-            << ftit->TestCount << " - " << ftit->Name.c_str() << " ("
-            << this->GetTestStatus(ftit->Status) << ")" << std::endl);
+          if ( ftit->Status != cmCTestTestHandler::COMPLETED )
+            {
+            ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
+            cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" << std::setw(3)
+                       << ftit->TestCount << " - " << ftit->Name.c_str() << " ("
+                       << this->GetTestStatus(ftit->Status) << ")" << std::endl);
+            }
           }
+        
         }
-
       }
     }
 
@@ -677,8 +683,8 @@ void cmCTestTestHandler::ProcessOneTest(cmCTestTestProperties *it,
   
   // add the arguments
   std::vector<std::string>::const_iterator j = args.begin();
-  ++j;
-  ++j;
+  ++j; // skip test name
+  ++j; // skip command as it is in actualCommand
   std::vector<const char*> arguments;
   this->GenerateTestCommand(arguments);
   arguments.push_back(actualCommand.c_str());
@@ -917,23 +923,20 @@ void cmCTestTestHandler::ProcessOneTest(cmCTestTestProperties *it,
 }
 
 //----------------------------------------------------------------------
-void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
-                                          std::vector<cmStdString> &failed)
+void cmCTestTestHandler::ComputeTestList()
 {
-  std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
-  this->TestList.clear();
-
-  this->GetListOfTests();
+  this->TestList.clear(); // clear list of test
+  if(this->CTest->GetParallelSubprocess())
+    {
+    this->LoadTestList();
+    return;
+    }
+  else
+    {
+    this->GetListOfTests();
+    }
   cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
 
-  this->StartTest = this->CTest->CurrentTime();
-  this->StartTestTime = static_cast<unsigned int>(cmSystemTools::GetTime());
-  double elapsed_time_start = cmSystemTools::GetTime();
-
-  *this->LogFile << "Start testing: " << this->StartTest << std::endl
-    << "----------------------------------------------------------"
-    << std::endl;
-
   // how many tests are in based on RegExp?
   int inREcnt = 0;
   cmCTestTestHandler::ListOfTests::iterator it;
@@ -953,10 +956,11 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
     {
     this->ExpandTestsToRunInformation(inREcnt);
     }
-
+  // Now create a final list of tests to run
   int cnt = 0;
   inREcnt = 0;
-  std::string last_directory = "";
+  std::string last_directory = ""; 
+  ListOfTests finalList;
   for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
     {
     cnt ++;
@@ -965,23 +969,6 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
       inREcnt++;
       }
 
-    // if we are out of time then skip this test, we leave two minutes 
-    // to submit results
-    if (this->CTest->GetRemainingTimeAllowed() - 120 <= 0)
-      {
-      continue;
-      }
-