Commit bdf30bda authored by Rodolfo Schulz de Lima's avatar Rodolfo Schulz de Lima
Browse files

CPack: add CPACK_THREADS variable to control compression threads

This allows setting how many threads the compressor will use.
Currently only implemented for XZ when using system's lzma library.

Fixes: #21715
parent bcdb5b52
......@@ -80,6 +80,8 @@ CPack generators which are essentially archives at their core. These include:
not all compression modes support threading in all environments. Currently,
only the XZ compression may support it.
See also the :variable:`CPACK_THREADS` variable.
.. note::
Official CMake binaries available on ``cmake.org`` ship with a ``liblzma``
......
cpack-compression-threads
-------------------------
* :module:`CPack` gained the :variable:`CPACK_THREADS` variable to
control the number of threads used for parallelized operations,
such as compressing the installer package.
......@@ -282,6 +282,28 @@ installers. The most commonly-used variables are:
received by the cpack program. Defaults to ``FALSE`` for backwards
compatibility.
.. variable:: CPACK_THREADS
.. versionadded:: 3.20
Number of threads to use when performing parallelized operations, such
as compressing the installer package.
Some compression methods used by CPack generators such as Debian or Archive
may take advantage of multiple CPU cores to speed up compression.
``CPACK_THREADS`` can be set to positive integer to specify how many threads
will be used for compression. If it is set to 0, CPack will set it so that
all available CPU cores are used.
By default ``CPACK_THREADS`` is set to ``1``.
Currently only ``xz`` compression *may* take advantage of multiple cores. Other
compression methods ignore this value and use only one thread.
.. note::
Official CMake binaries available on ``cmake.org`` ship with a ``liblzma``
that does not support parallel compression.
Variables for Source Package Generators
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
......@@ -746,6 +768,7 @@ _cpack_set_default(CPACK_INSTALL_CMAKE_PROJECTS
"${CMAKE_BINARY_DIR};${CMAKE_PROJECT_NAME};ALL;/")
_cpack_set_default(CPACK_CMAKE_GENERATOR "${CMAKE_GENERATOR}")
_cpack_set_default(CPACK_TOPLEVEL_TAG "${CPACK_SYSTEM_NAME}")
_cpack_set_default(CPACK_THREADS 1)
# if the user has set CPACK_NSIS_DISPLAY_NAME remember it
if(DEFINED CPACK_NSIS_DISPLAY_NAME)
set(CPACK_NSIS_DISPLAY_NAME_SET TRUE)
......
......@@ -353,8 +353,12 @@ bool cmCPackArchiveGenerator::SetArchiveOptions(cmArchiveWrite* archive)
// cause spurious errors to be raised from `strtoull`.
if (this->Compress == cmArchiveWrite::CompressXZ) {
const char* threads = "1";
// CPACK_ARCHIVE_THREADS overrides CPACK_THREADS
if (this->IsSet("CPACK_ARCHIVE_THREADS")) {
threads = this->GetOption("CPACK_ARCHIVE_THREADS");
} else if (this->IsSet("CPACK_THREADS")) {
threads = this->GetOption("CPACK_THREADS");
}
if (!archive->SetFilterOption("xz", "threads", threads)) {
......
......@@ -2,6 +2,7 @@
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmCPackDebGenerator.h"
#include <cstdlib>
#include <cstring>
#include <map>
#include <ostream>
......@@ -28,7 +29,7 @@ class DebGenerator
public:
DebGenerator(cmCPackLog* logger, std::string outputName, std::string workDir,
std::string topLevelDir, std::string temporaryDir,
const char* debianCompressionType,
const char* debianCompressionType, const char* numThreads,
const char* debianArchiveType,
std::map<std::string, std::string> controlValues,
bool genShLibs, std::string shLibsFilename, bool genPostInst,
......@@ -53,6 +54,7 @@ private:
const std::string TopLevelDir;
const std::string TemporaryDir;
const char* DebianArchiveType;
int NumThreads;
const std::map<std::string, std::string> ControlValues;
const bool GenShLibs;
const std::string ShLibsFilename;
......@@ -69,7 +71,8 @@ private:
DebGenerator::DebGenerator(
cmCPackLog* logger, std::string outputName, std::string workDir,
std::string topLevelDir, std::string temporaryDir,
const char* debianCompressionType, const char* debianArchiveType,
const char* debianCompressionType, const char* numThreads,
const char* debianArchiveType,
std::map<std::string, std::string> controlValues, bool genShLibs,
std::string shLibsFilename, bool genPostInst, std::string postInst,
bool genPostRm, std::string postRm, const char* controlExtra,
......@@ -115,6 +118,23 @@ DebGenerator::DebGenerator(
"Error unrecognized compression type: "
<< debianCompressionType << std::endl);
}
if (numThreads == nullptr) {
numThreads = "1";
}
char* endptr;
this->NumThreads = static_cast<int>(strtol(numThreads, &endptr, 10));
if (numThreads != endptr && *endptr != '\0') {
cmCPackLogger(cmCPackLog::LOG_ERROR,
"Unrecognized number of threads: " << numThreads
<< std::endl);
}
if (this->NumThreads < 0) {
cmCPackLogger(cmCPackLog::LOG_ERROR,
"Number of threads cannot be negative" << std::endl);
}
}
bool DebGenerator::generate() const
......@@ -173,7 +193,7 @@ bool DebGenerator::generateDataTar() const
return false;
}
cmArchiveWrite data_tar(fileStream_data_tar, this->TarCompressionType,
this->DebianArchiveType);
this->DebianArchiveType, 0, this->NumThreads);
data_tar.Open();
// uid/gid should be the one of the root user, and this root user has
......@@ -807,6 +827,7 @@ int cmCPackDebGenerator::createDeb()
this->GetOption("CPACK_TOPLEVEL_DIRECTORY"),
this->GetOption("CPACK_TEMPORARY_DIRECTORY"),
this->GetOption("GEN_CPACK_DEBIAN_COMPRESSION_TYPE"),
this->GetOption("CPACK_THREADS"),
this->GetOption("GEN_CPACK_DEBIAN_ARCHIVE_TYPE"), controlValues, gen_shibs,
shlibsfilename, this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTINST"), postinst,
this->IsOn("GEN_CPACK_DEBIAN_GENERATE_POSTRM"), postrm,
......@@ -864,6 +885,7 @@ int cmCPackDebGenerator::createDbgsymDDeb()
this->GetOption("CPACK_TOPLEVEL_DIRECTORY"),
this->GetOption("CPACK_TEMPORARY_DIRECTORY"),
this->GetOption("GEN_CPACK_DEBIAN_COMPRESSION_TYPE"),
this->GetOption("CPACK_THREADS"),
this->GetOption("GEN_CPACK_DEBIAN_ARCHIVE_TYPE"), controlValues, false, "",
false, "", false, "", nullptr,
this->IsSet("GEN_CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION"),
......
......@@ -2,6 +2,7 @@
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmArchiveWrite.h"
#include <cstdio>
#include <cstring>
#include <ctime>
#include <iostream>
......@@ -81,7 +82,8 @@ struct cmArchiveWrite::Callback
};
cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
std::string const& format, int compressionLevel)
std::string const& format, int compressionLevel,
int numThreads)
: Stream(os)
, Archive(archive_write_new())
, Disk(archive_read_disk_new())
......@@ -142,6 +144,18 @@ cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
cm_archive_error_string(this->Archive));
return;
}
{
char sNumThreads[8];
snprintf(sNumThreads, sizeof(sNumThreads), "%d", numThreads);
sNumThreads[7] = '\0'; // for safety
if (archive_write_set_filter_option(this->Archive, "xz", "threads",
sNumThreads) != ARCHIVE_OK) {
this->Error = cmStrCat("archive_compressor_xz_options: ",
cm_archive_error_string(this->Archive));
return;
}
}
break;
case CompressZstd:
if (archive_write_add_filter_zstd(this->Archive) != ARCHIVE_OK) {
......
......@@ -54,7 +54,8 @@ public:
/** Construct with output stream to which to write archive. */
cmArchiveWrite(std::ostream& os, Compress c = CompressNone,
std::string const& format = "paxr", int compressionLevel = 0);
std::string const& format = "paxr", int compressionLevel = 0,
int numThreads = 1);
~cmArchiveWrite();
......
......@@ -21,8 +21,8 @@ run_cpack_test(LONG_FILENAMES "DEB.LONG_FILENAMES" false "MONOLITHIC")
run_cpack_test_subtests(MAIN_COMPONENT "invalid;found" "RPM.MAIN_COMPONENT" false "COMPONENT")
run_cpack_test(MINIMAL "RPM.MINIMAL;DEB.MINIMAL;7Z;TBZ2;TGZ;TXZ;TZ;ZIP;STGZ;External" false "MONOLITHIC;COMPONENT")
run_cpack_test_package_target(MINIMAL "RPM.MINIMAL;DEB.MINIMAL;7Z;TBZ2;TGZ;TXZ;TZ;ZIP;STGZ;External" false "MONOLITHIC;COMPONENT")
run_cpack_test_package_target(THREADED_ALL "TXZ" false "MONOLITHIC;COMPONENT")
run_cpack_test_package_target(THREADED "TXZ" false "MONOLITHIC;COMPONENT")
run_cpack_test_package_target(THREADED_ALL "TXZ;DEB" false "MONOLITHIC;COMPONENT")
run_cpack_test_package_target(THREADED "TXZ;DEB" false "MONOLITHIC;COMPONENT")
run_cpack_test_subtests(PACKAGE_CHECKSUM "invalid;MD5;SHA1;SHA224;SHA256;SHA384;SHA512" "TGZ" false "MONOLITHIC")
run_cpack_test(PARTIALLY_RELOCATABLE_WARNING "RPM.PARTIALLY_RELOCATABLE_WARNING" false "COMPONENT")
run_cpack_test(PER_COMPONENT_FIELDS "RPM.PER_COMPONENT_FIELDS;DEB.PER_COMPONENT_FIELDS" false "COMPONENT")
......
install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
set(CPACK_ARCHIVE_THREADS 2)
set(CPACK_THREADS 2)
if(PACKAGING_TYPE STREQUAL "COMPONENT")
set(CPACK_COMPONENTS_ALL test)
......
install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
set(CPACK_ARCHIVE_THREADS 0)
set(CPACK_THREADS 0)
if(PACKAGING_TYPE STREQUAL "COMPONENT")
set(CPACK_COMPONENTS_ALL test)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment