Commit 3c165593 authored by Brad King's avatar Brad King Committed by Kitware Robot
Browse files

Merge topic 'lzma-threads'

c5c130e6 cmArchiveWrite: Consolidate multiple ways to set thread count
5380d858 liblzma: Enable multi threaded stream encoding support
e9065e96 Merge branch 'upstream-liblzma' into lzma-threads
ee909a8e liblzma 2020-03-17 (2327a461)
741b85b4

 liblzma: Revise update script to get version 5.2.5
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Acked-by: Ben Boeckel's avatarBen Boeckel <ben.boeckel@kitware.com>
Merge-request: !6014
parents e69a3287 c5c130e6
......@@ -579,6 +579,7 @@ macro (CMAKE_BUILD_UTILITIES)
set(LIBLZMA_INCLUDE_DIR
"${CMAKE_CURRENT_SOURCE_DIR}/Utilities/cmliblzma/liblzma/api")
set(LIBLZMA_LIBRARY cmliblzma)
set(HAVE_LZMA_STREAM_ENCODER_MT 1)
endif()
endif()
......
......@@ -82,7 +82,8 @@ CPack generators which are essentially archives at their core. These include:
See also the :variable:`CPACK_THREADS` variable.
.. note::
.. versionadded:: 3.21
Official CMake binaries available on ``cmake.org`` ship with a ``liblzma``
that does not support parallel compression.
Official CMake binaries available on ``cmake.org`` now ship
with a ``liblzma`` that supports parallel compression.
Older versions did not.
lzma-threads
------------
* The precompiled binaries provided on
`cmake.org <https://cmake.org/download/>`_ now support
``liblzma`` multi-threading. See the :variable:`CPACK_THREADS` and
:variable:`CPACK_ARCHIVE_THREADS` variables.
......@@ -291,18 +291,27 @@ installers. The most commonly-used variables are:
Some compression methods used by CPack generators such as Debian or Archive
may take advantage of multiple CPU cores to speed up compression.
``CPACK_THREADS`` can be set to positive integer to specify how many threads
will be used for compression. If it is set to 0, CPack will set it so that
all available CPU cores are used.
``CPACK_THREADS`` can be set to specify how many threads will be
used for compression.
A positive integer can be used to specify an exact desired thread count.
When given a negative integer CPack will use the absolute value
as the upper limit but may choose a lower value based on
the available hardware concurrency.
Given 0 CPack will try to use all available CPU cores.
By default ``CPACK_THREADS`` is set to ``1``.
Currently only ``xz`` compression *may* take advantage of multiple cores. Other
compression methods ignore this value and use only one thread.
Currently only ``xz`` compression *may* take advantage of multiple cores.
Other compression methods ignore this value and use only one thread.
.. note::
.. versionadded:: 3.21
Official CMake binaries available on ``cmake.org`` ship with a ``liblzma``
that does not support parallel compression.
Official CMake binaries available on ``cmake.org`` now ship
with a ``liblzma`` that supports parallel compression.
Older versions did not.
Variables for Source Package Generators
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
......
......@@ -2,14 +2,13 @@
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmCPackArchiveGenerator.h"
#include <cstdlib>
#include <cstring>
#include <map>
#include <ostream>
#include <utility>
#include <vector>
#include <cm3p/archive.h>
#include "cmCPackComponentGroup.h"
#include "cmCPackGenerator.h"
#include "cmCPackLog.h"
......@@ -154,15 +153,9 @@ int cmCPackArchiveGenerator::addOneComponentToArchive(
<< (filename) << ">." << std::endl); \
return 0; \
} \
cmArchiveWrite archive(gf, this->Compress, this->ArchiveFormat); \
cmArchiveWrite archive(gf, this->Compress, this->ArchiveFormat, 0, \
this->GetThreadCount()); \
do { \
if (!this->SetArchiveOptions(&archive)) { \
cmCPackLogger(cmCPackLog::LOG_ERROR, \
"Problem to set archive options <" \
<< (filename) << ">, ERROR = " << (archive).GetError() \
<< std::endl); \
return 0; \
} \
if (!archive.Open()) { \
cmCPackLogger(cmCPackLog::LOG_ERROR, \
"Problem to open archive <" \
......@@ -346,26 +339,16 @@ bool cmCPackArchiveGenerator::SupportsComponentInstallation() const
return this->IsOn("CPACK_ARCHIVE_COMPONENT_INSTALL");
}
bool cmCPackArchiveGenerator::SetArchiveOptions(cmArchiveWrite* archive)
int cmCPackArchiveGenerator::GetThreadCount() const
{
#if ARCHIVE_VERSION_NUMBER >= 3004000
// Upstream fixed an issue with their integer parsing in 3.4.0 which would
// cause spurious errors to be raised from `strtoull`.
if (this->Compress == cmArchiveWrite::CompressXZ) {
const char* threads = "1";
// CPACK_ARCHIVE_THREADS overrides CPACK_THREADS
if (this->IsSet("CPACK_ARCHIVE_THREADS")) {
threads = this->GetOption("CPACK_ARCHIVE_THREADS");
} else if (this->IsSet("CPACK_THREADS")) {
threads = this->GetOption("CPACK_THREADS");
}
int threads = 1;
if (!archive->SetFilterOption("xz", "threads", threads)) {
return false;
}
// CPACK_ARCHIVE_THREADS overrides CPACK_THREADS
if (this->IsSet("CPACK_ARCHIVE_THREADS")) {
threads = std::atoi(this->GetOption("CPACK_ARCHIVE_THREADS"));
} else if (this->IsSet("CPACK_THREADS")) {
threads = std::atoi(this->GetOption("CPACK_THREADS"));
}
#endif
return true;
return threads;
}
......@@ -85,7 +85,7 @@ private:
return this->OutputExtension.c_str();
}
bool SetArchiveOptions(cmArchiveWrite* archive);
int GetThreadCount() const;
private:
cmArchiveWrite::Compress Compress;
......
......@@ -130,11 +130,6 @@ DebGenerator::DebGenerator(
"Unrecognized number of threads: " << numThreads
<< std::endl);
}
if (this->NumThreads < 0) {
cmCPackLogger(cmCPackLog::LOG_ERROR,
"Number of threads cannot be negative" << std::endl);
}
}
bool DebGenerator::generate() const
......
......@@ -2,11 +2,16 @@
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmArchiveWrite.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <iostream>
#include <limits>
#include <sstream>
#include <string>
#include <thread>
#include <cm/algorithm>
#include <cm3p/archive.h>
#include <cm3p/archive_entry.h>
......@@ -144,16 +149,36 @@ cmArchiveWrite::cmArchiveWrite(std::ostream& os, Compress c,
cm_archive_error_string(this->Archive));
return;
}
{
char sNumThreads[8];
snprintf(sNumThreads, sizeof(sNumThreads), "%d", numThreads);
sNumThreads[7] = '\0'; // for safety
#if ARCHIVE_VERSION_NUMBER >= 3004000
// Upstream fixed an issue with their integer parsing in 3.4.0
// which would cause spurious errors to be raised from `strtoull`.
if (numThreads < 1) {
int upperLimit = (numThreads == 0) ? std::numeric_limits<int>::max()
: std::abs(numThreads);
numThreads =
cm::clamp<int>(std::thread::hardware_concurrency(), 1, upperLimit);
}
# ifdef _AIX
// FIXME: Using more than 2 threads creates an empty archive.
// Enforce this limit pending further investigation.
numThreads = std::min(numThreads, 2);
# endif
std::string sNumThreads = std::to_string(numThreads);
if (archive_write_set_filter_option(this->Archive, "xz", "threads",
sNumThreads) != ARCHIVE_OK) {
sNumThreads.c_str()) !=
ARCHIVE_OK) {
this->Error = cmStrCat("archive_compressor_xz_options: ",
cm_archive_error_string(this->Archive));
return;
}
#endif
}
break;
......@@ -425,16 +450,3 @@ bool cmArchiveWrite::AddData(const char* file, size_t size)
}
return true;
}
bool cmArchiveWrite::SetFilterOption(const char* module, const char* key,
const char* value)
{
if (archive_write_set_filter_option(this->Archive, module, key, value) !=
ARCHIVE_OK) {
this->Error = "archive_write_set_filter_option: ";
this->Error += cm_archive_error_string(this->Archive);
return false;
}
return true;
}
......@@ -141,9 +141,6 @@ public:
this->Gname = "";
}
//! Set an option on a filter;
bool SetFilterOption(const char* module, const char* key, const char* value);
private:
bool Okay() const { return this->Error.empty(); }
bool AddPath(const char* path, size_t skip, const char* prefix,
......
install(FILES CMakeLists.txt DESTINATION foo COMPONENT test)
set(CPACK_THREADS 0)
set(CPACK_THREADS "-4")
if(PACKAGING_TYPE STREQUAL "COMPONENT")
set(CPACK_COMPONENTS_ALL test)
......
......@@ -8,12 +8,17 @@ readonly name="liblzma"
readonly ownership="liblzma upstream <xz-devel@tukaani.org>"
readonly subtree="Utilities/cmliblzma"
readonly repo="https://git.tukaani.org/xz.git"
readonly tag="v5.2.4"
readonly tag="v5.2.5"
readonly shortlog=false
readonly paths="
COPYING
src/common/common_w32res.rc
src/common/mythread.h
src/common/sysdefs.h
src/common/tuklib_common.h
src/common/tuklib_config.h
src/common/tuklib_cpucores.c
src/common/tuklib_cpucores.h
src/common/tuklib_integer.h
src/liblzma/
"
......
......@@ -6,6 +6,12 @@ include(CheckSymbolExists)
include(CheckTypeSize)
include(TestBigEndian)
if(WIN32)
add_definitions(-DMYTHREAD_VISTA)
else()
add_definitions(-DMYTHREAD_POSIX)
endif()
CHECK_INCLUDE_FILE(byteswap.h HAVE_BYTESWAP_H)
CHECK_INCLUDE_FILE(limits.h HAVE_LIMITS_H)
CHECK_INCLUDE_FILE(memory.h HAVE_MEMORY_H)
......@@ -60,7 +66,10 @@ set(HAVE_MF_HC3 1)
set(HAVE_MF_HC4 1)
SET(LZMA_SRCS
common/mythread.h
common/sysdefs.h
common/tuklib_cpucores.c
common/tuklib_cpucores.h
common/tuklib_integer.h
liblzma/check/check.c
liblzma/check/crc32_fast.c
......@@ -91,14 +100,17 @@ SET(LZMA_SRCS
liblzma/common/filter_encoder.c
liblzma/common/filter_flags_decoder.c
liblzma/common/filter_flags_encoder.c
liblzma/common/hardware_cputhreads.c
liblzma/common/index.c
liblzma/common/index_decoder.c
liblzma/common/index_encoder.c
liblzma/common/index_hash.c
liblzma/common/outqueue.c
liblzma/common/stream_buffer_decoder.c
liblzma/common/stream_buffer_encoder.c
liblzma/common/stream_decoder.c
liblzma/common/stream_encoder.c
liblzma/common/stream_encoder_mt.c
liblzma/common/stream_flags_common.c
liblzma/common/stream_flags_decoder.c
liblzma/common/stream_flags_encoder.c
......
///////////////////////////////////////////////////////////////////////////////
//
/// \file mythread.h
/// \brief Some threading related helper macros and functions
//
// Author: Lasse Collin
//
// This file has been put into the public domain.
// You can do whatever you want with this file.
//
///////////////////////////////////////////////////////////////////////////////
#ifndef MYTHREAD_H
#define MYTHREAD_H
#include "sysdefs.h"
// If any type of threading is enabled, #define MYTHREAD_ENABLED.
#if defined(MYTHREAD_POSIX) || defined(MYTHREAD_WIN95) \
|| defined(MYTHREAD_VISTA)
# define MYTHREAD_ENABLED 1
#endif
#ifdef MYTHREAD_ENABLED
////////////////////////////////////////
// Shared between all threading types //
////////////////////////////////////////
// Locks a mutex for a duration of a block.
//
// Perform mythread_mutex_lock(&mutex) in the beginning of a block
// and mythread_mutex_unlock(&mutex) at the end of the block. "break"
// may be used to unlock the mutex and jump out of the block.
// mythread_sync blocks may be nested.
//
// Example:
//
// mythread_sync(mutex) {
// foo();
// if (some_error)
// break; // Skips bar()
// bar();
// }
//
// At least GCC optimizes the loops completely away so it doesn't slow
// things down at all compared to plain mythread_mutex_lock(&mutex)
// and mythread_mutex_unlock(&mutex) calls.
//
#define mythread_sync(mutex) mythread_sync_helper1(mutex, __LINE__)
#define mythread_sync_helper1(mutex, line) mythread_sync_helper2(mutex, line)
#define mythread_sync_helper2(mutex, line) \
for (unsigned int mythread_i_ ## line = 0; \
mythread_i_ ## line \
? (mythread_mutex_unlock(&(mutex)), 0) \
: (mythread_mutex_lock(&(mutex)), 1); \
mythread_i_ ## line = 1) \
for (unsigned int mythread_j_ ## line = 0; \
!mythread_j_ ## line; \
mythread_j_ ## line = 1)
#endif
#if !defined(MYTHREAD_ENABLED)
//////////////////
// No threading //
//////////////////
// Calls the given function once. This isn't thread safe.
#define mythread_once(func) \
do { \
static bool once_ = false; \
if (!once_) { \
func(); \
once_ = true; \
} \
} while (0)
#if !(defined(_WIN32) && !defined(__CYGWIN__))
// Use sigprocmask() to set the signal mask in single-threaded programs.
#include <signal.h>
static inline void
mythread_sigmask(int how, const sigset_t *restrict set,
sigset_t *restrict oset)
{
int ret = sigprocmask(how, set, oset);
assert(ret == 0);
(void)ret;
}
#endif
#elif defined(MYTHREAD_POSIX)
////////////////////
// Using pthreads //
////////////////////
#include <sys/time.h>
#include <pthread.h>
#include <signal.h>
#include <time.h>
#include <errno.h>
#define MYTHREAD_RET_TYPE void *
#define MYTHREAD_RET_VALUE NULL
typedef pthread_t mythread;
typedef pthread_mutex_t mythread_mutex;
typedef struct {
pthread_cond_t cond;
#ifdef HAVE_CLOCK_GETTIME
// Clock ID (CLOCK_REALTIME or CLOCK_MONOTONIC) associated with
// the condition variable.
clockid_t clk_id;
#endif
} mythread_cond;
typedef struct timespec mythread_condtime;
// Calls the given function once in a thread-safe way.
#define mythread_once(func) \
do { \
static pthread_once_t once_ = PTHREAD_ONCE_INIT; \
pthread_once(&once_, &func); \
} while (0)
// Use pthread_sigmask() to set the signal mask in multi-threaded programs.
// Do nothing on OpenVMS since it lacks pthread_sigmask().
static inline void
mythread_sigmask(int how, const sigset_t *restrict set,
sigset_t *restrict oset)
{
#ifdef __VMS
(void)how;
(void)set;
(void)oset;
#else
int ret = pthread_sigmask(how, set, oset);
assert(ret == 0);
(void)ret;
#endif
}
// Creates a new thread with all signals blocked. Returns zero on success
// and non-zero on error.
static inline int
mythread_create(mythread *thread, void *(*func)(void *arg), void *arg)
{
sigset_t old;
sigset_t all;
sigfillset(&all);
mythread_sigmask(SIG_SETMASK, &all, &old);
const int ret = pthread_create(thread, NULL, func, arg);
mythread_sigmask(SIG_SETMASK, &old, NULL);
return ret;
}
// Joins a thread. Returns zero on success and non-zero on error.
static inline int
mythread_join(mythread thread)
{
return pthread_join(thread, NULL);
}
// Initiatlizes a mutex. Returns zero on success and non-zero on error.
static inline int
mythread_mutex_init(mythread_mutex *mutex)
{
return pthread_mutex_init(mutex, NULL);
}
static inline void
mythread_mutex_destroy(mythread_mutex *mutex)
{
int ret = pthread_mutex_destroy(mutex);
assert(ret == 0);
(void)ret;
}
static inline void
mythread_mutex_lock(mythread_mutex *mutex)
{
int ret = pthread_mutex_lock(mutex);
assert(ret == 0);
(void)ret;
}
static inline void
mythread_mutex_unlock(mythread_mutex *mutex)
{
int ret = pthread_mutex_unlock(mutex);
assert(ret == 0);
(void)ret;
}
// Initializes a condition variable.
//
// Using CLOCK_MONOTONIC instead of the default CLOCK_REALTIME makes the
// timeout in pthread_cond_timedwait() work correctly also if system time
// is suddenly changed. Unfortunately CLOCK_MONOTONIC isn't available
// everywhere while the default CLOCK_REALTIME is, so the default is
// used if CLOCK_MONOTONIC isn't available.
//
// If clock_gettime() isn't available at all, gettimeofday() will be used.
static inline int
mythread_cond_init(mythread_cond *mycond)
{
#ifdef HAVE_CLOCK_GETTIME
// NOTE: HAVE_DECL_CLOCK_MONOTONIC is always defined to 0 or 1.
# if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && HAVE_DECL_CLOCK_MONOTONIC
struct timespec ts;
pthread_condattr_t condattr;
// POSIX doesn't seem to *require* that pthread_condattr_setclock()
// will fail if given an unsupported clock ID. Test that
// CLOCK_MONOTONIC really is supported using clock_gettime().
if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0
&& pthread_condattr_init(&condattr) == 0) {
int ret = pthread_condattr_setclock(
&condattr, CLOCK_MONOTONIC);
if (ret == 0)
ret = pthread_cond_init(&mycond->cond, &condattr);
pthread_condattr_destroy(&condattr);
if (ret == 0) {
mycond->clk_id = CLOCK_MONOTONIC;
return 0;
}
}
// If anything above fails, fall back to the default CLOCK_REALTIME.
// POSIX requires that all implementations of clock_gettime() must
// support at least CLOCK_REALTIME.
# endif
mycond->clk_id = CLOCK_REALTIME;
#endif
return pthread_cond_init(&mycond->cond, NULL);
}
static inline void
mythread_cond_destroy(mythread_cond *cond)
{
int ret = pthread_cond_destroy(&cond->cond);
assert(ret == 0);
(void)ret;
}
static inline void
mythread_cond_signal(mythread_cond *cond)
{
int ret = pthread_cond_signal(&cond->cond);
assert(ret == 0);
(void)ret;
}
static inline void
mythread_cond_wait(mythread_cond *cond, mythread_mutex *mutex)
{
int ret = pthread_cond_wait(&cond->cond, mutex);
assert(ret == 0);
(void)ret;
}
// Waits on a condition or until a timeout expires. If the timeout expires,
// non-zero is returned, otherwise zero is returned.
static inline int
mythread_cond_timedwait(mythread_cond *cond, mythread_mutex *mutex,
const mythread_condtime *condtime)
{
int ret = pthread_cond_timedwait(&cond->cond, mutex, condtime);
assert(ret == 0 || ret == ETIMEDOUT);
return ret;
}
// Sets condtime to the absolute time that is timeout_ms milliseconds
// in the future. The type of the clock to use is taken from cond.
static inline void
mythread_condtime_set(mythread_condtime *condtime, const mythread_cond *cond,
uint32_t timeout_ms)
{
condtime->tv_sec = timeout_ms / 1000;
condtime->tv_nsec = (timeout_ms % 1000) * 1000000;
#ifdef HAVE_CLOCK_GETTIME
struct timespec now;
int ret = clock_gettime(cond->clk_id, &now);
assert(ret == 0);
(void)ret;