Ninja Multi Config + MSVC race condition on same response file path
We've been having some flaky build failures in our CI when using Ninja Multi-Config
and MSVC
when building 2 different configurations in parallel.
Here's a log excerpt of the failure
[3419/3420] cmd.exe /C "cd . && C:\CMake\bin\cmake.exe -E vs_link_exe --intdir=qmake\CMakeFiles\qmake.dir\RelWithDebInfo --rc=C:\PROGRA~2\WI3CF2~1\10\bin\100183~1.0\x64\rc.exe --mt=C:\PROGRA~2\WI3CF2~1\10\bin\100183~1.0\x64\mt.exe --manifests -- C:\PROGRA~2\MIB055~1\2019\PROFES~1\VC\Tools\MSVC\1423~1.281\bin\Hostx64\x64\link.exe /nologo @CMakeFiles\qmake.rsp /out:bin\qmake.exe /implib:qmake\RelWithDebInfo\qmake.lib /pdb:bin\qmake.pdb /version:0.0 /machine:x64 /debug /INCREMENTAL /subsystem:console -DYNAMICBASE -NXCOMPAT -OPT:REF && cd ."
[3420/3420] cmd.exe /C "cd . && C:\CMake\bin\cmake.exe -E vs_link_exe --intdir=qmake\CMakeFiles\qmake.dir\Debug --rc=C:\PROGRA~2\WI3CF2~1\10\bin\100183~1.0\x64\rc.exe --mt=C:\PROGRA~2\WI3CF2~1\10\bin\100183~1.0\x64\mt.exe --manifests -- C:\PROGRA~2\MIB055~1\2019\PROFES~1\VC\Tools\MSVC\1423~1.281\bin\Hostx64\x64\link.exe /nologo @CMakeFiles\qmake.rsp /out:bin\Debug\qmake.exe /implib:qmake\Debug\qmake.lib /pdb:bin\Debug\qmake.pdb /version:0.0 /machine:x64 /debug /INCREMENTAL /subsystem:console -DYNAMICBASE -NXCOMPAT && cd ."
FAILED: cmd.exe /C "cd . && C:\CMake\bin\cmake.exe -E vs_link_exe --intdir=qmake\CMakeFiles\qmake.dir\Debug --rc=C:\PROGRA~2\WI3CF2~1\10\bin\100183~1.0\x64\rc.exe --mt=C:\PROGRA~2\WI3CF2~1\10\bin\100183~1.0\x64\mt.exe --manifests -- C:\PROGRA~2\MIB055~1\2019\PROFES~1\VC\Tools\MSVC\1423~1.281\bin\Hostx64\x64\link.exe /nologo @CMakeFiles\qmake.rsp /out:bin\Debug\qmake.exe /implib:qmake\Debug\qmake.lib /pdb:bin\Debug\qmake.pdb /version:0.0 /machine:x64 /debug /INCREMENTAL /subsystem:console -DYNAMICBASE -NXCOMPAT && cd ."
FINAL LINK: command "C:\PROGRA~2\MIB055~1\2019\PROFES~1\VC\Tools\MSVC\1423~1.281\bin\Hostx64\x64\link.exe /nologo @CMakeFiles\qmake.rsp /out:bin\Debug\qmake.exe /implib:qmake\Debug\qmake.lib /pdb:bin\Debug\qmake.pdb /version:0.0 /machine:x64 /debug /INCREMENTAL /subsystem:console -DYNAMICBASE -NXCOMPAT /MANIFEST /MANIFESTFILE:qmake\CMakeFiles\qmake.dir\Debug/intermediate.manifest qmake\CMakeFiles\qmake.dir\Debug/manifest.res" failed (exit code 1104) with the following output:
Microsoft (R) Incremental Linker Version 14.23.28105.4
Copyright (C) Microsoft Corporation. All rights reserved.
LINK : fatal error LNK1104: cannot open file 'CMakeFiles\qmake.rsp'
ninja: build stopped: subcommand failed.
As you can see both the Debug
and RelWithDebInfo
configurations of use the same response file path @CMakeFiles\qmake.rsp
, and I assume there's a race condition on deleting / writing to it?
With a minimal project
cmake_minimum_required(VERSION 3.17)
set(CMAKE_CONFIGURATION_TYPES "Release;Debug" CACHE STRING "" FORCE)
set(CMAKE_CROSS_CONFIGS "all" CACHE STRING "" FORCE)
set(CMAKE_DEFAULT_CONFIGS "all" CACHE STRING "" FORCE)
project(some_project LANGUAGES CXX)
add_library(mylib SHARED)
foreach(i RANGE 100)
set(path "${CMAKE_CURRENT_BINARY_DIR}/fooaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa${i}.cpp")
file(WRITE "${path}" "void foo${i}() {}")
target_sources(mylib PRIVATE "${path}")
endforeach()
Here is an excerpt from build.ninja
build Debug\mylib.dll Debug\mylib.lib: CXX_SHARED_LIBRARY_LINKER__mylib_Debug CMakeFiles\mylib.dir\Debug\fooaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa0.cpp.obj CMakeFiles\mylib.dir\Debug\fooaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1.cpp.obj .....
LANGUAGE_COMPILE_FLAGS = /DWIN32 /D_WINDOWS /W3 /GR /EHsc /MDd /Zi /Ob0 /Od /RTC1 -MDd
LINK_FLAGS = /machine:x64 /debug /INCREMENTAL
LINK_LIBRARIES = kernel32.lib user32.lib gdi32.lib winspool.lib shell32.lib ole32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib
OBJECT_DIR = CMakeFiles\mylib.dir\Debug
POST_BUILD = cd .
PRE_LINK = cd .
RESTAT = 1
TARGET_COMPILE_PDB = CMakeFiles\mylib.dir\Debug\
TARGET_FILE = Debug\mylib.dll
TARGET_IMPLIB = Debug\mylib.lib
TARGET_PDB = Debug\mylib.pdb
RSP_FILE = CMakeFiles\mylib.rsp
build Release\mylib.dll Release\mylib.lib: CXX_SHARED_LIBRARY_LINKER__mylib_Release CMakeFiles\mylib.dir\Release\fooaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa0.cpp.obj .....
LANGUAGE_COMPILE_FLAGS = /DWIN32 /D_WINDOWS /W3 /GR /EHsc /MD /O2 /Ob2 /DNDEBUG -MD
LINK_FLAGS = /machine:x64 /INCREMENTAL:NO
LINK_LIBRARIES = kernel32.lib user32.lib gdi32.lib winspool.lib shell32.lib ole32.lib oleaut32.lib uuid.lib comdlg32.lib advapi32.lib
OBJECT_DIR = CMakeFiles\mylib.dir\Release
POST_BUILD = cd .
PRE_LINK = cd .
RESTAT = 1
TARGET_COMPILE_PDB = CMakeFiles\mylib.dir\Release\
TARGET_FILE = Release\mylib.dll
TARGET_IMPLIB = Release\mylib.lib
TARGET_PDB = Release\mylib.pdb
RSP_FILE = CMakeFiles\mylib.rsp
Note also that if i place my add_library()
call into a subdirectory called subdir
which is added with add_subdirectory()
, the RSP_FILE
value is still CMakeFiles\mylib.rsp
without subdir
in the path.
This also might cause path conflicts. So I think the safest thing would be to include both the configuration and the sub-directories in the response file path.
We've also had some nasty runtime issues where Release and Debug configurations of libraries were mixed at linking time, which I thought might be related to either the response files or something similar, but I couldn't yet find enough evidence to confirm that.