Commit 8f991ab0 authored by Chuck Atkins's avatar Chuck Atkins Committed by Brad King
Browse files

SystemTools: Use extended paths on Windows for > 256 length

Many Windows filesystem calls (but not all) have a MAX_PATH limitation
of 260 characters (drive letter, colon, backslash, 256 char path, null).
This is especially problematic for interacting with large highly nested
build trees (the boost C++ libraries, for example). This limitation can
be overcome by using extended length paths instead.  By converting
local filesystem and network paths to extended length paths before
passing them to the underlying APIs the maximum path length can be as
large as 32767 characters. The new ConvertToWindowsExtendedPath
function will convert "E:/a.txt" to "\\?\E:\a.txt" and "\\Foo\a.txt" to
"\\?\UNC\Foo\a.txt".

See also the MSDN article on "Naming Files, Paths, and Namespaces":

 http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx

This also adds a hidden CMake option, KWSYS_TEST_SYSTEMTOOLS_LONG_PATHS,
which, when set to TRUE, will enable the extended length path tests.
This is off by default since if something does go awry, the resulting
leftovers could be difficult to delete manually.

Change-Id: Ib7ac1f657241ed389169678d1c078c0d836f1c7b
parent 97817ce7
......@@ -193,11 +193,13 @@ static inline char *realpath(const char *path, char *resolved_path)
inline int Mkdir(const char* dir)
{
return _wmkdir(KWSYS_NAMESPACE::Encoding::ToWide(dir).c_str());
return _wmkdir(
KWSYS_NAMESPACE::SystemTools::ConvertToWindowsExtendedPath(dir).c_str());
}
inline int Rmdir(const char* dir)
{
return _wrmdir(KWSYS_NAMESPACE::Encoding::ToWide(dir).c_str());
return _wrmdir(
KWSYS_NAMESPACE::SystemTools::ConvertToWindowsExtendedPath(dir).c_str());
}
inline const char* Getcwd(char* buf, unsigned int len)
{
......@@ -609,7 +611,7 @@ const char* SystemTools::GetExecutableExtension()
FILE* SystemTools::Fopen(const char* file, const char* mode)
{
#ifdef _WIN32
return _wfopen(Encoding::ToWide(file).c_str(),
return _wfopen(SystemTools::ConvertToWindowsExtendedPath(file).c_str(),
Encoding::ToWide(mode).c_str());
#else
return fopen(file, mode);
......@@ -1081,7 +1083,8 @@ bool SystemTools::FileExists(const char* filename)
}
return access(filename, R_OK) == 0;
#elif defined(_WIN32)
return (GetFileAttributesW(Encoding::ToWide(filename).c_str())
return (GetFileAttributesW(
SystemTools::ConvertToWindowsExtendedPath(filename).c_str())
!= INVALID_FILE_ATTRIBUTES);
#else
return access(filename, R_OK) == 0;
......@@ -1137,10 +1140,11 @@ bool SystemTools::Touch(const char* filename, bool create)
return false;
}
#if defined(_WIN32) && !defined(__CYGWIN__)
HANDLE h = CreateFileW(Encoding::ToWide(filename).c_str(),
FILE_WRITE_ATTRIBUTES,
FILE_SHARE_WRITE, 0, OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, 0);
HANDLE h = CreateFileW(
SystemTools::ConvertToWindowsExtendedPath(filename).c_str(),
FILE_WRITE_ATTRIBUTES,
FILE_SHARE_WRITE, 0, OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, 0);
if(!h)
{
return false;
......@@ -1242,13 +1246,15 @@ bool SystemTools::FileTimeCompare(const char* f1, const char* f2,
// Windows version. Get the modification time from extended file attributes.
WIN32_FILE_ATTRIBUTE_DATA f1d;
WIN32_FILE_ATTRIBUTE_DATA f2d;
if(!GetFileAttributesExW(Encoding::ToWide(f1).c_str(),
GetFileExInfoStandard, &f1d))
if(!GetFileAttributesExW(
SystemTools::ConvertToWindowsExtendedPath(f1).c_str(),
GetFileExInfoStandard, &f1d))
{
return false;
}
if(!GetFileAttributesExW(Encoding::ToWide(f2).c_str(),
GetFileExInfoStandard, &f2d))
if(!GetFileAttributesExW(
SystemTools::ConvertToWindowsExtendedPath(f2).c_str(),
GetFileExInfoStandard, &f2d))
{
return false;
}
......@@ -1830,6 +1836,71 @@ void SystemTools::ConvertToUnixSlashes(kwsys_stl::string& path)
}
}
#ifdef _WIN32
// Convert local paths to UNC style paths
kwsys_stl::wstring
SystemTools::ConvertToWindowsExtendedPath(const kwsys_stl::string &source)
{
kwsys_stl::wstring wsource = Encoding::ToWide(source);
// Resolve any relative paths
DWORD wfull_len;
/* The +3 is a workaround for a bug in some versions of GetFullPathNameW that
* won't return a large enough buffer size if the input is too small */
wfull_len = GetFullPathNameW(wsource.c_str(), 0, NULL, NULL) + 3;
kwsys_stl::vector<wchar_t> wfull(wfull_len);
GetFullPathNameW(wsource.c_str(), wfull_len, &wfull[0], NULL);
/* This should get the correct size without any extra padding from the
* previous size workaround. */
wfull_len = static_cast<DWORD>(wcslen(&wfull[0]));
if(wfull_len >= 2 && isalpha(wfull[0]) && wfull[1] == L':')
{ /* C:\Foo\bar\FooBar.txt */
return L"\\\\?\\" + kwsys_stl::wstring(&wfull[0]);
}
else if(wfull_len >= 2 && wfull[0] == L'\\' && wfull[1] == L'\\')
{ /* Starts with \\ */
if(wfull_len >= 4 && wfull[2] == L'?' && wfull[3] == L'\\')
{ /* Starts with \\?\ */
if(wfull_len >= 8 && wfull[4] == L'U' && wfull[5] == L'N' &&
wfull[6] == L'C' && wfull[7] == L'\\')
{ /* \\?\UNC\Foo\bar\FooBar.txt */
return kwsys_stl::wstring(&wfull[0]);
}
else if(wfull_len >= 6 && isalpha(wfull[4]) && wfull[5] == L':')
{ /* \\?\C:\Foo\bar\FooBar.txt */
return kwsys_stl::wstring(&wfull[0]);
}
else if(wfull_len >= 5)
{ /* \\?\Foo\bar\FooBar.txt */
return L"\\\\?\\UNC\\" + kwsys_stl::wstring(&wfull[4]);
}
}
else if(wfull_len >= 4 && wfull[2] == L'.' && wfull[3] == L'\\')
{ /* Starts with \\.\ a device name */
if(wfull_len >= 6 && isalpha(wfull[4]) && wfull[5] == L':')
{ /* \\.\C:\Foo\bar\FooBar.txt */
return L"\\\\?\\" + kwsys_stl::wstring(&wfull[4]);
}
else if(wfull_len >= 5)
{ /* \\.\Foo\bar\ Device name is left unchanged */
return kwsys_stl::wstring(&wfull[0]);
}
}
else if(wfull_len >= 3)
{ /* \\Foo\bar\FooBar.txt */
return L"\\\\?\\UNC\\" + kwsys_stl::wstring(&wfull[2]);
}
}
// If this case has been reached, then the path is invalid. Leave it
// unchanged
return Encoding::ToWide(source);
}
#endif
// change // to /, and escape any spaces in the path
kwsys_stl::string SystemTools::ConvertToUnixOutputPath(const char* path)
{
......@@ -1960,17 +2031,19 @@ bool SystemTools::FilesDiffer(const char* source,
#if defined(_WIN32)
WIN32_FILE_ATTRIBUTE_DATA statSource;
if (GetFileAttributesExW(Encoding::ToWide(source).c_str(),
GetFileExInfoStandard,
&statSource) == 0)
if (GetFileAttributesExW(
SystemTools::ConvertToWindowsExtendedPath(source).c_str(),
GetFileExInfoStandard,
&statSource) == 0)
{
return true;
}
WIN32_FILE_ATTRIBUTE_DATA statDestination;
if (GetFileAttributesExW(Encoding::ToWide(destination).c_str(),
GetFileExInfoStandard,
&statDestination) == 0)
if (GetFileAttributesExW(
SystemTools::ConvertToWindowsExtendedPath(destination).c_str(),
GetFileExInfoStandard,
&statDestination) == 0)
{
return true;
}
......@@ -2191,7 +2264,12 @@ bool SystemTools::CopyADirectory(const char* source, const char* destination,
bool always)
{
Directory dir;
#ifdef _WIN32
dir.Load(Encoding::ToNarrow(
SystemTools::ConvertToWindowsExtendedPath(source)).c_str());
#else
dir.Load(source);
#endif
size_t fileNum;
if ( !SystemTools::MakeDirectory(destination) )
{
......@@ -2234,14 +2312,27 @@ bool SystemTools::CopyADirectory(const char* source, const char* destination,
// return size of file; also returns zero if no file exists
unsigned long SystemTools::FileLength(const char* filename)
{
#ifdef _WIN32
WIN32_FILE_ATTRIBUTE_DATA fs;
if (GetFileAttributesExW(
SystemTools::ConvertToWindowsExtendedPath(filename).c_str(),
GetFileExInfoStandard, &fs) == TRUE)
{
/* To support the full 64-bit file size, use fs.nFileSizeHigh
* and fs.nFileSizeLow to construct the 64 bit size
*/
return static_cast<unsigned long>(fs.nFileSizeLow);
}
#else
struct stat fs;
if (stat(filename, &fs) != 0)
if (stat(filename, &fs) == 0)
{
return 0;
return static_cast<unsigned long>(fs.st_size);
}
#endif
else
{
return static_cast<unsigned long>(fs.st_size);
return 0;
}
}
......@@ -2406,7 +2497,8 @@ bool SystemTools::RemoveFile(const char* source)
SystemTools::SetPermissions(source, S_IWRITE);
#endif
#ifdef _WIN32
bool res = _wunlink(Encoding::ToWide(source).c_str()) != 0 ? false : true;
bool res =
_wunlink(SystemTools::ConvertToWindowsExtendedPath(source).c_str()) == 0;
#else
bool res = unlink(source) != 0 ? false : true;
#endif
......@@ -2435,7 +2527,12 @@ bool SystemTools::RemoveADirectory(const char* source)
}
Directory dir;
#ifdef _WIN32
dir.Load(Encoding::ToNarrow(
SystemTools::ConvertToWindowsExtendedPath(source)).c_str());
#else
dir.Load(source);
#endif
size_t fileNum;
for (fileNum = 0; fileNum < dir.GetNumberOfFiles(); ++fileNum)
{
......@@ -2849,7 +2946,8 @@ bool SystemTools::FileIsDirectory(const char* name)
// Now check the file node type.
#if defined( _WIN32 )
DWORD attr = GetFileAttributesW(Encoding::ToWide(name).c_str());
DWORD attr = GetFileAttributesW(
SystemTools::ConvertToWindowsExtendedPath(name).c_str());
if (attr != INVALID_FILE_ATTRIBUTES)
{
return (attr & FILE_ATTRIBUTE_DIRECTORY) != 0;
......@@ -4041,7 +4139,7 @@ bool SystemTools::GetShortPath(const kwsys_stl::string& path, kwsys_stl::string&
kwsys_stl::wstring wtempPath = Encoding::ToWide(tempPath);
kwsys_stl::vector<wchar_t> buffer(wtempPath.size()+1);
buffer[0] = 0;
ret = GetShortPathNameW(Encoding::ToWide(tempPath).c_str(),
ret = GetShortPathNameW(wtempPath.c_str(),
&buffer[0], static_cast<DWORD>(wtempPath.size()));
if(buffer[0] == 0 || ret > wtempPath.size())
......@@ -4279,7 +4377,8 @@ bool SystemTools::GetPermissions(const char* file, mode_t& mode)
}
#if defined(_WIN32)
DWORD attr = GetFileAttributesW(Encoding::ToWide(file).c_str());
DWORD attr = GetFileAttributesW(
SystemTools::ConvertToWindowsExtendedPath(file).c_str());
if(attr == INVALID_FILE_ATTRIBUTES)
{
return false;
......@@ -4331,7 +4430,8 @@ bool SystemTools::SetPermissions(const char* file, mode_t mode)
return false;
}
#ifdef _WIN32
if ( _wchmod(Encoding::ToWide(file).c_str(), mode) < 0 )
if ( _wchmod(SystemTools::ConvertToWindowsExtendedPath(file).c_str(),
mode) < 0 )
#else
if ( chmod(file, mode) < 0 )
#endif
......
......@@ -249,7 +249,18 @@ public:
* Replace Windows file system slashes with Unix-style slashes.
*/
static void ConvertToUnixSlashes(kwsys_stl::string& path);
#ifdef _WIN32
/**
* Convert the path to an extended length path to avoid MAX_PATH length
* limitations on Windows. If the input is a local path the result will be
* prefixed with \\?\; if the input is instead a network path, the result
* will be prefixed with \\?\UNC\. All output will also be converted to
* absolute paths with Windows-style backslashes.
**/
static kwsys_stl::wstring ConvertToWindowsExtendedPath(const kwsys_stl::string&);
#endif
/**
* For windows this calls ConvertToWindowsOutputPath and for unix
* it calls ConvertToUnixOutputPath
......
......@@ -102,6 +102,9 @@ static bool CheckFileOperations()
"/testSystemTools.bin");
const kwsys_stl::string testTxtFile(TEST_SYSTEMTOOLS_SOURCE_DIR
"/testSystemTools.cxx");
const kwsys_stl::string testNewDir(TEST_SYSTEMTOOLS_BINARY_DIR
"/testSystemToolsNewDir");
const kwsys_stl::string testNewFile(testNewDir + "/testNewFile.txt");
if (kwsys::SystemTools::DetectFileType(testBinFile.c_str()) !=
kwsys::SystemTools::FileTypeBinary)
......@@ -129,6 +132,91 @@ static bool CheckFileOperations()
res = false;
}
if (!kwsys::SystemTools::MakeDirectory(testNewDir.c_str()))
{
kwsys_ios::cerr
<< "Problem with MakeDirectory for: "
<< testNewDir << kwsys_ios::endl;
res = false;
}
if (!kwsys::SystemTools::Touch(testNewFile.c_str(), true))
{
kwsys_ios::cerr
<< "Problem with Touch for: "
<< testNewFile << kwsys_ios::endl;
res = false;
}
if (!kwsys::SystemTools::RemoveFile(testNewFile.c_str()))
{
kwsys_ios::cerr
<< "Problem with RemoveFile: "
<< testNewFile << kwsys_ios::endl;
res = false;
}
kwsys::SystemTools::Touch(testNewFile.c_str(), true);
if (!kwsys::SystemTools::RemoveADirectory(testNewDir.c_str()))
{
kwsys_ios::cerr
<< "Problem with RemoveADirectory for: "
<< testNewDir << kwsys_ios::endl;
res = false;
}
#ifdef KWSYS_TEST_SYSTEMTOOLS_LONG_PATHS
// Perform the same file and directory creation and deletion tests but
// with paths > 256 characters in length.
const kwsys_stl::string testNewLongDir(
TEST_SYSTEMTOOLS_BINARY_DIR "/"
"012345678901234567890123456789012345678901234567890123456789"
"012345678901234567890123456789012345678901234567890123456789"
"012345678901234567890123456789012345678901234567890123456789"
"012345678901234567890123456789012345678901234567890123456789"
"01234567890123");
const kwsys_stl::string testNewLongFile(testNewLongDir + "/"
"012345678901234567890123456789012345678901234567890123456789"
"012345678901234567890123456789012345678901234567890123456789"
"012345678901234567890123456789012345678901234567890123456789"
"012345678901234567890123456789012345678901234567890123456789"
"0123456789.txt");
if (!kwsys::SystemTools::MakeDirectory(testNewLongDir.c_str()))
{
kwsys_ios::cerr
<< "Problem with MakeDirectory for: "
<< testNewLongDir << kwsys_ios::endl;
res = false;
}
if (!kwsys::SystemTools::Touch(testNewLongFile.c_str(), true))
{
kwsys_ios::cerr
<< "Problem with Touch for: "
<< testNewLongFile << kwsys_ios::endl;
res = false;
}
if (!kwsys::SystemTools::RemoveFile(testNewLongFile.c_str()))
{
kwsys_ios::cerr
<< "Problem with RemoveFile: "
<< testNewLongFile << kwsys_ios::endl;
res = false;
}
kwsys::SystemTools::Touch(testNewLongFile.c_str(), true);
if (!kwsys::SystemTools::RemoveADirectory(testNewLongDir.c_str()))
{
kwsys_ios::cerr
<< "Problem with RemoveADirectory for: "
<< testNewLongDir << kwsys_ios::endl;
res = false;
}
#endif
return res;
}
......@@ -279,6 +367,113 @@ static bool CheckStringOperations()
res = false;
}
#ifdef _WIN32
if (kwsys::SystemTools::ConvertToWindowsExtendedPath
("L:\\Local Mojo\\Hex Power Pack\\Iffy Voodoo") !=
L"\\\\?\\L:\\Local Mojo\\Hex Power Pack\\Iffy Voodoo")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"L:\\Local Mojo\\Hex Power Pack\\Iffy Voodoo\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath
("L:/Local Mojo/Hex Power Pack/Iffy Voodoo") !=
L"\\\\?\\L:\\Local Mojo\\Hex Power Pack\\Iffy Voodoo")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"L:/Local Mojo/Hex Power Pack/Iffy Voodoo\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath
("\\\\Foo\\Local Mojo\\Hex Power Pack\\Iffy Voodoo") !=
L"\\\\?\\UNC\\Foo\\Local Mojo\\Hex Power Pack\\Iffy Voodoo")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"\\\\Foo\\Local Mojo\\Hex Power Pack\\Iffy Voodoo\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath
("//Foo/Local Mojo/Hex Power Pack/Iffy Voodoo") !=
L"\\\\?\\UNC\\Foo\\Local Mojo\\Hex Power Pack\\Iffy Voodoo")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"//Foo/Local Mojo/Hex Power Pack/Iffy Voodoo\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath("//") !=
L"//")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"//\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath("\\\\.\\") !=
L"\\\\.\\")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"\\\\.\\\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath("\\\\.\\X") !=
L"\\\\.\\X")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"\\\\.\\X\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath("\\\\.\\X:") !=
L"\\\\?\\X:")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"\\\\.\\X:\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath("\\\\.\\X:\\") !=
L"\\\\?\\X:\\")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"\\\\.\\X:\\\""
<< kwsys_ios::endl;
res = false;
}
if (kwsys::SystemTools::ConvertToWindowsExtendedPath("NUL") !=
L"\\\\.\\NUL")
{
kwsys_ios::cerr
<< "Problem with ConvertToWindowsExtendedPath "
<< "\"NUL\""
<< kwsys_ios::endl;
res = false;
}
#endif
if (kwsys::SystemTools::ConvertToWindowsOutputPath
("L://Local Mojo/Hex Power Pack/Iffy Voodoo") !=
"\"L:\\Local Mojo\\Hex Power Pack\\Iffy Voodoo\"")
......
......@@ -16,5 +16,6 @@
#define TEST_SYSTEMTOOLS_SOURCE_DIR "@TEST_SYSTEMTOOLS_SOURCE_DIR@"
#define TEST_SYSTEMTOOLS_BINARY_DIR "@TEST_SYSTEMTOOLS_BINARY_DIR@"
#cmakedefine KWSYS_TEST_SYSTEMTOOLS_LONG_PATHS
#endif
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment