Merge pull request libgit2#5857 from libgit2/ethomson/longpaths

Support `core.longpaths` on Windows

Author: ethomson

src/win32/path_w32.c

@@ -170,7 +170,7 @@ static int win32_path_cwd(wchar_t *out, size_t len)
 		 * '\'s, but we we add a 'UNC' specifier to the path, plus
 		 * a trailing directory separator, plus a NUL.
 		 */
-		if (cwd_len > MAX_PATH - 4) {
+		if (cwd_len > GIT_WIN_PATH_MAX - 4) {
 			errno = ENAMETOOLONG;
 			return -1;
 		}
@@ -187,7 +187,7 @@ static int win32_path_cwd(wchar_t *out, size_t len)
 	 * working directory.  (One character for the directory separator,
 	 * one for the null.
 	 */
-	else if (cwd_len > MAX_PATH - 2) {
+	else if (cwd_len > GIT_WIN_PATH_MAX - 2) {
 		errno = ENAMETOOLONG;
 		return -1;
 	}
@@ -205,13 +205,13 @@ int git_win32_path_from_utf8(git_win32_path out, const char *src)
 
 	/* See if this is an absolute path (beginning with a drive letter) */
 	if (git_path_is_absolute(src)) {
-		if (git__utf8_to_16(dest, MAX_PATH, src) < 0)
+		if (git__utf8_to_16(dest, GIT_WIN_PATH_MAX, src) < 0)
 			goto on_error;
 	}
 	/* File-prefixed NT-style paths beginning with \\?\ */
 	else if (path__is_nt_namespace(src)) {
 		/* Skip the NT prefix, the destination already contains it */
-		if (git__utf8_to_16(dest, MAX_PATH, src + PATH__NT_NAMESPACE_LEN) < 0)
+		if (git__utf8_to_16(dest, GIT_WIN_PATH_MAX, src + PATH__NT_NAMESPACE_LEN) < 0)
 			goto on_error;
 	}
 	/* UNC paths */
@@ -220,12 +220,12 @@ int git_win32_path_from_utf8(git_win32_path out, const char *src)
 		dest += 4;
 
 		/* Skip the leading "\\" */
-		if (git__utf8_to_16(dest, MAX_PATH - 2, src + 2) < 0)
+		if (git__utf8_to_16(dest, GIT_WIN_PATH_MAX - 2, src + 2) < 0)
 			goto on_error;
 	}
 	/* Absolute paths omitting the drive letter */
 	else if (path__startswith_slash(src)) {
-		if (path__cwd(dest, MAX_PATH) < 0)
+		if (path__cwd(dest, GIT_WIN_PATH_MAX) < 0)
 			goto on_error;
 
 		if (!git_path_is_absolute(dest)) {
@@ -234,19 +234,19 @@ int git_win32_path_from_utf8(git_win32_path out, const char *src)
 		}
 
 		/* Skip the drive letter specification ("C:") */
-		if (git__utf8_to_16(dest + 2, MAX_PATH - 2, src) < 0)
+		if (git__utf8_to_16(dest + 2, GIT_WIN_PATH_MAX - 2, src) < 0)
 			goto on_error;
 	}
 	/* Relative paths */
 	else {
 		int cwd_len;
 
-		if ((cwd_len = win32_path_cwd(dest, MAX_PATH)) < 0)
+		if ((cwd_len = win32_path_cwd(dest, GIT_WIN_PATH_MAX)) < 0)
 			goto on_error;
 
 		dest[cwd_len++] = L'\\';
 
-		if (git__utf8_to_16(dest + cwd_len, MAX_PATH - cwd_len, src) < 0)
+		if (git__utf8_to_16(dest + cwd_len, GIT_WIN_PATH_MAX - cwd_len, src) < 0)
 			goto on_error;
 	}
 
@@ -273,7 +273,7 @@ int git_win32_path_relative_from_utf8(git_win32_path out, const char *src)
 		return git_win32_path_from_utf8(out, src);
 	}
 
-	if ((len = git__utf8_to_16(dest, MAX_PATH, src)) < 0)
+	if ((len = git__utf8_to_16(dest, GIT_WIN_PATH_MAX, src)) < 0)
 		return -1;
 
 	for (p = dest; p < (dest + len); p++) {

src/win32/w32_common.h

@@ -8,24 +8,33 @@
 #ifndef INCLUDE_win32_w32_common_h__
 #define INCLUDE_win32_w32_common_h__
 
+#include <git2/common.h>
+
+/*
+ * 4096 is the max allowed Git path. `MAX_PATH` (260) is the typical max allowed
+ * Windows path length, however win32 Unicode APIs generally allow up to 32,767
+ * if prefixed with "\\?\" (i.e. converted to an NT-style name).
+ */
+#define GIT_WIN_PATH_MAX GIT_PATH_MAX
+
 /*
- * Provides a large enough buffer to support Windows paths:  MAX_PATH is
- * 260, corresponding to a maximum path length of 259 characters plus a
- * NULL terminator.  Prefixing with "\\?\" adds 4 characters, but if the
- * original was a UNC path, then we turn "\\server\share" into
+ * Provides a large enough buffer to support Windows Git paths:
+ * GIT_WIN_PATH_MAX is 4096, corresponding to a maximum path length of 4095
+ * characters plus a NULL terminator.  Prefixing with "\\?\" adds 4 characters,
+ * but if the original was a UNC path, then we turn "\\server\share" into
  * "\\?\UNC\server\share".  So we replace the first two characters with
- * 8 characters, a net gain of 6, so the maximum length is MAX_PATH+6.
+ * 8 characters, a net gain of 6, so the maximum length is GIT_WIN_PATH_MAX+6.
  */
-#define GIT_WIN_PATH_UTF16		MAX_PATH+6
+#define GIT_WIN_PATH_UTF16		GIT_WIN_PATH_MAX+6
 
-/* Maximum size of a UTF-8 Win32 path.  We remove the "\\?\" or "\\?\UNC\"
- * prefixes for presentation, bringing us back to 259 (non-NULL)
+/* Maximum size of a UTF-8 Win32 Git path.  We remove the "\\?\" or "\\?\UNC\"
+ * prefixes for presentation, bringing us back to 4095 (non-NULL)
  * characters.  UTF-8 does have 4-byte sequences, but they are encoded in
  * UTF-16 using surrogate pairs, which takes up the space of two characters.
  * Two characters in the range U+0800 -> U+FFFF take up more space in UTF-8
  * (6 bytes) than one surrogate pair (4 bytes).
  */
-#define GIT_WIN_PATH_UTF8		(259 * 3 + 1)
+#define GIT_WIN_PATH_UTF8		((GIT_WIN_PATH_MAX - 1) * 3 + 1)
 
 /*
  * The length of a Windows "shortname", for 8.3 compatibility.

tests/CMakeLists.txt

@@ -9,6 +9,7 @@ SET(CLAR_FIXTURES "${CMAKE_CURRENT_SOURCE_DIR}/resources/")
 SET(CLAR_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
 ADD_DEFINITIONS(-DCLAR_FIXTURE_PATH=\"${CLAR_FIXTURES}\")
 ADD_DEFINITIONS(-DCLAR_TMPDIR=\"libgit2_tests\")
+ADD_DEFINITIONS(-DCLAR_WIN32_LONGPATHS)
 ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
 
 # Ensure that we do not use deprecated functions internally

tests/clar/fs.h

@@ -8,6 +8,12 @@
 
 #ifdef _WIN32
 
+#ifdef CLAR_WIN32_LONGPATHS
+# define CLAR_MAX_PATH 4096
+#else
+# define CLAR_MAX_PATH MAX_PATH
+#endif
+
 #define RM_RETRY_COUNT	5
 #define RM_RETRY_DELAY	10
 
@@ -48,21 +54,48 @@ fs_rmdir_rmdir(WCHAR *_wpath)
 	return 0;
 }
 
+static void translate_path(WCHAR *path, size_t path_size)
+{
+    size_t path_len, i;
+
+    if (wcsncmp(path, L"\\\\?\\", 4) == 0)
+	return;
+
+    path_len = wcslen(path);
+    cl_assert(path_size > path_len + 4);
+
+    for (i = path_len; i > 0; i--) {
+	WCHAR c = path[i - 1];
+
+	if (c == L'/')
+	    path[i + 3] = L'\\';
+	else
+	    path[i + 3] = path[i - 1];
+    }
+
+    path[0] = L'\\';
+    path[1] = L'\\';
+    path[2] = L'?';
+    path[3] = L'\\';
+    path[path_len + 4] = L'\0';
+}
+
 static void
 fs_rmdir_helper(WCHAR *_wsource)
 {
-	WCHAR buffer[MAX_PATH];
+	WCHAR buffer[CLAR_MAX_PATH];
 	HANDLE find_handle;
 	WIN32_FIND_DATAW find_data;
 	size_t buffer_prefix_len;
 
 	/* Set up the buffer and capture the length */
-	wcscpy_s(buffer, MAX_PATH, _wsource);
-	wcscat_s(buffer, MAX_PATH, L"\\");
+	wcscpy_s(buffer, CLAR_MAX_PATH, _wsource);
+	translate_path(buffer, CLAR_MAX_PATH);
+	wcscat_s(buffer, CLAR_MAX_PATH, L"\\");
 	buffer_prefix_len = wcslen(buffer);
 
 	/* FindFirstFile needs a wildcard to match multiple items */
-	wcscat_s(buffer, MAX_PATH, L"*");
+	wcscat_s(buffer, CLAR_MAX_PATH, L"*");
 	find_handle = FindFirstFileW(buffer, &find_data);
 	cl_assert(INVALID_HANDLE_VALUE != find_handle);
 
@@ -72,7 +105,7 @@ fs_rmdir_helper(WCHAR *_wsource)
 		if (fs__dotordotdot(find_data.cFileName))
 			continue;
 
-		wcscpy_s(buffer + buffer_prefix_len, MAX_PATH - buffer_prefix_len, find_data.cFileName);
+		wcscpy_s(buffer + buffer_prefix_len, CLAR_MAX_PATH - buffer_prefix_len, find_data.cFileName);
 
 		if (FILE_ATTRIBUTE_DIRECTORY & find_data.dwFileAttributes)
 			fs_rmdir_helper(buffer);
@@ -123,7 +156,7 @@ fs_rm_wait(WCHAR *_wpath)
 static void
 fs_rm(const char *_source)
 {
-	WCHAR wsource[MAX_PATH];
+	WCHAR wsource[CLAR_MAX_PATH];
 	DWORD attrs;
 
 	/* The input path is UTF-8. Convert it to wide characters
@@ -133,7 +166,9 @@ fs_rm(const char *_source)
 				_source,
 				-1, /* Indicates NULL termination */
 				wsource,
-				MAX_PATH));
+				CLAR_MAX_PATH));
+
+	translate_path(wsource, CLAR_MAX_PATH);
 
 	/* Does the item exist? If not, we have no work to do */
 	attrs = GetFileAttributesW(wsource);
@@ -158,21 +193,23 @@ fs_rm(const char *_source)
 static void
 fs_copydir_helper(WCHAR *_wsource, WCHAR *_wdest)
 {
-	WCHAR buf_source[MAX_PATH], buf_dest[MAX_PATH];
+	WCHAR buf_source[CLAR_MAX_PATH], buf_dest[CLAR_MAX_PATH];
 	HANDLE find_handle;
 	WIN32_FIND_DATAW find_data;
 	size_t buf_source_prefix_len, buf_dest_prefix_len;
 
-	wcscpy_s(buf_source, MAX_PATH, _wsource);
-	wcscat_s(buf_source, MAX_PATH, L"\\");
+	wcscpy_s(buf_source, CLAR_MAX_PATH, _wsource);
+	wcscat_s(buf_source, CLAR_MAX_PATH, L"\\");
+	translate_path(buf_source, CLAR_MAX_PATH);
 	buf_source_prefix_len = wcslen(buf_source);
 
-	wcscpy_s(buf_dest, MAX_PATH, _wdest);
-	wcscat_s(buf_dest, MAX_PATH, L"\\");
+	wcscpy_s(buf_dest, CLAR_MAX_PATH, _wdest);
+	wcscat_s(buf_dest, CLAR_MAX_PATH, L"\\");
+	translate_path(buf_dest, CLAR_MAX_PATH);
 	buf_dest_prefix_len = wcslen(buf_dest);
 
 	/* Get an enumerator for the items in the source. */
-	wcscat_s(buf_source, MAX_PATH, L"*");
+	wcscat_s(buf_source, CLAR_MAX_PATH, L"*");
 	find_handle = FindFirstFileW(buf_source, &find_data);
 	cl_assert(INVALID_HANDLE_VALUE != find_handle);
 
@@ -185,8 +222,8 @@ fs_copydir_helper(WCHAR *_wsource, WCHAR *_wdest)
 		if (fs__dotordotdot(find_data.cFileName))
 			continue;
 
-		wcscpy_s(buf_source + buf_source_prefix_len, MAX_PATH - buf_source_prefix_len, find_data.cFileName);
-		wcscpy_s(buf_dest + buf_dest_prefix_len, MAX_PATH - buf_dest_prefix_len, find_data.cFileName);
+		wcscpy_s(buf_source + buf_source_prefix_len, CLAR_MAX_PATH - buf_source_prefix_len, find_data.cFileName);
+		wcscpy_s(buf_dest + buf_dest_prefix_len, CLAR_MAX_PATH - buf_dest_prefix_len, find_data.cFileName);
 
 		if (FILE_ATTRIBUTE_DIRECTORY & find_data.dwFileAttributes)
 			fs_copydir_helper(buf_source, buf_dest);
@@ -205,7 +242,7 @@ fs_copydir_helper(WCHAR *_wsource, WCHAR *_wdest)
 static void
 fs_copy(const char *_source, const char *_dest)
 {
-	WCHAR wsource[MAX_PATH], wdest[MAX_PATH];
+	WCHAR wsource[CLAR_MAX_PATH], wdest[CLAR_MAX_PATH];
 	DWORD source_attrs, dest_attrs;
 	HANDLE find_handle;
 	WIN32_FIND_DATAW find_data;
@@ -217,14 +254,17 @@ fs_copy(const char *_source, const char *_dest)
 				_source,
 				-1,
 				wsource,
-				MAX_PATH));
+				CLAR_MAX_PATH));
 
 	cl_assert(MultiByteToWideChar(CP_UTF8,
 				MB_ERR_INVALID_CHARS,
 				_dest,
 				-1,
 				wdest,
-				MAX_PATH));
+				CLAR_MAX_PATH));
+
+	translate_path(wsource, CLAR_MAX_PATH);
+	translate_path(wdest, CLAR_MAX_PATH);
 
 	/* Check the source for existence */
 	source_attrs = GetFileAttributesW(wsource);
@@ -238,8 +278,8 @@ fs_copy(const char *_source, const char *_dest)
 		 * Use FindFirstFile to parse the path */
 		find_handle = FindFirstFileW(wsource, &find_data);
 		cl_assert(INVALID_HANDLE_VALUE != find_handle);
-		wcscat_s(wdest, MAX_PATH, L"\\");
-		wcscat_s(wdest, MAX_PATH, find_data.cFileName);
+		wcscat_s(wdest, CLAR_MAX_PATH, L"\\");
+		wcscat_s(wdest, CLAR_MAX_PATH, find_data.cFileName);
 		FindClose(find_handle);
 
 		/* Check the new target for existence */

tests/path/win32.c

@@ -78,13 +78,11 @@ void test_path_win32__honors_max_path(void)
 #ifdef GIT_WIN32
 	git_win32_path path_utf16;
 
-	test_utf8_to_utf16("C:\\This path is 259 chars and is the max length in windows\\0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij",
-		L"\\\\?\\C:\\This path is 259 chars and is the max length in windows\\0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij");
-	test_utf8_to_utf16("\\\\unc\\paths may also be 259 characters including the server\\123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij",
-		L"\\\\?\\UNC\\unc\\paths may also be 259 characters including the server\\123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij");
+	test_utf8_to_utf16("C:\\This path is 261 characters which is fine for our path handling functions which cope with paths longer than MAX_PATH\\0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghijk",
+		L"\\\\?\\C:\\This path is 261 characters which is fine for our path handling functions which cope with paths longer than MAX_PATH\\0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghijk");
+
+	cl_check_fail(git_win32_path_from_utf8(path_utf16, "C:\\This path is 4097 chars and exceeds our maximum path length on Windows which is limited to 4096 characters\\alas\\0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij01"));
 
-	cl_check_fail(git_win32_path_from_utf8(path_utf16, "C:\\This path is 260 chars and is sadly too long for windows\\0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij"));
-	cl_check_fail(git_win32_path_from_utf8(path_utf16, "\\\\unc\\paths are also bound by 260 character restrictions\\including the server name portion\\bcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij0123456789abcdefghij"));
 #endif
 }