buf::oom tests: use custom allocator for oom failures

Create a custom allocator for the `buf::oom` tests that will fail with
out-of-memory errors in predictable ways.  We were previously trying to
guess the way that various allocators on various platforms would fail
in a way such that `malloc`/`realloc` would return `NULL` (instead of
aborting the application, or appearing suspicious to various
instrumentation or static code analysis tools like valgrind.)

Introduce a fake `malloc` and `realloc` that will return `NULL` on
allocations requesting more than 100 bytes.  Otherwise, we proxy to the
default allocator.  (It's important to use the _default_ allocator, not
just call `malloc`, since the default allocator on Windows CI builds may
be the debugging C runtime allocators which would not be compatible with
a standard `malloc`.)

Author: ethomson

tests/buf/oom.c

@@ -1,57 +1,59 @@
 #include "clar_libgit2.h"
 #include "buffer.h"
 
-/*
- * We want to use some ridiculous size that `malloc` will fail with
- * but that does not otherwise interfere with testing.  On Linux, choose
- * a number that is large enough to fail immediately but small enough
- * that valgrind doesn't believe it to erroneously be a negative number.
- * On macOS, choose a number that is large enough to fail immediately
- * without having libc print warnings to stderr.
- */
-#if defined(GIT_ARCH_64) && defined(__linux__)
-# define TOOBIG 0x0fffffffffffffff
-#elif defined(GIT_ARCH_64)
-# define TOOBIG 0xffffffffffffff00
-#endif
-
-/**
- * If we make a ridiculously large request the first time we
- * actually allocate some space in the git_buf, the realloc()
- * will fail.  And because the git_buf_grow() wrapper always
- * sets mark_oom, the code in git_buf_try_grow() will free
- * the internal buffer and set it to git_buf__oom.
- *
- * We initialized the internal buffer to (the static variable)
- * git_buf__initbuf.  The purpose of this test is to make sure
- * that we don't try to free the static buffer.
- *
- * Skip this test entirely on 32-bit platforms; a buffer large enough
- * to guarantee malloc failures is so large that valgrind considers
- * it likely to be an error.
- */
+/* Override default allocators with ones that will fail predictably. */
+
+static git_allocator std_alloc;
+static git_allocator oom_alloc;
+
+static void *oom_malloc(size_t n, const char *file, int line)
+{
+	/* Reject any allocation of more than 100 bytes */
+	return (n > 100) ? NULL : std_alloc.gmalloc(n, file, line);
+}
+
+static void *oom_realloc(void *p, size_t n, const char *file, int line)
+{
+	/* Reject any allocation of more than 100 bytes */
+	return (n > 100) ? NULL : std_alloc.grealloc(p, n, file, line);
+}
+
+void test_buf_oom__initialize(void)
+{
+	git_stdalloc_init_allocator(&std_alloc);
+	git_stdalloc_init_allocator(&oom_alloc);
+
+	oom_alloc.gmalloc = oom_malloc;
+	oom_alloc.grealloc = oom_realloc;
+
+	cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ALLOCATOR, &oom_alloc));
+}
+
+void test_buf_oom__cleanup(void)
+{
+	cl_git_pass(git_libgit2_opts(GIT_OPT_SET_ALLOCATOR, NULL));
+}
+
 void test_buf_oom__grow(void)
 {
-#ifdef GIT_ARCH_64
 	git_buf buf = GIT_BUF_INIT;
 
-	git_buf_clear(&buf);
+	cl_git_pass(git_buf_grow(&buf, 42));
+	cl_assert(!git_buf_oom(&buf));
 
-	cl_assert(git_buf_grow(&buf, TOOBIG) == -1);
+	cl_assert(git_buf_grow(&buf, 101) == -1);
 	cl_assert(git_buf_oom(&buf));
 
 	git_buf_dispose(&buf);
-#else
-    cl_skip();
-#endif
 }
 
 void test_buf_oom__grow_by(void)
 {
 	git_buf buf = GIT_BUF_INIT;
 
-	buf.size = SIZE_MAX-10;
+	cl_git_pass(git_buf_grow_by(&buf, 42));
+	cl_assert(!git_buf_oom(&buf));
 
-	cl_assert(git_buf_grow_by(&buf, 50) == -1);
+	cl_assert(git_buf_grow_by(&buf, 101) == -1);
 	cl_assert(git_buf_oom(&buf));
 }