merge: check return value of git_commit_list_insert

The function `git_commit_list_insert` dynamically allocates memory and
may thus fail to insert a given commit, but we didn't check for that in
several places in "merge.c".

Convert surrounding functions to return error codes and check whether
`git_commit_list_insert` was successful, returning an error if not.

Author: pks-t

src/merge.c

@@ -310,46 +310,55 @@ static int interesting(git_pqueue *list)
 	return 0;
 }
 
-static void clear_commit_marks_1(git_commit_list **plist,
+static int clear_commit_marks_1(git_commit_list **plist,
 		git_commit_list_node *commit, unsigned int mark)
 {
 	while (commit) {
 		unsigned int i;
 
 		if (!(mark & commit->flags))
-			return;
+			return 0;
 
 		commit->flags &= ~mark;
 
 		for (i = 1; i < commit->out_degree; i++) {
 			git_commit_list_node *p = commit->parents[i];
-			git_commit_list_insert(p, plist);
+			if (git_commit_list_insert(p, plist) == NULL)
+				return -1;
 		}
 
 		commit = commit->out_degree ? commit->parents[0] : NULL;
 	}
+
+	return 0;
 }
 
-static void clear_commit_marks_many(git_vector *commits, unsigned int mark)
+static int clear_commit_marks_many(git_vector *commits, unsigned int mark)
 {
 	git_commit_list *list = NULL;
 	git_commit_list_node *c;
 	unsigned int i;
 
 	git_vector_foreach(commits, i, c) {
-		git_commit_list_insert(c, &list);
+		if (git_commit_list_insert(c, &list) == NULL)
+			return -1;
 	}
 
 	while (list)
-		clear_commit_marks_1(&list, git_commit_list_pop(&list), mark);
+		if (clear_commit_marks_1(&list, git_commit_list_pop(&list), mark) < 0)
+			return -1;
+	return 0;
 }
 
-static void clear_commit_marks(git_commit_list_node *commit, unsigned int mark)
+static int clear_commit_marks(git_commit_list_node *commit, unsigned int mark)
 {
 	git_commit_list *list = NULL;
-	git_commit_list_insert(commit, &list);
+	if (git_commit_list_insert(commit, &list) == NULL)
+		return -1;
 	while (list)
-		clear_commit_marks_1(&list, git_commit_list_pop(&list), mark);
+		if (clear_commit_marks_1(&list, git_commit_list_pop(&list), mark) < 0)
+			return -1;
+	return 0;
 }
 
 static int paint_down_to_common(
@@ -466,10 +475,11 @@ static int remove_redundant(git_revwalk *walk, git_vector *commits)
 				redundant[filled_index[j]] = 1;
 		}
 
-		clear_commit_marks(commit, ALL_FLAGS);
-		clear_commit_marks_many(&work, ALL_FLAGS);
-
 		git_commit_list_free(&common);
+
+		if ((error = clear_commit_marks(commit, ALL_FLAGS)) < 0 ||
+		    (error = clear_commit_marks_many(&work, ALL_FLAGS)) < 0)
+				goto done;
 	}
 
 	for (i = 0; i < commits->length; ++i) {
@@ -531,10 +541,9 @@ int git_merge__bases_many(git_commit_list **out, git_revwalk *walk, git_commit_l
 		while (result)
 			git_vector_insert(&redundant, git_commit_list_pop(&result));
 
-		clear_commit_marks(one, ALL_FLAGS);
-		clear_commit_marks_many(twos, ALL_FLAGS);
-
-		if ((error = remove_redundant(walk, &redundant)) < 0) {
+		if ((error = clear_commit_marks(one, ALL_FLAGS)) < 0 ||
+		    (error = clear_commit_marks_many(twos, ALL_FLAGS)) < 0 ||
+		    (error = remove_redundant(walk, &redundant)) < 0) {
 			git_vector_free(&redundant);
 			return error;
 		}