transport: teach transports about oid types and SHA256

Author: ethomson

include/git2/sys/transport.h

@@ -57,6 +57,18 @@ struct git_transport {
 		unsigned int *capabilities,
 		git_transport *transport);
 
+#ifdef GIT_EXPERIMENTAL_SHA256
+	/**
+	 * Gets the object type for the remote repository.
+	 *
+	 * This function may be called after a successful call to
+	 * `connect()`.
+	 */
+	int GIT_CALLBACK(oid_type)(
+		git_oid_t *object_type,
+		git_transport *transport);
+#endif
+
 	/**
 	 * Get the list of available references in the remote repository.
 	 *

src/libgit2/clone.c

@@ -393,12 +393,19 @@ static int checkout_branch(git_repository *repo, git_remote *remote, const git_c
 	return error;
 }
 
-static int clone_into(git_repository *repo, git_remote *_remote, const git_fetch_options *opts, const git_checkout_options *co_opts, const char *branch)
+static int clone_into(
+	git_repository *repo,
+	git_remote *_remote,
+	const git_fetch_options *opts,
+	const git_checkout_options *co_opts,
+	const char *branch)
 {
 	int error;
 	git_str reflog_message = GIT_STR_INIT;
+	git_remote_connect_options connect_opts = GIT_REMOTE_CONNECT_OPTIONS_INIT;
 	git_fetch_options fetch_opts;
 	git_remote *remote;
+	git_oid_t oid_type;
 
 	GIT_ASSERT_ARG(repo);
 	GIT_ASSERT_ARG(_remote);
@@ -414,8 +421,25 @@ static int clone_into(git_repository *repo, git_remote *_remote, const git_fetch
 	memcpy(&fetch_opts, opts, sizeof(git_fetch_options));
 	fetch_opts.update_fetchhead = 0;
 	fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL;
+
+	if ((error = git_remote_connect_options__from_fetch_opts(&connect_opts, remote, &fetch_opts)) < 0)
+		return error;
+
 	git_str_printf(&reflog_message, "clone: from %s", git_remote_url(remote));
 
+	/*
+	 * Connect to the server so that we can identify the remote
+	 * object format.
+	 */
+
+	if ((error = git_remote_connect_ext(remote, GIT_DIRECTION_FETCH,
+			&connect_opts)) < 0)
+		goto cleanup;
+
+	if ((error = git_remote_oid_type(&oid_type, remote)) < 0 ||
+	    (error = git_repository__set_objectformat(repo, oid_type)) < 0)
+		goto cleanup;
+
 	if ((error = git_remote_fetch(remote, NULL, &fetch_opts, git_str_cstr(&reflog_message))) != 0)
 		goto cleanup;
 

src/libgit2/fetch.c

@@ -95,7 +95,6 @@ static int filter_wants(git_remote *remote, const git_fetch_options *opts)
 	git_remote_head **heads;
 	git_refspec tagspec, head, *spec;
 	int error = 0;
-	git_odb *odb;
 	size_t i, heads_len;
 	unsigned int remote_caps;
 	unsigned int oid_mask = GIT_REMOTE_CAPABILITY_TIP_OID |
@@ -126,9 +125,6 @@ static int filter_wants(git_remote *remote, const git_fetch_options *opts)
 			goto cleanup;
 	}
 
-	if ((error = git_repository_odb__weakptr(&odb, remote->repo)) < 0)
-		goto cleanup;
-
 	if ((error = git_remote_ls((const git_remote_head ***)&heads, &heads_len, remote)) < 0 ||
 	    (error = git_remote_capabilities(&remote_caps, remote)) < 0)
 		goto cleanup;

src/libgit2/indexer.c

@@ -56,8 +56,8 @@ struct git_indexer {
 	git_vector deltas;
 	unsigned int fanout[256];
 	git_hash_ctx hash_ctx;
-	unsigned char checksum[GIT_HASH_SHA1_SIZE];
-	char name[(GIT_HASH_SHA1_SIZE * 2) + 1];
+	unsigned char checksum[GIT_HASH_MAX_SIZE];
+	char name[(GIT_HASH_MAX_SIZE * 2) + 1];
 	git_indexer_progress_cb progress_cb;
 	void *progress_payload;
 	char objbuf[8*1024];
@@ -69,7 +69,7 @@ struct git_indexer {
 	git_odb *odb;
 
 	/* Fields for calculating the packfile trailer (hash of everything before it) */
-	char inbuf[GIT_OID_MAX_SIZE];
+	char inbuf[GIT_HASH_MAX_SIZE];
 	size_t inbuf_len;
 	git_hash_ctx trailer;
 };
@@ -137,6 +137,20 @@ int git_indexer_init_options(git_indexer_options *opts, unsigned int version)
 }
 #endif
 
+GIT_INLINE(git_hash_algorithm_t) indexer_hash_algorithm(git_indexer *idx)
+{
+	switch (idx->oid_type) {
+		case GIT_OID_SHA1:
+			return GIT_HASH_ALGORITHM_SHA1;
+#ifdef GIT_EXPERIMENTAL_SHA256
+		case GIT_OID_SHA256:
+			return GIT_HASH_ALGORITHM_SHA256;
+#endif
+	}
+
+	return GIT_HASH_ALGORITHM_NONE;
+}
+
 static int indexer_new(
 	git_indexer **out,
 	const char *prefix,
@@ -149,6 +163,7 @@ static int indexer_new(
 	git_indexer *idx;
 	git_str path = GIT_STR_INIT, tmp_path = GIT_STR_INIT;
 	static const char suff[] = "/pack";
+	git_hash_algorithm_t checksum_type;
 	int error, fd = -1;
 
 	if (in_opts)
@@ -163,8 +178,10 @@ static int indexer_new(
 	idx->mode = mode ? mode : GIT_PACK_FILE_MODE;
 	git_str_init(&idx->entry_data, 0);
 
-	if ((error = git_hash_ctx_init(&idx->hash_ctx, GIT_HASH_ALGORITHM_SHA1)) < 0 ||
-	    (error = git_hash_ctx_init(&idx->trailer, GIT_HASH_ALGORITHM_SHA1)) < 0 ||
+	checksum_type = indexer_hash_algorithm(idx);
+
+	if ((error = git_hash_ctx_init(&idx->hash_ctx, checksum_type)) < 0 ||
+	    (error = git_hash_ctx_init(&idx->trailer, checksum_type)) < 0 ||
 	    (error = git_oidmap_new(&idx->expected_oids)) < 0)
 		goto cleanup;
 
@@ -182,8 +199,7 @@ static int indexer_new(
 	if (fd < 0)
 		goto cleanup;
 
-	/* TODO: SHA256 */
-	error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path), 0);
+	error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path), oid_type);
 	git_str_dispose(&tmp_path);
 
 	if (error < 0)
@@ -614,7 +630,7 @@ static int do_progress_callback(git_indexer *idx, git_indexer_progress *stats)
 	return 0;
 }
 
-/* Hash everything but the last 20B of input */
+/* Hash everything but the checksum trailer */
 static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
 {
 	size_t to_expell, to_keep;
@@ -623,7 +639,10 @@ static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
 	if (size == 0)
 		return;
 
-	/* Easy case, dump the buffer and the data minus the last 20 bytes */
+	/*
+	 * Easy case, dump the buffer and the data minus the trailing
+	 * checksum (SHA1 or SHA256).
+	 */
 	if (size >= oid_size) {
 		git_hash_update(&idx->trailer, idx->inbuf, idx->inbuf_len);
 		git_hash_update(&idx->trailer, data, size - oid_size);
@@ -761,12 +780,14 @@ static int read_stream_object(git_indexer *idx, git_indexer_progress *stats)
 {
 	git_packfile_stream *stream = &idx->stream;
 	off64_t entry_start = idx->off;
-	size_t entry_size;
+	size_t oid_size, entry_size;
 	git_object_t type;
 	git_mwindow *w = NULL;
 	int error;
 
-	if (idx->pack->mwf.size <= idx->off + 20)
+	oid_size = git_oid_size(idx->oid_type);
+
+	if (idx->pack->mwf.size <= idx->off + (long long)oid_size)
 		return GIT_EBUFS;
 
 	if (!idx->have_stream) {
@@ -963,15 +984,17 @@ static int inject_object(git_indexer *idx, git_oid *id)
 	git_odb_object *obj = NULL;
 	struct entry *entry = NULL;
 	struct git_pack_entry *pentry = NULL;
-	unsigned char empty_checksum[GIT_HASH_SHA1_SIZE] = {0};
+	unsigned char empty_checksum[GIT_HASH_MAX_SIZE] = {0};
 	unsigned char hdr[64];
 	git_str buf = GIT_STR_INIT;
 	off64_t entry_start;
 	const void *data;
 	size_t len, hdr_len;
-	size_t checksum_size = GIT_HASH_SHA1_SIZE;
+	size_t checksum_size;
 	int error;
 
+	checksum_size = git_hash_size(indexer_hash_algorithm(idx));
+
 	if ((error = seek_back_trailer(idx)) < 0)
 		goto cleanup;
 
@@ -1205,17 +1228,20 @@ int git_indexer_commit(git_indexer *idx, git_indexer_progress *stats)
 	struct git_pack_idx_header hdr;
 	git_str filename = GIT_STR_INIT;
 	struct entry *entry;
-	unsigned char checksum[GIT_HASH_SHA1_SIZE];
+	unsigned char checksum[GIT_HASH_MAX_SIZE];
 	git_filebuf index_file = {0};
 	void *packfile_trailer;
-	size_t checksum_size = GIT_HASH_SHA1_SIZE;
+	size_t checksum_size;
 	bool mismatch;
 
 	if (!idx->parsed_header) {
 		git_error_set(GIT_ERROR_INDEXER, "incomplete pack header");
 		return -1;
 	}
 
+	checksum_size = git_hash_size(indexer_hash_algorithm(idx));
+	GIT_ASSERT(checksum_size);
+
 	/* Test for this before resolve_deltas(), as it plays with idx->off */
 	if (idx->off + (ssize_t)checksum_size < idx->pack->mwf.size) {
 		git_error_set(GIT_ERROR_INDEXER, "unexpected data at the end of the pack");

src/libgit2/remote.c

@@ -1026,6 +1026,24 @@ int git_remote_capabilities(unsigned int *out, git_remote *remote)
 	return remote->transport->capabilities(out, remote->transport);
 }
 
+int git_remote_oid_type(git_oid_t *out, git_remote *remote)
+{
+	GIT_ASSERT_ARG(remote);
+
+	if (!remote->transport) {
+		git_error_set(GIT_ERROR_NET, "this remote has never connected");
+		*out = 0;
+		return -1;
+	}
+
+#ifdef GIT_EXPERIMENTAL_SHA256
+	return remote->transport->oid_type(out, remote->transport);
+#else
+	*out = GIT_OID_SHA1;
+	return 0;
+#endif
+}
+
 static int lookup_config(char **out, git_config *cfg, const char *name)
 {
 	git_config_entry *ce = NULL;
@@ -1225,24 +1243,6 @@ static int ls_to_vector(git_vector *out, git_remote *remote)
 	return 0;
 }
 
-#define copy_opts(out, in) \
-	if (in) { \
-		(out)->callbacks = (in)->callbacks; \
-		(out)->proxy_opts = (in)->proxy_opts; \
-		(out)->custom_headers = (in)->custom_headers; \
-		(out)->follow_redirects = (in)->follow_redirects; \
-	}
-
-GIT_INLINE(int) connect_opts_from_fetch_opts(
-	git_remote_connect_options *out,
-	git_remote *remote,
-	const git_fetch_options *fetch_opts)
-{
-	git_remote_connect_options tmp = GIT_REMOTE_CONNECT_OPTIONS_INIT;
-	copy_opts(&tmp, fetch_opts);
-	return git_remote_connect_options_normalize(out, remote->repo, &tmp);
-}
-
 static int connect_or_reset_options(
 	git_remote *remote,
 	int direction,
@@ -1330,7 +1330,8 @@ int git_remote_download(
 		return -1;
 	}
 
-	if (connect_opts_from_fetch_opts(&connect_opts, remote, opts) < 0)
+	if (git_remote_connect_options__from_fetch_opts(&connect_opts,
+			remote, opts) < 0)
 		return -1;
 
 	if ((error = connect_or_reset_options(remote, GIT_DIRECTION_FETCH, &connect_opts)) < 0)
@@ -1350,6 +1351,8 @@ int git_remote_fetch(
 	bool prune = false;
 	git_str reflog_msg_buf = GIT_STR_INIT;
 	git_remote_connect_options connect_opts = GIT_REMOTE_CONNECT_OPTIONS_INIT;
+	unsigned int capabilities;
+	git_oid_t oid_type;
 
 	GIT_ASSERT_ARG(remote);
 
@@ -1358,7 +1361,8 @@ int git_remote_fetch(
 		return -1;
 	}
 
-	if (connect_opts_from_fetch_opts(&connect_opts, remote, opts) < 0)
+	if (git_remote_connect_options__from_fetch_opts(&connect_opts,
+			remote, opts) < 0)
 		return -1;
 
 	if ((error = connect_or_reset_options(remote, GIT_DIRECTION_FETCH, &connect_opts)) < 0)
@@ -1369,6 +1373,10 @@ int git_remote_fetch(
 		tagopt = opts->download_tags;
 	}
 
+	if ((error = git_remote_capabilities(&capabilities, remote)) < 0 ||
+	    (error = git_remote_oid_type(&oid_type, remote)) < 0)
+		return error;
+
 	/* Connect and download everything */
 	error = git_remote__download(remote, refspecs, opts);
 
@@ -2896,16 +2904,6 @@ int git_remote__default_branch(git_str *out, git_remote *remote)
 	return error;
 }
 
-GIT_INLINE(int) connect_opts_from_push_opts(
-	git_remote_connect_options *out,
-	git_remote *remote,
-	const git_push_options *push_opts)
-{
-	git_remote_connect_options tmp = GIT_REMOTE_CONNECT_OPTIONS_INIT;
-	copy_opts(&tmp, push_opts);
-	return git_remote_connect_options_normalize(out, remote->repo, &tmp);
-}
-
 int git_remote_upload(
 	git_remote *remote,
 	const git_strarray *refspecs,
@@ -2924,7 +2922,8 @@ int git_remote_upload(
 		return -1;
 	}
 
-	if ((error = connect_opts_from_push_opts(&connect_opts, remote, opts)) < 0)
+	if ((error = git_remote_connect_options__from_push_opts(
+			&connect_opts, remote, opts)) < 0)
 		goto cleanup;
 
 	if ((error = connect_or_reset_options(remote, GIT_DIRECTION_PUSH, &connect_opts)) < 0)
@@ -2985,7 +2984,8 @@ int git_remote_push(
 		return -1;
 	}
 
-	if (connect_opts_from_push_opts(&connect_opts, remote, opts) < 0)
+	if (git_remote_connect_options__from_push_opts(&connect_opts,
+			remote, opts) < 0)
 		return -1;
 
 	if ((error = git_remote_upload(remote, refspecs, opts)) < 0)