Merge pull request libgit2#6456 from libgit2/ethomson/sha256_experimental

SHA256: more SHA256 support

Author: ethomson

.github/workflows/main.yml

@@ -232,7 +232,7 @@ jobs:
           env:
             CC: clang
             CMAKE_GENERATOR: Ninja
-            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON -DEXPERIMENTAL_SHA256=ON
           os: ubuntu-latest
         - name: "macOS (SHA256)"
           id: macos

.github/workflows/nightly.yml

@@ -265,10 +265,40 @@ jobs:
             RUN_INVASIVE_TESTS: true
             SKIP_PROXY_TESTS: true
           os: ubuntu-latest
+
+        # Experimental: SHA256 support
+        - name: "Linux (SHA256, Xenial, Clang, OpenSSL)"
+          id: xenial-clang-openssl
+          container:
+            name: xenial
+          env:
+            CC: clang
+            CMAKE_GENERATOR: Ninja
+            CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON -DUSE_SSH=ON
+          os: ubuntu-latest
+        - name: "macOS (SHA256)"
+          id: macos
+          os: macos-10.15
+          env:
+            CC: clang
+            CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=leaks -DUSE_GSSAPI=ON -DEXPERIMENTAL_SHA256=ON
+            PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
+          setup-script: osx
+        - name: "Windows (SHA256, amd64, Visual Studio)"
+          id: windows-amd64-vs
+          os: windows-2019
+          env:
+            ARCH: amd64
+            CMAKE_GENERATOR: Visual Studio 16 2019
+            CMAKE_OPTIONS: -A x64 -DWIN32_LEAKCHECK=ON -DDEPRECATE_HARD=ON -DEXPERIMENTAL_SHA256=ON
+            SKIP_SSH_TESTS: true
+            SKIP_NEGOTIATE_TESTS: true
       fail-fast: false
-    name: "Build ${{ matrix.platform.name }}"
     env: ${{ matrix.platform.env }}
     runs-on: ${{ matrix.platform.os }}
+    name: "Build ${{ matrix.platform.name }}"
     steps:
     - name: Check out repository
       uses: actions/checkout@v3

ci/docker/xenial

@@ -7,11 +7,13 @@ RUN apt-get update && \
         clang \
         cmake \
         curl \
+	gettext \
         gcc \
-        git \
         krb5-user \
         libcurl4-gnutls-dev \
+	libexpat1-dev \
         libgcrypt20-dev \
+	libintl-perl \
         libkrb5-dev \
         libpcre3-dev \
         libssl-dev \
@@ -28,7 +30,17 @@ RUN apt-get update && \
         && \
     rm -rf /var/lib/apt/lists/*
 
-FROM apt AS mbedtls
+FROM apt AS git
+RUN cd /tmp && \
+    curl --location --silent --show-error https://github.com/git/git/archive/refs/tags/v2.39.1.tar.gz | \
+        tar -xz && \
+    cd git-2.39.1 && \
+    make && \
+    make prefix=/usr install && \
+    cd .. && \
+    rm -rf git-2.39.1
+
+FROM git AS mbedtls
 RUN cd /tmp && \
     curl --location --silent --show-error https://github.com/Mbed-TLS/mbedtls/archive/refs/tags/mbedtls-2.16.2.tar.gz | \
         tar -xz && \

ci/test.sh

@@ -37,6 +37,11 @@ cleanup() {
 		kill $GIT_NAMESPACE_PID
 	fi
 
+	if [ ! -z "$GIT_SHA256_PID" ]; then
+		echo "Stopping git daemon (sha256)..."
+		kill $GIT_SHA256_PID
+	fi
+
 	if [ ! -z "$PROXY_BASIC_PID" ]; then
 		echo "Stopping proxy (Basic)..."
 		kill $PROXY_BASIC_PID
@@ -145,6 +150,12 @@ if [ -z "$SKIP_GITDAEMON_TESTS" ]; then
 	cp -R "${SOURCE_DIR}/tests/resources/namespace.git" "${GIT_NAMESPACE_DIR}/namespace.git"
 	GIT_NAMESPACE="name1" git daemon --listen=localhost --port=9419 --export-all --enable=receive-pack --base-path="${GIT_NAMESPACE_DIR}" "${GIT_NAMESPACE_DIR}" &
 	GIT_NAMESPACE_PID=$!
+
+	echo "Starting git daemon (sha256)..."
+	GIT_SHA256_DIR=`mktemp -d ${TMPDIR}/git_sha256.XXXXXXXX`
+	cp -R "${SOURCE_DIR}/tests/resources/testrepo_256.git" "${GIT_SHA256_DIR}/testrepo_256.git"
+	git daemon --listen=localhost --port=9420 --export-all --enable=receive-pack --base-path="${GIT_SHA256_DIR}" "${GIT_SHA256_DIR}" &
+	GIT_SHA256_PID=$!
 fi
 
 if [ -z "$SKIP_PROXY_TESTS" ]; then
@@ -291,6 +302,14 @@ if [ -z "$SKIP_GITDAEMON_TESTS" ]; then
 	run_test gitdaemon_namespace
 	unset GITTEST_REMOTE_URL
 	unset GITTEST_REMOTE_BRANCH
+
+	echo ""
+	echo "Running gitdaemon (sha256) tests"
+	echo ""
+
+	export GITTEST_REMOTE_URL="git://localhost:9420/testrepo_256.git"
+	run_test gitdaemon_sha256
+	unset GITTEST_REMOTE_URL
 fi
 
 if [ -z "$SKIP_PROXY_TESTS" ]; then

cmake/ExperimentalFeatures.cmake

@@ -13,7 +13,7 @@ if(EXPERIMENTAL_SHA256)
 
 	set(EXPERIMENTAL 1)
 	set(GIT_EXPERIMENTAL_SHA256 1)
-	add_compile_definitions(GIT_EXPERIMENTAL_SHA256)
+	add_definitions(-DGIT_EXPERIMENTAL_SHA256=1)
 else()
 	add_feature_info("SHA256 API" OFF "experimental SHA256 APIs")
 endif()

examples/index-pack.c

@@ -28,11 +28,18 @@ int lg2_index_pack(git_repository *repo, int argc, char **argv)
 		return EXIT_FAILURE;
 	}
 
-	if (git_indexer_new(&idx, ".", 0, NULL, NULL) < 0) {
+#ifdef GIT_EXPERIMENTAL_SHA256
+	error = git_indexer_new(&idx, ".", git_repository_oid_type(repo), NULL);
+#else
+	error = git_indexer_new(&idx, ".", 0, NULL, NULL);
+#endif
+
+	if (error < 0) {
 		puts("bad idx");
 		return -1;
 	}
 
+
 	if ((fd = open(argv[1], 0)) < 0) {
 		perror("open");
 		return -1;

fuzzers/objects_fuzzer.c

@@ -39,7 +39,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	 * to do.
 	 */
 	for (i = 0; i < ARRAY_SIZE(types); i++) {
-		if (git_object__from_raw(&object, (const char *) data, size, types[i]) < 0)
+		if (git_object__from_raw(&object, (const char *) data, size, types[i], GIT_OID_SHA1) < 0)
 			continue;
 		git_object_free(object);
 		object = NULL;

fuzzers/packfile_fuzzer.c

@@ -67,6 +67,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 	git_str path = GIT_STR_INIT;
 	git_oid oid;
 	bool append_hash = false;
+	int error;
 
 	if (size == 0)
 		return 0;
@@ -82,7 +83,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 		abort();
 	}
 
-	if (git_indexer_new(&indexer, ".", 0, odb, NULL) < 0) {
+#ifdef GIT_EXPERIMENTAL_SHA256
+	error = git_indexer_new(&indexer, ".", GIT_OID_SHA1, NULL);
+#else
+	error = git_indexer_new(&indexer, ".", 0, odb, NULL);
+#endif
+
+	if (error < 0) {
 		fprintf(stderr, "Failed to create the indexer: %s\n",
 			git_error_last()->message);
 		abort();

include/git2/indexer.h

@@ -62,6 +62,19 @@ typedef int GIT_CALLBACK(git_indexer_progress_cb)(const git_indexer_progress *st
 typedef struct git_indexer_options {
 	unsigned int version;
 
+#ifdef GIT_EXPERIMENTAL_SHA256
+	/** permissions to use creating packfile or 0 for defaults */
+	unsigned int mode;
+
+	/**
+	 * object database from which to read base objects when
+	 * fixing thin packs. This can be NULL if there are no thin
+	 * packs; if a thin pack is encountered, an error will be
+	 * returned if there are bases missing.
+	 */
+	git_odb *odb;
+#endif
+
 	/** progress_cb function to call with progress information */
 	git_indexer_progress_cb progress_cb;
 
@@ -87,6 +100,21 @@ GIT_EXTERN(int) git_indexer_options_init(
 	git_indexer_options *opts,
 	unsigned int version);
 
+#ifdef GIT_EXPERIMENTAL_SHA256
+/**
+ * Create a new indexer instance
+ *
+ * @param out where to store the indexer instance
+ * @param path to the directory where the packfile should be stored
+ * @param oid_type the oid type to use for objects
+ * @return 0 or an error code.
+ */
+GIT_EXTERN(int) git_indexer_new(
+		git_indexer **out,
+		const char *path,
+		git_oid_t oid_type,
+		git_indexer_options *opts);
+#else
 /**
  * Create a new indexer instance
  *
@@ -106,6 +134,7 @@ GIT_EXTERN(int) git_indexer_new(
 		unsigned int mode,
 		git_odb *odb,
 		git_indexer_options *opts);
+#endif
 
 /**
  * Add data to the indexer

include/git2/object.h

@@ -225,6 +225,7 @@ GIT_EXTERN(int) git_object_peel(
  */
 GIT_EXTERN(int) git_object_dup(git_object **dest, git_object *source);
 
+#ifdef GIT_EXPERIMENTAL_SHA256
 /**
  * Analyzes a buffer of raw object content and determines its validity.
  * Tree, commit, and tag objects will be parsed and ensured that they
@@ -238,14 +239,39 @@ GIT_EXTERN(int) git_object_dup(git_object **dest, git_object *source);
  * @param valid Output pointer to set with validity of the object content
  * @param buf The contents to validate
  * @param len The length of the buffer
- * @param type The type of the object in the buffer
+ * @param object_type The type of the object in the buffer
+ * @param oid_type The object ID type for the OIDs in the given buffer
  * @return 0 on success or an error code
  */
 GIT_EXTERN(int) git_object_rawcontent_is_valid(
 	int *valid,
 	const char *buf,
 	size_t len,
-	git_object_t type);
+	git_object_t object_type,
+	git_oid_t oid_type);
+#else
+/**
+ * Analyzes a buffer of raw object content and determines its validity.
+ * Tree, commit, and tag objects will be parsed and ensured that they
+ * are valid, parseable content.  (Blobs are always valid by definition.)
+ * An error message will be set with an informative message if the object
+ * is not valid.
+ *
+ * @warning This function is experimental and its signature may change in
+ * the future.
+ *
+ * @param valid Output pointer to set with validity of the object content
+ * @param buf The contents to validate
+ * @param len The length of the buffer
+ * @param object_type The type of the object in the buffer
+ * @return 0 on success or an error code
+ */
+GIT_EXTERN(int) git_object_rawcontent_is_valid(
+	int *valid,
+	const char *buf,
+	size_t len,
+	git_object_t object_type);
+#endif
 
 /** @} */
 GIT_END_DECL