odb: freshen existing objects when writing

When writing an object, we calculate its OID and see if it exists in the
object database.  If it does, we need to freshen the file that contains
it.

Author: Edward Thomson

CHANGELOG.md

@@ -63,6 +63,13 @@ v0.24 + 1
 * `git_diff_file` now includes an `id_abbrev` field that reflects the
   number of nibbles set in the `id` field.
 
+* `git_odb_backend` now has a `freshen` function pointer.  This optional
+  function pointer is similar to the `exists` function, but it will update
+  a last-used marker.  For filesystem-based object databases, this updates
+  the timestamp of the file containing the object, to indicate "freshness".
+  If this is `NULL`, then it will not be called and the `exists` function
+  will be used instead.
+
 v0.24
 -------
 

include/git2/sys/odb_backend.h

@@ -83,6 +83,17 @@ struct git_odb_backend {
 		git_odb_writepack **, git_odb_backend *, git_odb *odb,
 		git_transfer_progress_cb progress_cb, void *progress_payload);
 
+	/**
+	 * "Freshens" an already existing object, updating its last-used
+	 * time.  This occurs when `git_odb_write` was called, but the
+	 * object already existed (and will not be re-written).  The
+	 * underlying implementation may want to update last-used timestamps.
+	 *
+	 * If callers implement this, they should return `0` if the object
+	 * exists and was freshened, and non-zero otherwise.
+	 */
+	int (* freshen)(git_odb_backend *, const git_oid *);
+
 	/**
 	 * Frees any resources held by the odb (including the `git_odb_backend`
 	 * itself). An odb backend implementation must provide this function.

src/fileops.c

@@ -837,6 +837,22 @@ int git_futils_cp(const char *from, const char *to, mode_t filemode)
 	return cp_by_fd(ifd, ofd, true);
 }
 
+int git_futils_touch(const char *path)
+{
+	struct p_timeval times[2];
+	time_t now = time(NULL);
+	int ret;
+
+	times[0].tv_sec = now;
+	times[0].tv_usec = 0;
+	times[1].tv_sec = now;
+	times[1].tv_usec = 0;
+
+	ret = p_utimes(path, times);
+
+	return (ret < 0) ? git_path_set_error(errno, path, "touch") : 0;
+}
+
 static int cp_link(const char *from, const char *to, size_t link_size)
 {
 	int error = 0;

src/fileops.h

@@ -184,6 +184,11 @@ extern int git_futils_cp(
 	const char *to,
 	mode_t filemode);
 
+/**
+ * Set the files atime and mtime to the current time.
+ */
+extern int git_futils_touch(const char *path);
+
 /**
  * Flags that can be passed to `git_futils_cp_r`.
  *

src/odb.c

@@ -654,7 +654,10 @@ void git_odb_free(git_odb *db)
 	GIT_REFCOUNT_DEC(db, odb_free);
 }
 
-static int odb_exists_1(git_odb *db, const git_oid *id, bool only_refreshed)
+static int odb_exists_1(
+	git_odb *db,
+	const git_oid *id,
+	bool only_refreshed)
 {
 	size_t i;
 	bool found = false;
@@ -673,6 +676,44 @@ static int odb_exists_1(git_odb *db, const git_oid *id, bool only_refreshed)
 	return (int)found;
 }
 
+static int odb_freshen_1(
+	git_odb *db,
+	const git_oid *id,
+	bool only_refreshed)
+{
+	size_t i;
+	bool found = false;
+
+	for (i = 0; i < db->backends.length && !found; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		if (only_refreshed && !b->refresh)
+			continue;
+
+		if (b->freshen != NULL)
+			found = !b->freshen(b, id);
+		else if (b->exists != NULL)
+			found = b->exists(b, id);
+	}
+
+	return (int)found;
+}
+
+static int odb_freshen(git_odb *db, const git_oid *id)
+{
+	assert(db && id);
+
+	if (odb_freshen_1(db, id, false))
+		return 1;
+
+	if (!git_odb_refresh(db))
+		return odb_freshen_1(db, id, true);
+
+	/* Failed to refresh, hence not found */
+	return 0;
+}
+
 int git_odb_exists(git_odb *db, const git_oid *id)
 {
 	git_odb_object *object;
@@ -1131,7 +1172,7 @@ int git_odb_write(
 	assert(oid && db);
 
 	git_odb_hash(oid, data, len, type);
-	if (git_odb_exists(db, oid))
+	if (odb_freshen(db, oid))
 		return 0;
 
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
@@ -1257,7 +1298,7 @@ int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
 
 	git_hash_final(out, stream->hash_ctx);
 
-	if (git_odb_exists(stream->backend->odb, out))
+	if (odb_freshen(stream->backend->odb, out))
 		return 0;
 
 	return stream->finalize_write(stream, out);

src/odb_loose.c

@@ -918,6 +918,23 @@ static int loose_backend__write(git_odb_backend *_backend, const git_oid *oid, c
 	return error;
 }
 
+static int loose_backend__freshen(
+	git_odb_backend *_backend,
+	const git_oid *oid)
+{
+	loose_backend *backend = (loose_backend *)_backend;
+	git_buf path = GIT_BUF_INIT;
+	int error;
+
+	if (object_file_name(&path, backend, oid) < 0)
+		return -1;
+
+	error = git_futils_touch(path.ptr);
+	git_buf_free(&path);
+
+	return error;
+}
+
 static void loose_backend__free(git_odb_backend *_backend)
 {
 	loose_backend *backend;
@@ -975,6 +992,7 @@ int git_odb_backend_loose(
 	backend->parent.exists = &loose_backend__exists;
 	backend->parent.exists_prefix = &loose_backend__exists_prefix;
 	backend->parent.foreach = &loose_backend__foreach;
+	backend->parent.freshen = &loose_backend__freshen;
 	backend->parent.free = &loose_backend__free;
 
 	*backend_out = (git_odb_backend *)backend;

src/odb_pack.c

@@ -363,6 +363,18 @@ static int pack_backend__read_header(
 	return git_packfile_resolve_header(len_p, type_p, e.p, e.offset);
 }
 
+static int pack_backend__freshen(
+	git_odb_backend *backend, const git_oid *oid)
+{
+	struct git_pack_entry e;
+	int error;
+
+	if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < 0)
+		return error;
+
+	return git_futils_touch(e.p->pack_name);
+}
+
 static int pack_backend__read(
 	void **buffer_p, size_t *len_p, git_otype *type_p,
 	git_odb_backend *backend, const git_oid *oid)
@@ -560,6 +572,7 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size)
 	backend->parent.refresh = &pack_backend__refresh;
 	backend->parent.foreach = &pack_backend__foreach;
 	backend->parent.writepack = &pack_backend__writepack;
+	backend->parent.freshen = &pack_backend__freshen;
 	backend->parent.free = &pack_backend__free;
 
 	*out = backend;

tests/odb/freshen.c

@@ -0,0 +1,79 @@
+#include "clar_libgit2.h"
+#include "odb.h"
+#include "posix.h"
+
+static git_repository *repo;
+static git_odb *odb;
+
+void test_odb_freshen__initialize(void)
+{
+	repo = cl_git_sandbox_init("testrepo.git");
+	cl_git_pass(git_repository_odb(&odb, repo));
+}
+
+void test_odb_freshen__cleanup(void)
+{
+	git_odb_free(odb);
+	cl_git_sandbox_cleanup();
+}
+
+#define LOOSE_STR "hey\n"
+#define LOOSE_ID  "1385f264afb75a56a5bec74243be9b367ba4ca08"
+#define LOOSE_FN  "13/85f264afb75a56a5bec74243be9b367ba4ca08"
+
+void test_odb_freshen__loose_object(void)
+{
+	git_oid expected_id, id;
+	struct stat before, after;
+	struct p_timeval old_times[2];
+
+	cl_git_pass(git_oid_fromstr(&expected_id, LOOSE_ID));
+
+	old_times[0].tv_sec = 1234567890;
+	old_times[0].tv_usec = 0;
+	old_times[1].tv_sec = 1234567890;
+	old_times[1].tv_usec = 0;
+
+	/* set time to way back */
+	cl_must_pass(p_utimes("testrepo.git/objects/" LOOSE_FN, old_times));
+	cl_must_pass(p_lstat("testrepo.git/objects/" LOOSE_FN, &before));
+
+	cl_git_pass(git_odb_write(&id, odb, LOOSE_STR, CONST_STRLEN(LOOSE_STR),
+		GIT_OBJ_BLOB));
+	cl_assert_equal_oid(&expected_id, &id);
+	cl_must_pass(p_lstat("testrepo.git/objects/" LOOSE_FN, &after));
+
+	cl_assert(before.st_atime < after.st_atime);
+	cl_assert(before.st_mtime < after.st_mtime);
+}
+
+#define PACKED_STR "Testing a readme.txt\n"
+#define PACKED_ID  "6336846bd5c88d32f93ae57d846683e61ab5c530"
+#define PACKED_FN  "pack-d85f5d483273108c9d8dd0e4728ccf0b2982423a.pack"
+
+void test_odb_freshen__packed_object(void)
+{
+	git_oid expected_id, id;
+	struct stat before, after;
+	struct p_timeval old_times[2];
+
+	cl_git_pass(git_oid_fromstr(&expected_id, PACKED_ID));
+
+	old_times[0].tv_sec = 1234567890;
+	old_times[0].tv_usec = 0;
+	old_times[1].tv_sec = 1234567890;
+	old_times[1].tv_usec = 0;
+
+	/* set time to way back */
+	cl_must_pass(p_utimes("testrepo.git/objects/pack/" PACKED_FN, old_times));
+	cl_must_pass(p_lstat("testrepo.git/objects/pack/" PACKED_FN, &before));
+
+	cl_git_pass(git_odb_write(&id, odb, PACKED_STR,
+		CONST_STRLEN(PACKED_STR), GIT_OBJ_BLOB));
+	cl_assert_equal_oid(&expected_id, &id);
+	cl_must_pass(p_lstat("testrepo.git/objects/pack/" PACKED_FN, &after));
+
+	cl_assert(before.st_atime < after.st_atime);
+	cl_assert(before.st_mtime < after.st_mtime);
+}
+