Skip to content

Commit

Permalink
Merge branch 'tb/weak-sha1-for-tail-sum' into seen
Browse files Browse the repository at this point in the history 
The checksum at the tail of files are now computed without
collision detection protection.

* tb/weak-sha1-for-tail-sum:
  csum-file.c: use fast SHA-1 implementation when available
  Makefile: allow specifying a SHA-1 for non-cryptographic uses
  hash.h: scaffolding for _fast hashing variants
  sha1: do not redefine `platform_SHA_CTX` and friends
  i5500-git-daemon.sh: use compile-able version of Git without OpenSSL
  pack-objects: use finalize_object_file() to rename pack/idx/etc
  finalize_object_file(): implement collision check
  finalize_object_file(): refactor unlink_or_warn() placement
  finalize_object_file(): check for name collision before renaming
  • Loading branch information
@gitster
gitster committed Sep 7, 2024
2 parents d68fa3a + abe19da commit df8b62d
Show file tree
Hide file tree
Showing 10 changed files with 230 additions and 18 deletions.
25 changes: 25 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,10 @@ include shared.mak
# Define APPLE_COMMON_CRYPTO_SHA1 to use Apple's CommonCrypto for
# SHA-1.
#
# Define the same Makefile knobs as above, but suffixed with _FAST to
# use the corresponding implementations for "fast" SHA-1 hashing for
# non-cryptographic purposes.
#
# If don't enable any of the *_SHA1 settings in this section, Git will
# default to its built-in sha1collisiondetection library, which is a
# collision-detecting sha1 This is slower, but may detect attempted
Expand Down Expand Up @@ -2000,6 +2004,27 @@ endif
endif
endif

ifdef OPENSSL_SHA1_FAST
ifndef OPENSSL_SHA1
EXTLIBS += $(LIB_4_CRYPTO)
BASIC_CFLAGS += -DSHA1_OPENSSL_FAST
endif
else
ifdef BLK_SHA1_FAST
ifndef BLK_SHA1
LIB_OBJS += block-sha1/sha1.o
BASIC_CFLAGS += -DSHA1_BLK_FAST
endif
else
ifdef APPLE_COMMON_CRYPTO_SHA1_FAST
ifndef APPLE_COMMON_CRYPTO_SHA1
COMPAT_CFLAGS += -DCOMMON_DIGEST_FOR_OPENSSL
BASIC_CFLAGS += -DSHA1_APPLE_FAST
endif
endif
endif
endif

ifdef OPENSSL_SHA256
EXTLIBS += $(LIB_4_CRYPTO)
BASIC_CFLAGS += -DSHA256_OPENSSL
Expand Down
2 changes: 2 additions & 0 deletions block-sha1/sha1.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ void blk_SHA1_Init(blk_SHA_CTX *ctx);
void blk_SHA1_Update(blk_SHA_CTX *ctx, const void *dataIn, size_t len);
void blk_SHA1_Final(unsigned char hashout[20], blk_SHA_CTX *ctx);

#ifndef platform_SHA_CTX
#define platform_SHA_CTX blk_SHA_CTX
#define platform_SHA1_Init blk_SHA1_Init
#define platform_SHA1_Update blk_SHA1_Update
#define platform_SHA1_Final blk_SHA1_Final
#endif
18 changes: 9 additions & 9 deletions csum-file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ void hashflush(struct hashfile *f)

if (offset) {
if (!f->skip_hash)
the_hash_algo->update_fn(&f->ctx, f->buffer, offset);
the_hash_algo->fast_update_fn(&f->ctx, f->buffer, offset);
flush(f, f->buffer, offset);
f->offset = 0;
}
Expand All @@ -73,7 +73,7 @@ int finalize_hashfile(struct hashfile *f, unsigned char *result,
if (f->skip_hash)
hashclr(f->buffer, the_repository->hash_algo);
else
the_hash_algo->final_fn(f->buffer, &f->ctx);
the_hash_algo->fast_final_fn(f->buffer, &f->ctx);

if (result)
hashcpy(result, f->buffer, the_repository->hash_algo);
Expand Down Expand Up @@ -128,7 +128,7 @@ void hashwrite(struct hashfile *f, const void *buf, unsigned int count)
* f->offset is necessarily zero.
*/
if (!f->skip_hash)
the_hash_algo->update_fn(&f->ctx, buf, nr);
the_hash_algo->fast_update_fn(&f->ctx, buf, nr);
flush(f, buf, nr);
} else {
/*
Expand Down Expand Up @@ -174,7 +174,7 @@ static struct hashfile *hashfd_internal(int fd, const char *name,
f->name = name;
f->do_crc = 0;
f->skip_hash = 0;
the_hash_algo->init_fn(&f->ctx);
the_hash_algo->fast_init_fn(&f->ctx);

f->buffer_len = buffer_len;
f->buffer = xmalloc(buffer_len);
Expand Down Expand Up @@ -208,7 +208,7 @@ void hashfile_checkpoint(struct hashfile *f, struct hashfile_checkpoint *checkpo
{
hashflush(f);
checkpoint->offset = f->total;
the_hash_algo->clone_fn(&checkpoint->ctx, &f->ctx);
the_hash_algo->fast_clone_fn(&checkpoint->ctx, &f->ctx);
}

int hashfile_truncate(struct hashfile *f, struct hashfile_checkpoint *checkpoint)
Expand All @@ -219,7 +219,7 @@ int hashfile_truncate(struct hashfile *f, struct hashfile_checkpoint *checkpoint
lseek(f->fd, offset, SEEK_SET) != offset)
return -1;
f->total = offset;
the_hash_algo->clone_fn(&f->ctx, &checkpoint->ctx);
the_hash_algo->fast_clone_fn(&f->ctx, &checkpoint->ctx);
f->offset = 0; /* hashflush() was called in checkpoint */
return 0;
}
Expand All @@ -245,9 +245,9 @@ int hashfile_checksum_valid(const unsigned char *data, size_t total_len)
if (total_len < the_hash_algo->rawsz)
return 0; /* say "too short"? */

the_hash_algo->init_fn(&ctx);
the_hash_algo->update_fn(&ctx, data, data_len);
the_hash_algo->final_fn(got, &ctx);
the_hash_algo->fast_init_fn(&ctx);
the_hash_algo->fast_update_fn(&ctx, data, data_len);
the_hash_algo->fast_final_fn(got, &ctx);

return hasheq(got, data + data_len, the_repository->hash_algo);
}
72 changes: 72 additions & 0 deletions hash.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,36 @@
#include "block-sha1/sha1.h"
#endif

#if defined(SHA1_APPLE_FAST)
# include <CommonCrypto/CommonDigest.h>
# define platform_SHA_CTX_fast CC_SHA1_CTX
# define platform_SHA1_Init_fast CC_SHA1_Init
# define platform_SHA1_Update_fast CC_SHA1_Update
# define platform_SHA1_Final_fast CC_SHA1_Final
#elif defined(SHA1_OPENSSL_FAST)
# include <openssl/sha.h>
# if defined(OPENSSL_API_LEVEL) && OPENSSL_API_LEVEL >= 3
# define SHA1_NEEDS_CLONE_HELPER_FAST
# include "sha1/openssl.h"
# define platform_SHA_CTX_fast openssl_SHA1_CTX
# define platform_SHA1_Init_fast openssl_SHA1_Init
# define platform_SHA1_Clone_fast openssl_SHA1_Clone
# define platform_SHA1_Update_fast openssl_SHA1_Update
# define platform_SHA1_Final_fast openssl_SHA1_Final
# else
# define platform_SHA_CTX_fast SHA_CTX
# define platform_SHA1_Init_fast SHA1_Init
# define platform_SHA1_Update_fast SHA1_Update
# define platform_SHA1_Final_fast SHA1_Final
# endif
#elif defined(SHA1_BLK_FAST)
# include "block-sha1/sha1.h"
# define platform_SHA_CTX_fast blk_SHA_CTX
# define platform_SHA1_Init_fast blk_SHA1_Init
# define platform_SHA1_Update_fast blk_SHA1_Update
# define platform_SHA1_Final_fast blk_SHA1_Final
#endif

#if defined(SHA256_NETTLE)
#include "sha256/nettle.h"
#elif defined(SHA256_GCRYPT)
Expand Down Expand Up @@ -44,14 +74,32 @@
#define platform_SHA1_Final SHA1_Final
#endif

#ifndef platform_SHA_CTX_fast
# define platform_SHA_CTX_fast platform_SHA_CTX
# define platform_SHA1_Init_fast platform_SHA1_Init
# define platform_SHA1_Update_fast platform_SHA1_Update
# define platform_SHA1_Final_fast platform_SHA1_Final
# ifdef platform_SHA1_Clone
# define platform_SHA1_Clone_fast platform_SHA1_Clone
# endif
#endif

#define git_SHA_CTX platform_SHA_CTX
#define git_SHA1_Init platform_SHA1_Init
#define git_SHA1_Update platform_SHA1_Update
#define git_SHA1_Final platform_SHA1_Final

#define git_SHA_CTX_fast platform_SHA_CTX_fast
#define git_SHA1_Init_fast platform_SHA1_Init_fast
#define git_SHA1_Update_fast platform_SHA1_Update_fast
#define git_SHA1_Final_fast platform_SHA1_Final_fast

#ifdef platform_SHA1_Clone
#define git_SHA1_Clone platform_SHA1_Clone
#endif
#ifdef platform_SHA1_Clone_fast
# define git_SHA1_Clone_fast platform_SHA1_Clone_fast
#endif

#ifndef platform_SHA256_CTX
#define platform_SHA256_CTX SHA256_CTX
Expand Down Expand Up @@ -81,6 +129,13 @@ static inline void git_SHA1_Clone(git_SHA_CTX *dst, const git_SHA_CTX *src)
memcpy(dst, src, sizeof(*dst));
}
#endif
#ifndef SHA1_NEEDS_CLONE_HELPER_FAST
static inline void git_SHA1_Clone_fast(git_SHA_CTX_fast *dst,
const git_SHA_CTX_fast *src)
{
memcpy(dst, src, sizeof(*dst));
}
#endif

#ifndef SHA256_NEEDS_CLONE_HELPER
static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *src)
Expand Down Expand Up @@ -178,6 +233,8 @@ enum get_oid_result {
/* A suitably aligned type for stack allocations of hash contexts. */
union git_hash_ctx {
git_SHA_CTX sha1;
git_SHA_CTX_fast sha1_fast;

git_SHA256_CTX sha256;
};
typedef union git_hash_ctx git_hash_ctx;
Expand Down Expand Up @@ -222,6 +279,21 @@ struct git_hash_algo {
/* The hash finalization function for object IDs. */
git_hash_final_oid_fn final_oid_fn;

/* The fast / non-cryptographic hash initialization function. */
git_hash_init_fn fast_init_fn;

/* The fast / non-cryptographic hash context cloning function. */
git_hash_clone_fn fast_clone_fn;

/* The fast / non-cryptographic hash update function. */
git_hash_update_fn fast_update_fn;

/* The fast / non-cryptographic hash finalization function. */
git_hash_final_fn fast_final_fn;

/* The fast / non-cryptographic hash finalization function. */
git_hash_final_oid_fn fast_final_oid_fn;

/* The OID of the empty tree. */
const struct object_id *empty_tree;

Expand Down
110 changes: 106 additions & 4 deletions object-file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,33 @@ static void git_hash_sha1_final_oid(struct object_id *oid, git_hash_ctx *ctx)
oid->algo = GIT_HASH_SHA1;
}

static void git_hash_sha1_init_fast(git_hash_ctx *ctx)
{
git_SHA1_Init_fast(&ctx->sha1_fast);
}

static void git_hash_sha1_clone_fast(git_hash_ctx *dst, const git_hash_ctx *src)
{
git_SHA1_Clone_fast(&dst->sha1_fast, &src->sha1_fast);
}

static void git_hash_sha1_update_fast(git_hash_ctx *ctx, const void *data,
size_t len)
{
git_SHA1_Update_fast(&ctx->sha1_fast, data, len);
}

static void git_hash_sha1_final_fast(unsigned char *hash, git_hash_ctx *ctx)
{
git_SHA1_Final_fast(hash, &ctx->sha1_fast);
}

static void git_hash_sha1_final_oid_fast(struct object_id *oid, git_hash_ctx *ctx)
{
git_SHA1_Final_fast(oid->hash, &ctx->sha1_fast);
memset(oid->hash + GIT_SHA1_RAWSZ, 0, GIT_MAX_RAWSZ - GIT_SHA1_RAWSZ);
oid->algo = GIT_HASH_SHA1;
}

static void git_hash_sha256_init(git_hash_ctx *ctx)
{
Expand Down Expand Up @@ -189,6 +216,11 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
.update_fn = git_hash_unknown_update,
.final_fn = git_hash_unknown_final,
.final_oid_fn = git_hash_unknown_final_oid,
.fast_init_fn = git_hash_unknown_init,
.fast_clone_fn = git_hash_unknown_clone,
.fast_update_fn = git_hash_unknown_update,
.fast_final_fn = git_hash_unknown_final,
.fast_final_oid_fn = git_hash_unknown_final_oid,
.empty_tree = NULL,
.empty_blob = NULL,
.null_oid = NULL,
Expand All @@ -204,6 +236,11 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
.update_fn = git_hash_sha1_update,
.final_fn = git_hash_sha1_final,
.final_oid_fn = git_hash_sha1_final_oid,
.fast_init_fn = git_hash_sha1_init_fast,
.fast_clone_fn = git_hash_sha1_clone_fast,
.fast_update_fn = git_hash_sha1_update_fast,
.fast_final_fn = git_hash_sha1_final_fast,
.fast_final_oid_fn = git_hash_sha1_final_oid_fast,
.empty_tree = &empty_tree_oid,
.empty_blob = &empty_blob_oid,
.null_oid = &null_oid_sha1,
Expand All @@ -219,6 +256,11 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = {
.update_fn = git_hash_sha256_update,
.final_fn = git_hash_sha256_final,
.final_oid_fn = git_hash_sha256_final_oid,
.fast_init_fn = git_hash_sha256_init,
.fast_clone_fn = git_hash_sha256_clone,
.fast_update_fn = git_hash_sha256_update,
.fast_final_fn = git_hash_sha256_final,
.fast_final_oid_fn = git_hash_sha256_final_oid,
.empty_tree = &empty_tree_oid_sha256,
.empty_blob = &empty_blob_oid_sha256,
.null_oid = &null_oid_sha256,
Expand Down Expand Up @@ -1944,17 +1986,71 @@ static void write_object_file_prepare_literally(const struct git_hash_algo *algo
hash_object_body(algo, &c, buf, len, oid, hdr, hdrlen);
}

static int check_collision(const char *filename_a, const char *filename_b)
{
char buf_a[4096], buf_b[4096];
int fd_a = -1, fd_b = -1;
int ret = 0;

fd_a = open(filename_a, O_RDONLY);
if (fd_a < 0) {
ret = error_errno(_("unable to open %s"), filename_a);
goto out;
}

fd_b = open(filename_b, O_RDONLY);
if (fd_b < 0) {
ret = error_errno(_("unable to open %s"), filename_b);
goto out;
}

while (1) {
ssize_t sz_a, sz_b;

sz_a = read_in_full(fd_a, buf_a, sizeof(buf_a));
if (sz_a < 0) {
ret = error_errno(_("unable to read %s"), filename_a);
goto out;
}

sz_b = read_in_full(fd_b, buf_b, sizeof(buf_b));
if (sz_b < 0) {
ret = error_errno(_("unable to read %s"), filename_b);
goto out;
}

if (sz_a != sz_b || memcmp(buf_a, buf_b, sz_a)) {
ret = error(_("files '%s' and '%s' differ in contents"),
filename_a, filename_b);
goto out;
}

if (sz_a < sizeof(buf_a))
break;
}

out:
if (fd_a > -1)
close(fd_a);
if (fd_b > -1)
close(fd_b);
return ret;
}

/*
* Move the just written object into its final resting place.
*/
int finalize_object_file(const char *tmpfile, const char *filename)
{
struct stat st;
int ret = 0;

if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
goto try_rename;
else if (link(tmpfile, filename))
ret = errno;
else
unlink_or_warn(tmpfile);

/*
* Coda hack - coda doesn't like cross-directory links,
Expand All @@ -1969,16 +2065,22 @@ int finalize_object_file(const char *tmpfile, const char *filename)
*/
if (ret && ret != EEXIST) {
try_rename:
if (!rename(tmpfile, filename))
if (!stat(filename, &st))
ret = EEXIST;
else if (!rename(tmpfile, filename))
goto out;
ret = errno;
else
ret = errno;
}
unlink_or_warn(tmpfile);
if (ret) {
if (ret != EEXIST) {
int saved_errno = errno;
unlink_or_warn(tmpfile);
errno = saved_errno;
return error_errno(_("unable to write file %s"), filename);
}
/* FIXME!!! Collision check here ? */
if (check_collision(tmpfile, filename))
return -1;
}

out:
Expand Down
Loading

0 comments on commit df8b62d

Please sign in to comment.