dedup

data deduplication program
git clone git://git.2f30.org/dedup.git
Log | Files | Refs | README | LICENSE

commit c2dba9fa308d74e7ab4740301de5ae88170d41b8
parent 55a0beec4a87dc5ceecb64828c77aef6d49d6805
Author: sin <sin@2f30.org>
Date:   Sat,  6 Apr 2019 18:29:25 +0100

Add flag to switch between blake2b and blake2bp

Needs some refactoring so the code is not duplicated.

Diffstat:
Mdedup.1 | 9+++++++--
Mdedup.c | 75+++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------
Mdedup.h | 3+++
3 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/dedup.1 b/dedup.1 @@ -1,4 +1,4 @@ -.Dd March 7, 2019 +.Dd April 6, 2019 .Dt DEDUP 1 .Os .Sh NAME @@ -6,7 +6,7 @@ .Nd data deduplication program .Sh SYNOPSIS .Nm dedup -.Op Fl Zcilv +.Op Fl PZcilv .Op Fl e Ar id .Op Fl r Ar root .Op Fl m Ar message @@ -27,6 +27,11 @@ should be used and piped into .Nm . .Sh OPTIONS .Bl -tag -width "-m message" +.It Fl P +Use the blake2bp variant which is a parallel version of blake2b. +These two variants are incompatible as they produce different +hashes. This flag only has an effect when initializing the +repository. By default blake2b is used. .It Fl Z Disable compression support for this repository. This flag only has an effect when initializing the repository. diff --git a/dedup.c b/dedup.c @@ -33,6 +33,7 @@ static struct blk_hdr blk_hdr; static struct icache *icache; static int ifd; static int sfd; +static int blake2b_parallel; int verbose; char *argv0; @@ -104,17 +105,38 @@ free_snap(struct snap *snap) static void hash_snap(struct snap *snap, uint8_t *md) { - blake2bp_state ctx; - uint64_t i; + switch (blake2b_parallel) { + case 0: { + blake2b_state ctx; + uint64_t i; + + blake2b_init(&ctx, MD_SIZE); + for (i = 0; i < snap->nr_blk_descs; i++) { + struct blk_desc *blk_desc; + + blk_desc = &snap->blk_desc[i]; + blake2b_update(&ctx, blk_desc->md, + sizeof(blk_desc->md)); + } + blake2b_final(&ctx, md, MD_SIZE); + break; + } + case 1: { + blake2bp_state ctx; + uint64_t i; - blake2bp_init(&ctx, MD_SIZE); - for (i = 0; i < snap->nr_blk_descs; i++) { - struct blk_desc *blk_desc; + blake2bp_init(&ctx, MD_SIZE); + for (i = 0; i < snap->nr_blk_descs; i++) { + struct blk_desc *blk_desc; - blk_desc = &snap->blk_desc[i]; - blake2bp_update(&ctx, blk_desc->md, sizeof(blk_desc->md)); + blk_desc = &snap->blk_desc[i]; + blake2bp_update(&ctx, blk_desc->md, + sizeof(blk_desc->md)); + } + blake2bp_final(&ctx, md, MD_SIZE); + break; + } } - blake2bp_final(&ctx, md, MD_SIZE); } static struct snap * @@ -180,11 +202,24 @@ free_buf(uint8_t *buf) static void hash_blk(uint8_t *buf, size_t size, uint8_t *md) { - blake2bp_state ctx; + switch (blake2b_parallel) { + case 0: { + blake2b_state ctx; + + blake2b_init(&ctx, MD_SIZE); + blake2b_update(&ctx, buf, size); + blake2b_final(&ctx, md, MD_SIZE); + break; + } + case 1: { + blake2bp_state ctx; - blake2bp_init(&ctx, MD_SIZE); - blake2bp_update(&ctx, buf, size); - blake2bp_final(&ctx, md, MD_SIZE); + blake2bp_init(&ctx, MD_SIZE); + blake2bp_update(&ctx, buf, size); + blake2bp_final(&ctx, md, MD_SIZE); + break; + } + } } static void @@ -337,15 +372,11 @@ check_snap(struct snap *snap, void *arg) buf = alloc_buf(compr_size(BLKSIZE_MAX)); for (i = 0; i < snap->nr_blk_descs; i++) { uint8_t md[MD_SIZE]; - blake2bp_state ctx; struct blk_desc *blk_desc; blk_desc = &snap->blk_desc[i]; read_blk(buf, blk_desc); - - blake2bp_init(&ctx, MD_SIZE); - blake2bp_update(&ctx, buf, blk_desc->size); - blake2bp_final(&ctx, md, MD_SIZE); + hash_blk(buf, blk_desc->size, md); if (memcmp(blk_desc->md, md, sizeof(blk_desc->md)) == 0) continue; @@ -436,6 +467,7 @@ init_blk_hdr(void) { blk_hdr.flags = (VER_MAJ << VER_MAJ_SHIFT) | VER_MIN; blk_hdr.flags |= compr_enabled << COMPR_ENABLED_SHIFT; + blk_hdr.flags |= blake2b_parallel << BLAKE2BP_ENABLED_SHIFT; blk_hdr.size = BLK_HDR_SIZE; } @@ -451,6 +483,10 @@ load_blk_hdr(void) v = blk_hdr.flags >> COMPR_ENABLED_SHIFT; v &= COMPR_ENABLED_MASK; compr_enabled = v; + + v = blk_hdr.flags >> BLAKE2BP_ENABLED_SHIFT; + v &= BLAKE2BP_ENABLED_MASK; + blake2b_parallel = v; } static void @@ -537,7 +573,7 @@ term(void) static void usage(void) { - fprintf(stderr, "usage: %s [-Zcilv] [-e id] [-r root] [-m message] [file]\n", argv0); + fprintf(stderr, "usage: %s [-PZcilv] [-e id] [-r root] [-m message] [file]\n", argv0); exit(1); } @@ -550,6 +586,9 @@ main(int argc, char *argv[]) int fd = -1; ARGBEGIN { + case 'P': + blake2b_parallel = 1; + break; case 'Z': compr_enabled = 0; break; diff --git a/dedup.h b/dedup.h @@ -25,6 +25,9 @@ #define COMPR_ENABLED_SHIFT 16 #define COMPR_ENABLED_MASK 0x1 +#define BLAKE2BP_ENABLED_SHIFT 17 +#define BLAKE2BP_ENABLED_MASK 0x1 + struct icache; struct chunker;