dedup

commit c2dba9fa308d74e7ab4740301de5ae88170d41b8
parent 55a0beec4a87dc5ceecb64828c77aef6d49d6805
Author: sin <sin@2f30.org>
Date:   Sat,  6 Apr 2019 18:29:25 +0100

Add flag to switch between blake2b and blake2bp

Needs some refactoring so the code is not duplicated.

Diffstat:
M
dedup.1
 | 
9
+++++++--
M
dedup.c
 | 
75
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------
M
dedup.h
 | 
3
+++

3 files changed, 67 insertions(+), 20 deletions(-)
diff --git a/dedup.1 b/dedup.1
@@ -1,4 +1,4 @@
-.Dd March 7, 2019
+.Dd April 6, 2019
 .Dt DEDUP 1
 .Os
 .Sh NAME
@@ -6,7 +6,7 @@
 .Nd data deduplication program
 .Sh SYNOPSIS
 .Nm dedup
-.Op Fl Zcilv
+.Op Fl PZcilv
 .Op Fl e Ar id
 .Op Fl r Ar root
 .Op Fl m Ar message
@@ -27,6 +27,11 @@ should be used and piped into
 .Nm .
 .Sh OPTIONS
 .Bl -tag -width "-m message"
+.It Fl P
+Use the blake2bp variant which is a parallel version of blake2b.
+These two variants are incompatible as they produce different
+hashes.  This flag only has an effect when initializing the
+repository.  By default blake2b is used.
 .It Fl Z
 Disable compression support for this repository.
 This flag only has an effect when initializing the repository.
diff --git a/dedup.c b/dedup.c
@@ -33,6 +33,7 @@ static struct blk_hdr blk_hdr;
 static struct icache *icache;
 static int ifd;
 static int sfd;
+static int blake2b_parallel;
 
 int verbose;
 char *argv0;
@@ -104,17 +105,38 @@ free_snap(struct snap *snap)
 static void
 hash_snap(struct snap *snap, uint8_t *md)
 {
-	blake2bp_state ctx;
-	uint64_t i;
+	switch (blake2b_parallel) {
+	case 0: {
+		blake2b_state ctx;
+		uint64_t i;
+
+		blake2b_init(&ctx, MD_SIZE);
+		for (i = 0; i < snap->nr_blk_descs; i++) {
+			struct blk_desc *blk_desc;
+
+			blk_desc = &snap->blk_desc[i];
+			blake2b_update(&ctx, blk_desc->md,
+			                sizeof(blk_desc->md));
+		}
+		blake2b_final(&ctx, md, MD_SIZE);
+		break;
+	}
+	case 1: {
+		blake2bp_state ctx;
+		uint64_t i;
 
-	blake2bp_init(&ctx, MD_SIZE);
-	for (i = 0; i < snap->nr_blk_descs; i++) {
-		struct blk_desc *blk_desc;
+		blake2bp_init(&ctx, MD_SIZE);
+		for (i = 0; i < snap->nr_blk_descs; i++) {
+			struct blk_desc *blk_desc;
 
-		blk_desc = &snap->blk_desc[i];
-		blake2bp_update(&ctx, blk_desc->md, sizeof(blk_desc->md));
+			blk_desc = &snap->blk_desc[i];
+			blake2bp_update(&ctx, blk_desc->md,
+			                sizeof(blk_desc->md));
+		}
+		blake2bp_final(&ctx, md, MD_SIZE);
+		break;
+	}
 	}
-	blake2bp_final(&ctx, md, MD_SIZE);
 }
 
 static struct snap *
@@ -180,11 +202,24 @@ free_buf(uint8_t *buf)
 static void
 hash_blk(uint8_t *buf, size_t size, uint8_t *md)
 {
-	blake2bp_state ctx;
+	switch (blake2b_parallel) {
+	case 0: {
+		blake2b_state ctx;
+
+		blake2b_init(&ctx, MD_SIZE);
+		blake2b_update(&ctx, buf, size);
+		blake2b_final(&ctx, md, MD_SIZE);
+		break;
+	}
+	case 1: {
+		blake2bp_state ctx;
 
-	blake2bp_init(&ctx, MD_SIZE);
-	blake2bp_update(&ctx, buf, size);
-	blake2bp_final(&ctx, md, MD_SIZE);
+		blake2bp_init(&ctx, MD_SIZE);
+		blake2bp_update(&ctx, buf, size);
+		blake2bp_final(&ctx, md, MD_SIZE);
+		break;
+	}
+	}
 }
 
 static void
@@ -337,15 +372,11 @@ check_snap(struct snap *snap, void *arg)
 	buf = alloc_buf(compr_size(BLKSIZE_MAX));
 	for (i = 0; i < snap->nr_blk_descs; i++) {
 		uint8_t md[MD_SIZE];
-		blake2bp_state ctx;
 		struct blk_desc *blk_desc;
 
 		blk_desc = &snap->blk_desc[i];
 		read_blk(buf, blk_desc);
-
-		blake2bp_init(&ctx, MD_SIZE);
-		blake2bp_update(&ctx, buf, blk_desc->size);
-		blake2bp_final(&ctx, md, MD_SIZE);
+		hash_blk(buf, blk_desc->size, md);
 
 		if (memcmp(blk_desc->md, md, sizeof(blk_desc->md)) == 0)
 			continue;
@@ -436,6 +467,7 @@ init_blk_hdr(void)
 {
 	blk_hdr.flags = (VER_MAJ << VER_MAJ_SHIFT) | VER_MIN;
 	blk_hdr.flags |= compr_enabled << COMPR_ENABLED_SHIFT;
+	blk_hdr.flags |= blake2b_parallel << BLAKE2BP_ENABLED_SHIFT;
 	blk_hdr.size = BLK_HDR_SIZE;
 }
 
@@ -451,6 +483,10 @@ load_blk_hdr(void)
 	v = blk_hdr.flags >> COMPR_ENABLED_SHIFT;
 	v &= COMPR_ENABLED_MASK;
 	compr_enabled = v;
+
+	v = blk_hdr.flags >> BLAKE2BP_ENABLED_SHIFT;
+	v &= BLAKE2BP_ENABLED_MASK;
+	blake2b_parallel = v;
 }
 
 static void
@@ -537,7 +573,7 @@ term(void)
 static void
 usage(void)
 {
-	fprintf(stderr, "usage: %s [-Zcilv] [-e id] [-r root] [-m message] [file]\n", argv0);
+	fprintf(stderr, "usage: %s [-PZcilv] [-e id] [-r root] [-m message] [file]\n", argv0);
 	exit(1);
 }
 
@@ -550,6 +586,9 @@ main(int argc, char *argv[])
 	int fd = -1;
 
 	ARGBEGIN {
+	case 'P':
+		blake2b_parallel = 1;
+		break;
 	case 'Z':
 		compr_enabled = 0;
 		break;
diff --git a/dedup.h b/dedup.h
@@ -25,6 +25,9 @@
 #define COMPR_ENABLED_SHIFT	16
 #define COMPR_ENABLED_MASK	0x1
 
+#define BLAKE2BP_ENABLED_SHIFT	17
+#define BLAKE2BP_ENABLED_MASK	0x1
+
 struct icache;
 struct chunker;