dedup

data deduplication program
git clone git://git.2f30.org/dedup.git
Log | Files | Refs | README | LICENSE

commit 94a49be0c19300d3a57a3d25ba58236c67e912a2
parent 78273e89af8024ff892a83cdb04b8b2374aa9521
Author: sin <sin@2f30.org>
Date:   Thu, 18 Apr 2019 10:42:49 +0100

Move block helpers to utils.c

Diffstat:
Mdcheck.c | 45++-------------------------------------------
Mdedup.h | 6++++++
Mdpack.c | 37++-----------------------------------
Mdunpack.c | 32+-------------------------------
Mutils.c | 52++++++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 63 insertions(+), 109 deletions(-)

diff --git a/dcheck.c b/dcheck.c @@ -36,47 +36,6 @@ print_md(FILE *fp, uint8_t *md, size_t size) fprintf(fp, "%02x", md[i]); } -static uint8_t * -alloc_buf(size_t size) -{ - void *p; - - p = calloc(1, size); - if (p == NULL) - err(1, "%s", __func__); - return p; -} - -static void -free_buf(uint8_t *buf) -{ - free(buf); -} - -static void -hash_blk(uint8_t *buf, size_t size, uint8_t *md) -{ - struct hash_ctx ctx; - - if (hash_init(&ctx, hash_algo, MD_SIZE) < 0) - errx(1, "hash_init failed"); - hash_update(&ctx, buf, size); - hash_final(&ctx, md, MD_SIZE); -} - -static void -read_blk(uint8_t *buf, struct blk_desc *blk_desc) -{ - ssize_t n; - - xlseek(sfd, blk_desc->offset, SEEK_SET); - n = xread(sfd, buf, blk_desc->size); - if (n == 0) - errx(1, "%s: unexpected EOF", __func__); - if (n != blk_desc->size) - errx(1, "%s: short read", __func__); -} - /* * Hash every block referenced by the given snapshot * and compare its hash with the one stored in the corresponding @@ -104,8 +63,8 @@ check_snap(struct snap *snap, void *arg) struct blk_desc *blk_desc; blk_desc = &snap->blk_desc[i]; - read_blk(buf, blk_desc); - hash_blk(buf, blk_desc->size, md); + read_blk(sfd, buf, blk_desc); + hash_blk(buf, blk_desc->size, md, hash_algo); if (memcmp(blk_desc->md, md, sizeof(blk_desc->md)) == 0) continue; diff --git a/dedup.h b/dedup.h @@ -222,3 +222,9 @@ void append_snap(int fd, struct snap_hdr *hdr, struct snap *snap); void hash_snap(struct snap *snap, uint8_t *md, int hash_algo); void walk_snap(int fd, struct snap_hdr *hdr, int (*fn)(struct snap *, void *), void *arg); +uint8_t *alloc_buf(size_t size); +void free_buf(uint8_t *buf); +void read_blk(int fd, uint8_t *buf, struct blk_desc *blk_desc); +void append_blk(int fd, struct blk_hdr *hdr, uint8_t *buf, + struct blk_desc *blk_desc); +void hash_blk(uint8_t *buf, size_t size, uint8_t *md, int hash_algo); diff --git a/dpack.c b/dpack.c @@ -28,39 +28,6 @@ static int compr_algo = COMPR_LZ4; int verbose; char *argv0; -static uint8_t * -alloc_buf(size_t size) -{ - void *p; - - p = calloc(1, size); - if (p == NULL) - err(1, "%s", __func__); - return p; -} - -static void -hash_blk(uint8_t *buf, size_t size, uint8_t *md) -{ - struct hash_ctx ctx; - - if (hash_init(&ctx, hash_algo, MD_SIZE) < 0) - errx(1, "hash_init failed"); - hash_update(&ctx, buf, size); - hash_final(&ctx, md, MD_SIZE); -} - -static void -append_blk(uint8_t *buf, struct blk_desc *blk_desc) -{ - xlseek(sfd, blk_hdr.size, SEEK_SET); - xwrite(sfd, buf, blk_desc->size); - - if (blk_hdr.size > UINT64_MAX - blk_desc->size) - errx(1, "%s: overflow", __func__); - blk_hdr.size += blk_desc->size; -} - static void dedup_chunk(struct snap *snap, uint8_t *chunkp, size_t chunk_size) { @@ -76,7 +43,7 @@ dedup_chunk(struct snap *snap, uint8_t *chunkp, size_t chunk_size) compr_buf = alloc_buf(csize); n = compr(&ctx, chunkp, compr_buf, chunk_size, csize); - hash_blk(compr_buf, n, md); + hash_blk(compr_buf, n, md, hash_algo); snap_hdr.st.orig_size += chunk_size; snap_hdr.st.compr_size += n; @@ -87,7 +54,7 @@ dedup_chunk(struct snap *snap, uint8_t *chunkp, size_t chunk_size) blk_desc.size = n; snap->blk_desc[snap->nr_blk_descs++] = blk_desc; - append_blk(compr_buf, &blk_desc); + append_blk(sfd, &blk_hdr, compr_buf, &blk_desc); insert_icache(icache, &blk_desc); diff --git a/dunpack.c b/dunpack.c @@ -33,36 +33,6 @@ static int compr_algo = COMPR_LZ4; int verbose; char *argv0; -static uint8_t * -alloc_buf(size_t size) -{ - void *p; - - p = calloc(1, size); - if (p == NULL) - err(1, "%s", __func__); - return p; -} - -static void -free_buf(uint8_t *buf) -{ - free(buf); -} - -static void -read_blk(uint8_t *buf, struct blk_desc *blk_desc) -{ - ssize_t n; - - xlseek(sfd, blk_desc->offset, SEEK_SET); - n = xread(sfd, buf, blk_desc->size); - if (n == 0) - errx(1, "%s: unexpected EOF", __func__); - if (n != blk_desc->size) - errx(1, "%s: short read", __func__); -} - static int extract(struct snap *snap, void *arg) { @@ -83,7 +53,7 @@ extract(struct snap *snap, void *arg) size_t blksize; blk_desc = &snap->blk_desc[i]; - read_blk(buf[1], blk_desc); + read_blk(sfd, buf[1], blk_desc); blksize = decompr(&ctx, buf[1], buf[0], blk_desc->size, BLKSIZE_MAX); xwrite(args->fd, buf[0], blksize); } diff --git a/utils.c b/utils.c @@ -234,3 +234,55 @@ walk_snap(int fd, struct snap_hdr *hdr, break; } } + +uint8_t * +alloc_buf(size_t size) +{ + void *p; + + p = calloc(1, size); + if (p == NULL) + err(1, "%s", __func__); + return p; +} + +void +free_buf(uint8_t *buf) +{ + free(buf); +} + +void +read_blk(int fd, uint8_t *buf, struct blk_desc *blk_desc) +{ + ssize_t n; + + xlseek(fd, blk_desc->offset, SEEK_SET); + n = xread(fd, buf, blk_desc->size); + if (n == 0) + errx(1, "%s: unexpected EOF", __func__); + if (n != blk_desc->size) + errx(1, "%s: short read", __func__); +} + +void +append_blk(int fd, struct blk_hdr *hdr, uint8_t *buf, struct blk_desc *blk_desc) +{ + xlseek(fd, hdr->size, SEEK_SET); + xwrite(fd, buf, blk_desc->size); + + if (hdr->size > UINT64_MAX - blk_desc->size) + errx(1, "%s: overflow", __func__); + hdr->size += blk_desc->size; +} + +void +hash_blk(uint8_t *buf, size_t size, uint8_t *md, int hash_algo) +{ + struct hash_ctx ctx; + + if (hash_init(&ctx, hash_algo, MD_SIZE) < 0) + errx(1, "hash_init failed"); + hash_update(&ctx, buf, size); + hash_final(&ctx, md, MD_SIZE); +}