dedup

data deduplication program
git clone git://git.2f30.org/dedup.git
Log | Files | Refs | README | LICENSE

commit f28b07e0a5e1b3adfd9b2ac9d252b499ae763918
parent 6d6cce9ed24f393e09afec8f45d3c2566210038d
Author: sin <sin@2f30.org>
Date:   Sun, 17 Feb 2019 22:03:43 +0000

Use a "double" buffer for compression

Diffstat:
Mdedup.c | 43++++++++++++++++++++++---------------------
1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/dedup.c b/dedup.c @@ -394,35 +394,36 @@ lookup_blk(uint8_t *md) void dedup(int fd, char *msg) { - uint8_t *buf, *cbuf; + uint8_t *buf[2]; struct ent *ent; SHA256_CTX ctx; ssize_t n; - buf = alloc_buf(BLKSIZE); - cbuf = alloc_buf(comp_size(BLKSIZE)); + buf[0] = alloc_buf(BLKSIZE); + buf[1] = alloc_buf(comp_size(BLKSIZE)); ent = alloc_ent(); SHA256_Init(&ctx); - while ((n = xread(fd, buf, BLKSIZE)) > 0) { - uint8_t *bp = buf; + while ((n = xread(fd, buf[0], BLKSIZE)) > 0) { + uint8_t *inp = buf[0]; /* input buf */ + uint8_t *outp = buf[1]; /* compressed buf */ while (n > 0) { uint8_t md[MDSIZE]; struct bdescr bdescr; size_t blksize, csize; - blksize = chunk_blk(bp, n); - csize = comp(bp, cbuf, blksize, comp_size(BLKSIZE)); + blksize = chunk_blk(inp, n); + csize = comp(inp, outp, blksize, comp_size(BLKSIZE)); memcpy(bdescr.md, md, sizeof(bdescr)); bdescr.offset = enthdr.store_size; bdescr.size = csize; - hash_blk(cbuf, bdescr.size, bdescr.md); + hash_blk(outp, bdescr.size, bdescr.md); /* Calculate file hash one block at a time */ - SHA256_Update(&ctx, cbuf, bdescr.size); + SHA256_Update(&ctx, outp, bdescr.size); ent = grow_ent(ent, ent->nblks + 1); @@ -433,7 +434,7 @@ dedup(int fd, char *msg) ent->bdescr[ent->nblks++] = bdescr; /* Store block */ - append_blk(cbuf, &bdescr); + append_blk(outp, &bdescr); /* Create a cache entry for this block */ cent = alloc_cent(); @@ -443,7 +444,7 @@ dedup(int fd, char *msg) ent->bdescr[ent->nblks++] = bdescr; } - bp += blksize; + inp += blksize; n -= blksize; } } @@ -466,31 +467,31 @@ dedup(int fd, char *msg) } free(ent); - free(cbuf); - free(buf); + free(buf[1]); + free(buf[0]); } int extract(struct ent *ent, void *arg) { - uint8_t *buf, *cbuf; + uint8_t *buf[2]; struct extract_args *args = arg; uint64_t i; if (memcmp(ent->md, args->md, sizeof(ent->md)) != 0) return WALK_CONTINUE; - buf = alloc_buf(BLKSIZE); - cbuf = alloc_buf(comp_size(BLKSIZE)); + buf[0] = alloc_buf(BLKSIZE); + buf[1] = alloc_buf(comp_size(BLKSIZE)); for (i = 0; i < ent->nblks; i++) { size_t blksize; - read_blk(cbuf, &ent->bdescr[i]); - blksize = decomp(cbuf, buf, ent->bdescr[i].size, BLKSIZE); - xwrite(args->fd, buf, blksize); + read_blk(buf[1], &ent->bdescr[i]); + blksize = decomp(buf[1], buf[0], ent->bdescr[i].size, BLKSIZE); + xwrite(args->fd, buf[0], blksize); } - free(cbuf); - free(buf); + free(buf[1]); + free(buf[0]); return WALK_STOP; }