dedup

deduplicating backup program
git clone git://git.2f30.org/dedup.git
Log | Files | Refs | README | LICENSE

commit 03aa3e43c5b81940bab9e6990d71bf036a188f27
parent 14a97f465c31b9e0c707730ca8798dfd4140cb03
Author: sin <sin@2f30.org>
Date:   Tue,  7 May 2019 20:56:44 +0100

Change snapshot format

Include a header at the front so we can store the nonce etc.

Diffstat:
Msnap.c | 136++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 128 insertions(+), 8 deletions(-)

diff --git a/snap.c b/snap.c @@ -2,6 +2,7 @@ #include <sys/types.h> #include <sys/stat.h> +#include <assert.h> #include <errno.h> #include <fcntl.h> #include <limits.h> @@ -12,13 +13,37 @@ #include <string.h> #include <unistd.h> +#include <sodium.h> + #include "config.h" #include "misc.h" #include "queue.h" #include "snap.h" +#define SHDRMAGIC "SNAPSNAPPYSNOOP" +#define NSHDRMAGIC sizeof(SHDRMAGIC) +#define VMIN 0 +#define VMAJ 1 +#define VMINMASK 0xff +#define VMAJSHIFT 8 +#define VMAJMASK 0xff +#define SHDRSIZE (NSHDRMAGIC + 24 + 24 + 8 + 8) + #define NERRBUF 128 +/* misc helpers */ +extern int pack(unsigned char *, char *, ...); +extern int unpack(unsigned char *, char *, ...); + +/* Snapshot header structure */ +struct shdr { + char magic[NSHDRMAGIC]; /* magic number for file(1) */ + unsigned char nonce[crypto_aead_xchacha20poly1305_ietf_NPUBBYTES]; + unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES]; + uint64_t flags; /* version number */ + uint64_t nbd; /* number of block hashes */ +}; + struct mdnode { unsigned char md[MDSIZE]; /* hash of block */ TAILQ_ENTRY(mdnode) e; /* mdhead link node */ @@ -29,10 +54,54 @@ struct sctx { struct mdnode *mdnext; /* next hash to be returned via sget() */ int fd; /* underlying snapshot file descriptor */ int rdonly; /* when set to 1, the ssync() operation is a no-op */ + struct shdr shdr; }; static char errbuf[NERRBUF]; +/* Read snapshot header */ +static int +unpackshdr(int fd, struct shdr *shdr) +{ + unsigned char buf[SHDRSIZE]; + int n; + + if (xread(fd, buf, sizeof(buf)) != sizeof(buf)) { + sseterr("failed to read snapshot header: %s", strerror(errno)); + return -1; + } + + n = unpack(buf, "'16'24qq", + shdr->magic, + shdr->nonce, + &shdr->flags, + &shdr->nbd); + + assert(n == sizeof(buf)); + return n; +} + +/* Write snapshot header */ +static int +packshdr(int fd, struct shdr *shdr) +{ + unsigned char buf[SHDRSIZE]; + int n; + + n = pack(buf, "'16'24qq", + shdr->magic, + shdr->nonce, + shdr->flags, + shdr->nbd); + + assert(n == SHDRSIZE); + if (xwrite(fd, buf, n) != n) { + sseterr("failed to write snapshot header: %s", strerror(errno)); + return -1; + } + return n; +} + static int loadmd(struct sctx *sctx) { @@ -55,15 +124,11 @@ loadmd(struct sctx *sctx) static int initmdhead(struct sctx *sctx) { - struct stat st; - uint64_t i, n; + struct shdr *shdr; + uint64_t i; - if (fstat(sctx->fd, &st) < 0) { - sseterr("fstat: %s", strerror(errno)); - return -1; - } - n = st.st_size / MDSIZE; - for (i = 0; i < n; i++) { + shdr = &sctx->shdr; + for (i = 0; i < shdr->nbd; i++) { if (loadmd(sctx) == 0) continue; @@ -83,6 +148,7 @@ initmdhead(struct sctx *sctx) int screat(char *path, int mode, struct sctx **sctx) { + struct shdr *shdr; int fd; if (path == NULL || sctx == NULL) { @@ -90,6 +156,11 @@ screat(char *path, int mode, struct sctx **sctx) return -1; } + if (sodium_init() < 0) { + sseterr("sodium_init: failed"); + return -1; + } + fd = open(path, O_RDWR | O_CREAT | O_EXCL, mode); if (fd < 0) { sseterr("open: %s", strerror(errno)); @@ -106,12 +177,25 @@ screat(char *path, int mode, struct sctx **sctx) TAILQ_INIT(&(*sctx)->mdhead); (*sctx)->mdnext = NULL; (*sctx)->fd = fd; + + shdr = &(*sctx)->shdr; + memcpy(shdr->magic, SHDRMAGIC, NSHDRMAGIC); + shdr->flags = (VMAJ << VMAJSHIFT) | VMIN; + shdr->nbd = 0; + + if (packshdr(fd, shdr) < 0) { + free(*sctx); + close(fd); + return -1; + } + return 0; } int sopen(char *path, int flags, int mode, struct sctx **sctx) { + struct shdr *shdr; int fd; if (path == NULL || sctx == NULL) { @@ -125,6 +209,11 @@ sopen(char *path, int flags, int mode, struct sctx **sctx) return -1; } + if (sodium_init() < 0) { + sseterr("sodium_init: failed"); + return -1; + } + fd = open(path, O_RDONLY, mode); if (fd < 0) { sseterr("open: %s", strerror(errno)); @@ -143,6 +232,29 @@ sopen(char *path, int flags, int mode, struct sctx **sctx) (*sctx)->fd = fd; (*sctx)->rdonly = 1; + shdr = &(*sctx)->shdr; + + if (unpackshdr(fd, shdr) < 0) { + free(sctx); + close(fd); + return -1; + } + + if (memcmp(shdr->magic, SHDRMAGIC, NSHDRMAGIC) != 0) { + free(sctx); + close(fd); + sseterr("unknown snapshot header magic"); + return -1; + } + + /* If the major version is different, the format is incompatible */ + if (((shdr->flags >> VMAJSHIFT) & VMAJMASK) != VMAJ) { + free(sctx); + close(fd); + sseterr("snapshot header version mismatch"); + return -1; + } + if (initmdhead(*sctx) < 0) { free(*sctx); close(fd); @@ -154,6 +266,7 @@ sopen(char *path, int flags, int mode, struct sctx **sctx) int sput(struct sctx *sctx, unsigned char *md) { + struct shdr *shdr; struct mdnode *mdnode; if (sctx == NULL || md == NULL) { @@ -166,6 +279,8 @@ sput(struct sctx *sctx, unsigned char *md) sseterr("calloc: %s", strerror(errno)); return -1; } + shdr = &sctx->shdr; + shdr->nbd++; memcpy(mdnode->md, md, MDSIZE); TAILQ_INSERT_TAIL(&sctx->mdhead, mdnode, e); return 0; @@ -208,6 +323,7 @@ srewind(struct sctx *sctx) int ssync(struct sctx *sctx) { + struct shdr *shdr; struct mdnode *mdnode; if (sctx == NULL) { @@ -222,6 +338,10 @@ ssync(struct sctx *sctx) sseterr("lseek: %s", strerror(errno)); return -1; } + + shdr = &sctx->shdr; + if (packshdr(sctx->fd, shdr) < 0) + return -1; TAILQ_FOREACH(mdnode, &sctx->mdhead, e) { if (xwrite(sctx->fd, mdnode->md, MDSIZE) != MDSIZE) { sseterr("failed to write block hash: %s",