dedup

data deduplication program
git clone git://git.2f30.org/dedup.git
Log | Files | Refs | README | LICENSE

utils.c (5534B)


      1 #include <sys/types.h>
      2 
      3 #include <err.h>
      4 #include <stdint.h>
      5 #include <stdio.h>
      6 #include <stdlib.h>
      7 #include <string.h>
      8 #include <unistd.h>
      9 
     10 #include "blake2.h"
     11 #include "dedup.h"
     12 
     13 static void
     14 match_ver(uint64_t v)
     15 {
     16 	uint8_t maj, min;
     17 
     18 	min = v & VER_MIN_MASK;
     19 	maj = (v >> VER_MAJ_SHIFT) & VER_MAJ_MASK;
     20 	if (maj == VER_MAJ && min == VER_MIN)
     21 		return;
     22 	errx(1, "format version mismatch: expected %u.%u but got %u.%u",
     23 	     VER_MAJ, VER_MIN, maj, min);
     24 }
     25 
     26 void
     27 str2bin(char *s, uint8_t *d)
     28 {
     29 	size_t i, size = strlen(s) / 2;
     30 
     31 	for (i = 0; i < size; i++, s += 2)
     32 		sscanf(s, "%2hhx", &d[i]);
     33 }
     34 
     35 off_t
     36 xlseek(int fd, off_t offset, int whence)
     37 {
     38 	off_t ret;
     39 
     40 	ret = lseek(fd, offset, whence);
     41 	if (ret < 0)
     42 		err(1, "lseek");
     43 	return ret;
     44 }
     45 
     46 ssize_t
     47 xread(int fd, void *buf, size_t nbytes)
     48 {
     49 	uint8_t *bp = buf;
     50 	ssize_t total = 0;
     51 
     52 	while (nbytes > 0) {
     53 		ssize_t n;
     54 
     55 		n = read(fd, &bp[total], nbytes);
     56 		if (n < 0)
     57 			err(1, "read");
     58 		else if (n == 0)
     59 			return total;
     60 		total += n;
     61 		nbytes -= n;
     62 	}
     63 	return total;
     64 }
     65 
     66 ssize_t
     67 xwrite(int fd, const void *buf, size_t nbytes)
     68 {
     69 	const uint8_t *bp = buf;
     70 	ssize_t total = 0;
     71 
     72 	while (nbytes > 0) {
     73 		ssize_t n;
     74 
     75 		n = write(fd, &bp[total], nbytes);
     76 		if (n < 0)
     77 			err(1, "write");
     78 		else if (n == 0)
     79 			return total;
     80 		total += n;
     81 		nbytes -= n;
     82 	}
     83 	return total;
     84 }
     85 
     86 void
     87 init_blk_hdr(struct blk_hdr *hdr, int compr_algo, int hash_algo)
     88 {
     89 	hdr->flags = (VER_MAJ << VER_MAJ_SHIFT) | VER_MIN;
     90 	hdr->flags |= compr_algo << COMPR_ALGO_SHIFT;
     91 	hdr->flags |= hash_algo << HASH_ALGO_SHIFT;
     92 	hdr->size = BLK_HDR_SIZE;
     93 }
     94 
     95 void
     96 init_snap_hdr(struct snap_hdr *hdr)
     97 {
     98 	hdr->flags = (VER_MAJ << VER_MAJ_SHIFT) | VER_MIN;
     99 	hdr->size = SNAP_HDR_SIZE;
    100 	hdr->st.min_blk_size = UINT64_MAX;
    101 }
    102 
    103 void
    104 load_blk_hdr(int fd, struct blk_hdr *hdr, int *compr_algo, int *hash_algo)
    105 {
    106 	uint64_t v;
    107 
    108 	read_blk_hdr(fd, hdr);
    109 	match_ver(hdr->flags);
    110 
    111 	v = hdr->flags >> COMPR_ALGO_SHIFT;
    112 	v &= COMPR_ALGO_MASK;
    113 	*compr_algo = v;
    114 
    115 	if (*compr_algo < 0 || *compr_algo >= NR_COMPRS)
    116 		errx(1, "unsupported compression algorithm: %d", *compr_algo);
    117 
    118 	v = hdr->flags >> HASH_ALGO_SHIFT;
    119 	v &= HASH_ALGO_MASK;
    120 	*hash_algo = v;
    121 
    122 	if (*hash_algo < 0 || *hash_algo >= NR_HASHES)
    123 		errx(1, "unsupported hash algorithm: %d", *hash_algo);
    124 }
    125 
    126 void
    127 load_snap_hdr(int fd, struct snap_hdr *hdr)
    128 {
    129 	read_snap_hdr(fd, hdr);
    130 	match_ver(hdr->flags);
    131 }
    132 
    133 struct snap *
    134 alloc_snap(void)
    135 {
    136 	struct snap *snap;
    137 
    138 	snap = calloc(1, sizeof(*snap));
    139 	if (snap == NULL)
    140 		err(1, "%s", __func__);
    141 	return snap;
    142 }
    143 
    144 void
    145 free_snap(struct snap *snap)
    146 {
    147 	free(snap);
    148 }
    149 
    150 struct snap *
    151 grow_snap(struct snap *snap, uint64_t nr_blk_descs)
    152 {
    153 	size_t size;
    154 
    155 	if (nr_blk_descs > SIZE_MAX / sizeof(snap->blk_desc[0]))
    156 		errx(1, "%s: overflow", __func__);
    157 	size = nr_blk_descs * sizeof(snap->blk_desc[0]);
    158 
    159 	if (size > SIZE_MAX - sizeof(*snap))
    160 		errx(1, "%s: overflow", __func__);
    161 	size += sizeof(*snap);
    162 
    163 	snap = realloc(snap, size);
    164 	if (snap == NULL)
    165 		err(1, "%s", __func__);
    166 	return snap;
    167 }
    168 
    169 void
    170 append_snap(int fd, struct snap_hdr *hdr, struct snap *snap)
    171 {
    172 	if (snap->nr_blk_descs > UINT64_MAX / BLK_DESC_SIZE)
    173 	        errx(1, "%s: overflow", __func__);
    174 	snap->size = snap->nr_blk_descs * BLK_DESC_SIZE;
    175 
    176 	if (snap->size > UINT64_MAX - SNAPSHOT_SIZE)
    177 	        errx(1, "%s: overflow", __func__);
    178 	snap->size += SNAPSHOT_SIZE;
    179 
    180 	xlseek(fd, hdr->size, SEEK_SET);
    181 	write_snap(fd, snap);
    182 	write_snap_blk_descs(fd, snap);
    183 
    184 	if (hdr->size > UINT64_MAX - snap->size)
    185 	        errx(1, "%s: overflow", __func__);
    186 	hdr->size += snap->size;
    187 
    188 	if (hdr->nr_snaps > UINT64_MAX - 1)
    189 	        errx(1, "%s: overflow", __func__);
    190 	hdr->nr_snaps++;
    191 }
    192 
    193 /*
    194  * The snapshot hash is calculated over the
    195  * hash of its block descriptors.
    196  */
    197 void
    198 hash_snap(struct snap *snap, uint8_t *md, int hash_algo)
    199 {
    200 	struct hash_ctx ctx;
    201 	uint64_t i;
    202 
    203 	if (hash_init(&ctx, hash_algo, MD_SIZE) < 0)
    204 		errx(1, "hash_init failed");
    205 	for (i = 0; i < snap->nr_blk_descs; i++) {
    206 		struct blk_desc *blk_desc;
    207 
    208 		blk_desc = &snap->blk_desc[i];
    209 		hash_update(&ctx, blk_desc->md, sizeof(blk_desc->md));
    210 	}
    211 	hash_final(&ctx, md, MD_SIZE);
    212 }
    213 
    214 /* Walk through all snapshots and call fn() on each one */
    215 void
    216 walk_snap(int fd, struct snap_hdr *hdr,
    217           int (*fn)(struct snap *, void *), void *arg)
    218 {
    219 	uint64_t i;
    220 
    221 	xlseek(fd, SNAP_HDR_SIZE, SEEK_SET);
    222 	for (i = 0; i < hdr->nr_snaps; i++) {
    223 		struct snap *snap;
    224 		int ret;
    225 
    226 		snap = alloc_snap();
    227 		read_snap(fd, snap);
    228 		snap = grow_snap(snap, snap->nr_blk_descs);
    229 		read_snap_descs(fd, snap);
    230 
    231 		ret = (*fn)(snap, arg);
    232 		free_snap(snap);
    233 		if (ret == WALK_STOP)
    234 			break;
    235 	}
    236 }
    237 
    238 uint8_t *
    239 alloc_buf(size_t size)
    240 {
    241 	void *p;
    242 
    243 	p = calloc(1, size);
    244 	if (p == NULL)
    245 		err(1, "%s", __func__);
    246 	return p;
    247 }
    248 
    249 void
    250 free_buf(uint8_t *buf)
    251 {
    252 	free(buf);
    253 }
    254 
    255 void
    256 read_blk(int fd, uint8_t *buf, struct blk_desc *blk_desc)
    257 {
    258 	ssize_t n;
    259 
    260 	xlseek(fd, blk_desc->offset, SEEK_SET);
    261 	n = xread(fd, buf, blk_desc->size);
    262 	if (n == 0)
    263 		errx(1, "%s: unexpected EOF", __func__);
    264 	if (n != blk_desc->size)
    265 		errx(1, "%s: short read", __func__);
    266 }
    267 
    268 void
    269 append_blk(int fd, struct blk_hdr *hdr, uint8_t *buf, struct blk_desc *blk_desc)
    270 {
    271 	xlseek(fd, hdr->size, SEEK_SET);
    272 	xwrite(fd, buf, blk_desc->size);
    273 
    274 	if (hdr->size > UINT64_MAX - blk_desc->size)
    275 		errx(1, "%s: overflow", __func__);
    276 	hdr->size += blk_desc->size;
    277 }
    278 
    279 void
    280 hash_blk(uint8_t *buf, size_t size, uint8_t *md, int hash_algo)
    281 {
    282 	struct hash_ctx ctx;
    283 
    284 	if (hash_init(&ctx, hash_algo, MD_SIZE) < 0)
    285 		errx(1, "hash_init failed");
    286 	hash_update(&ctx, buf, size);
    287 	hash_final(&ctx, md, MD_SIZE);
    288 }