dedup

deduplicating backup program
git clone git://git.2f30.org/dedup.git
Log | Files | Refs | README | LICENSE

bcompress.c (6415B)


      1 /* Compression layer implementation */
      2 #include <sys/types.h>
      3 #include <sys/stat.h>
      4 
      5 #include <assert.h>
      6 #include <fcntl.h>
      7 #include <stdint.h>
      8 #include <stdio.h>
      9 #include <stdlib.h>
     10 #include <string.h>
     11 #include <strings.h>
     12 #include <unistd.h>
     13 
     14 #include <lz4.h>
     15 #include <snappy-c.h>
     16 
     17 #include "block.h"
     18 #include "config.h"
     19 #include "misc.h"
     20 #include "state.h"
     21 
     22 #define CDNONETYPE	0x200
     23 #define CDSNAPPYTYPE	0x201
     24 #define CDLZ4TYPE	0x202
     25 #define CDSIZE		(8 + 8)
     26 
     27 extern struct param param;
     28 
     29 extern int pack(unsigned char *, char *, ...);
     30 extern int unpack(unsigned char *, char *, ...);
     31 
     32 static int bccreat(struct bctx *bctx, char *path, int mode);
     33 static int bcopen(struct bctx *bctx, char *path, int flags, int mode);
     34 static int bcput(struct bctx *bctx, void *buf, size_t n, unsigned char *md);
     35 static int bcget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n);
     36 static int bcrm(struct bctx *bctx, unsigned char *md);
     37 static int bcgc(struct bctx *bctx);
     38 static int bcsync(struct bctx *bctx);
     39 static int bcclose(struct bctx *bctx);
     40 
     41 static struct bops bops = {
     42 	.creat = bccreat,
     43 	.open = bcopen,
     44 	.put = bcput,
     45 	.get = bcget,
     46 	.rm = bcrm,
     47 	.gc = bcgc,
     48 	.sync = bcsync,
     49 	.close = bcclose,
     50 };
     51 
     52 /* Compression layer context */
     53 struct cctx {
     54 	int type;	/* compression algorithm type for new blocks */
     55 };
     56 
     57 /* Compression descriptor */
     58 struct cd {
     59 	uint16_t type;			/* compression algorithm type */
     60 	unsigned char reserved[6];	/* should be set to 0 when writing */
     61 	uint64_t size;			/* size of compressed block */
     62 };
     63 
     64 /* Unpack compression descriptor */
     65 static int
     66 unpackcd(void *buf, struct cd *cd)
     67 {
     68 	int n;
     69 
     70 	n = unpack(buf, "s'6q",
     71 	           &cd->type,
     72 	           cd->reserved,
     73 	           &cd->size);
     74 
     75 	assert(n == CDSIZE);
     76 	return n;
     77 }
     78 
     79 /* Pack compression descriptor */
     80 static int
     81 packcd(void *buf, struct cd *cd)
     82 {
     83 	int n;
     84 
     85 	n = pack(buf, "s'6q",
     86 	         cd->type,
     87 	         cd->reserved,
     88 	         cd->size);
     89 
     90 	assert(n == CDSIZE);
     91 	return n;
     92 }
     93 
     94 static int
     95 bccreat(struct bctx *bctx, char *path, int mode)
     96 {
     97 	struct cctx *cctx;
     98 	int type;
     99 
    100 	if (strcasecmp(param.calgo, "none") == 0) {
    101 		type = CDNONETYPE;
    102 	} else if (strcasecmp(param.calgo, "snappy") == 0) {
    103 		type = CDSNAPPYTYPE;
    104 	} else if (strcasecmp(param.calgo, "lz4") == 0) {
    105 		type = CDLZ4TYPE;
    106 	} else {
    107 		seterr("invalid compression type: %s", param.calgo);
    108 		return -1;
    109 	}
    110 
    111 	bctx->cctx = calloc(1, sizeof(struct cctx));
    112 	if (bctx->cctx == NULL) {
    113 		seterr("calloc: out of memory");
    114 		return -1;
    115 	}
    116 	cctx = bctx->cctx;
    117 	cctx->type = type;
    118 
    119 	if (bencryptops()->creat(bctx, path, mode) < 0) {
    120 		free(cctx);
    121 		return -1;
    122 	}
    123 	return 0;
    124 }
    125 
    126 static int
    127 bcopen(struct bctx *bctx, char *path, int flags, int mode)
    128 {
    129 	struct cctx *cctx;
    130 	int type;
    131 
    132 	if (strcasecmp(param.calgo, "none") == 0) {
    133 		type = CDNONETYPE;
    134 	} else if (strcasecmp(param.calgo, "snappy") == 0) {
    135 		type = CDSNAPPYTYPE;
    136 	} else if (strcasecmp(param.calgo, "lz4") == 0) {
    137 		type = CDLZ4TYPE;
    138 	} else {
    139 		seterr("invalid compression type: %s", param.calgo);
    140 		return -1;
    141 	}
    142 
    143 	bctx->cctx = calloc(1, sizeof(struct cctx));
    144 	if (bctx->cctx == NULL) {
    145 		seterr("calloc: out of memory");
    146 		return -1;
    147 	}
    148 	cctx = bctx->cctx;
    149 	cctx->type = type;
    150 
    151 	if (bencryptops()->open(bctx, path, flags, mode) < 0) {
    152 		free(cctx);
    153 		return -1;
    154 	}
    155 	return 0;
    156 }
    157 
    158 static int
    159 bcput(struct bctx *bctx, void *buf, size_t n, unsigned char *md)
    160 {
    161 	struct cctx *cctx;
    162 	struct cd cd;
    163 	char *cbuf;
    164 	size_t cn;
    165 	int r;
    166 
    167 	/* Calculate compressed block size */
    168 	cctx = bctx->cctx;
    169 	switch (cctx->type) {
    170 	case CDNONETYPE:
    171 		cn = n;
    172 		break;
    173 	case CDSNAPPYTYPE:
    174 		cn = snappy_max_compressed_length(n);
    175 		break;
    176 	case CDLZ4TYPE:
    177 		cn = LZ4_compressBound(n);
    178 		break;
    179 	}
    180 
    181 	cbuf = malloc(CDSIZE + cn);
    182 	if (cbuf == NULL) {
    183 		seterr("malloc: out of memory");
    184 		return -1;
    185 	}
    186 
    187 	/* Compress block */
    188 	switch (cctx->type) {
    189 	case CDNONETYPE:
    190 		memcpy(&cbuf[CDSIZE], buf, cn);
    191 		break;
    192 	case CDSNAPPYTYPE:
    193 		if (snappy_compress(buf, n, &cbuf[CDSIZE], &cn) != SNAPPY_OK) {
    194 			free(cbuf);
    195 			seterr("snappy_compress: failed");
    196 			return -1;
    197 		}
    198 		break;
    199 	case CDLZ4TYPE:
    200 		r = LZ4_compress_default(buf, &cbuf[CDSIZE], n, cn);
    201 		if (r < 0) {
    202 			free(cbuf);
    203 			seterr("LZ4_compress_default: failed");
    204 			return -1;
    205 		}
    206 		cn = r;
    207 		break;
    208 	}
    209 
    210 	/* Prepare compression descriptor */
    211 	cd.type = cctx->type;
    212 	memset(cd.reserved, 0, sizeof(cd.reserved));
    213 	cd.size = cn;
    214 	/* Prepend compression descriptor */
    215 	packcd(cbuf, &cd);
    216 
    217 	if (bencryptops()->put(bctx, cbuf, CDSIZE + cn, md) < 0) {
    218 		free(cbuf);
    219 		return -1;
    220 	}
    221 
    222 	free(cbuf);
    223 	return cd.size;
    224 }
    225 
    226 static int
    227 bcget(struct bctx *bctx, unsigned char *md, void *buf, size_t *n)
    228 {
    229 	struct cd cd;
    230 	char *cbuf;
    231 	size_t cn, un, size;
    232 	int r;
    233 
    234 	/* Calculate maximum compressed block size */
    235 	size = *n;
    236 	cn = snappy_max_compressed_length(*n);
    237 	if (cn > size)
    238 		size = cn;
    239 	cn = LZ4_compressBound(*n);
    240 	if (cn > size)
    241 		size = cn;
    242 	size += CDSIZE;
    243 
    244 	cbuf = malloc(size);
    245 	if (cbuf == NULL) {
    246 		seterr("malloc: out of memory");
    247 		return -1;
    248 	}
    249 
    250 	if (bencryptops()->get(bctx, md, cbuf, &size) < 0) {
    251 		free(cbuf);
    252 		return -1;
    253 	}
    254 
    255 	unpackcd(cbuf, &cd);
    256 
    257 	/* Decompress block */
    258 	switch (cd.type) {
    259 	case CDNONETYPE:
    260 		un = cd.size;
    261 		if (*n < un) {
    262 			free(cbuf);
    263 			seterr("buffer too small");
    264 			return -1;
    265 		}
    266 		memcpy(buf, &cbuf[CDSIZE], un);
    267 		break;
    268 	case CDSNAPPYTYPE:
    269 		if (snappy_uncompressed_length(&cbuf[CDSIZE], cd.size,
    270 		                               &un) != SNAPPY_OK) {
    271 			free(cbuf);
    272 			seterr("snappy_uncompressed_length: failed");
    273 			return -1;
    274 		}
    275 
    276 		if (*n < un) {
    277 			free(cbuf);
    278 			seterr("buffer too small");
    279 			return -1;
    280 		}
    281 
    282 		if (snappy_uncompress(&cbuf[CDSIZE], cd.size, buf,
    283 		                      &un) != SNAPPY_OK) {
    284 			free(cbuf);
    285 			seterr("snappy_uncompress: failed");
    286 			return -1;
    287 		}
    288 		break;
    289 	case CDLZ4TYPE:
    290 		r = LZ4_decompress_safe(&cbuf[CDSIZE], buf, cd.size, *n);
    291 		if (r < 0) {
    292 			free(cbuf);
    293 			seterr("LZ4_decompress_safe: failed");
    294 			return -1;
    295 		}
    296 		un = r;
    297 		break;
    298 	}
    299 
    300 	free(cbuf);
    301 	*n = un;
    302 	return 0;
    303 }
    304 
    305 static int
    306 bcrm(struct bctx *bctx, unsigned char *md)
    307 {
    308 	return bencryptops()->rm(bctx, md);
    309 }
    310 
    311 static int
    312 bcgc(struct bctx *bctx)
    313 {
    314 	return bencryptops()->gc(bctx);
    315 }
    316 
    317 static int
    318 bcsync(struct bctx *bctx)
    319 {
    320 	return bencryptops()->sync(bctx);
    321 }
    322 
    323 static int
    324 bcclose(struct bctx *bctx)
    325 {
    326 	struct cctx *cctx = bctx->cctx;
    327 
    328 	free(cctx);
    329 	return bencryptops()->close(bctx);
    330 }
    331 
    332 struct bops *
    333 bcompressops(void)
    334 {
    335 	return &bops;
    336 }