Commit 0e1b3b19 authored by Mark Fasheh's avatar Mark Fasheh

- initial commit for duperemove from my private repo. This will be the

  official development repo from now on.

- De-dupe support is very experimental and untested right now.

- You can also find a tool, btrfs-extent-same here which I use for testing
  the extent-same ioctl in btrfs.
Signed-off-by: 's avatarMark Fasheh <mfasheh@suse.de>
parents
*.o
duperemove
btrfs-extent-same
*~
This diff is collapsed.
CC=gcc
RELEASE=v0.04
CFLAGS=-Wall -ggdb -D_FILE_OFFSET_BITS=64 -DVERSTRING=\"$(RELEASE)\"
LIBRARY_FLAGS=-lmhash
DIST_SOURCES=csum.c csum.h duperemove.c hash-tree.c hash-tree.h results-tree.c results-tree.h kernel.h LICENSE list.h Makefile rbtree.c rbtree.h rbtree.txt README TODO dedupe.c dedupe.h btrfs-ioctl.h
DIST=duperemove-$(RELEASE)
DIST_TARBALL=$(DIST).tar.gz
TEMP_INSTALL_DIR:=$(shell mktemp -du -p .)
objects = duperemove.o rbtree.o csum.o hash-tree.o results-tree.o dedupe.o
progs = duperemove
all: $(progs) kernel.h list.h btrfs-ioctl.h
duperemove: $(objects) kernel.h duperemove.c
$(CC) $(objects) $(LIBRARY_FLAGS) -o duperemove
tarball: clean
mkdir -p $(TEMP_INSTALL_DIR)/$(DIST)
cp $(DIST_SOURCES) $(TEMP_INSTALL_DIR)/$(DIST)
tar -C $(TEMP_INSTALL_DIR) -zcf $(DIST_TARBALL) $(DIST)
rm -fr $(TEMP_INSTALL_DIR)
btrfs-extent-same: btrfs-extent-same.c
$(CC) -Wall -o btrfs-extent-same btrfs-extent-same.c
clean:
rm -fr $(objects) $(progs) $(DIST_TARBALL) btrfs-extent-same *~
HIGH LEVEL
- Test on actual VM images
- Add code to take a directory as a command line option
LOW LEVEL
- Replace ugly pointer comparison in walk_dupe_block()
- Wrap bytes <-> blocks conversions
- Possibly use mmap (with madvise) for the checksumming phase
- Do checksumming in seperate threads
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stddef.h>
#include <errno.h>
#include <string.h>
#include <sys/ioctl.h>
#define BTRFS_IOCTL_MAGIC 0x94
#define BTRFS_IOC_FILE_EXTENT_SAME _IOWR(BTRFS_IOCTL_MAGIC, 54, \
struct btrfs_ioctl_same_args)
#define BTRFS_SAME_DATA_DIFFERS 1
/* For extent-same ioctl */
struct btrfs_ioctl_same_extent_info {
int64_t fd; /* in - destination file */
uint64_t logical_offset; /* in - start of extent in destination */
uint64_t bytes_deduped; /* out - total # of bytes we
* were able to dedupe from
* this file */
/* status of this dedupe operation:
* 0 if dedup succeeds
* < 0 for error
* == BTRFS_SAME_DATA_DIFFERS if data differs
*/
int32_t status; /* out - see above description */
uint32_t reserved;
};
struct btrfs_ioctl_same_args {
uint64_t logical_offset; /* in - start of extent in source */
uint64_t length; /* in - length of extent */
uint16_t total_files; /* in - total elements in info array */
uint16_t files_deduped; /* out - number of files that got deduped */
uint32_t reserved;
struct btrfs_ioctl_same_extent_info info[0];
};
static void usage(const char *prog)
{
printf("Usage: %s len file1 loff1 file2 loff2\n", prog);
}
int main(int argc, char **argv)
{
int ret, src_fd, i, numfiles;
char *srcf, *destf;
struct btrfs_ioctl_same_args *same;
struct btrfs_ioctl_same_extent_info *info;
unsigned long long bytes = 0ULL;
if (argc < 6 || (argc % 2)) {
usage(argv[0]);
return 1;
}
numfiles = (argc / 2) - 2;
printf("Deduping %d total files\n", numfiles + 1);
same = calloc(1,
sizeof(struct btrfs_ioctl_same_args) +
sizeof(struct btrfs_ioctl_same_extent_info) * numfiles);
if (!same)
return -ENOMEM;
srcf = argv[2];
same->length = atoll(argv[1]);
same->logical_offset = atoll(argv[3]);
same->total_files = numfiles;
ret = open(srcf, O_RDONLY);
if (ret < 0) {
ret = errno;
fprintf(stderr, "Could not open file %s: (%d) %s\n", srcf, ret,
strerror(ret));
return -ret;
}
src_fd = ret;
printf("(%llu, %llu): %s\n", (unsigned long long)same->logical_offset,
(unsigned long long)same->length, srcf);
for (i = 0; i < same->total_files; i++) {
destf = argv[4 + (i * 2)];
ret = open(destf, O_RDONLY);
if (ret < 0) {
ret = errno;
fprintf(stderr, "Could not open file %s: (%d) %s\n",
destf, ret, strerror(ret));
return -ret;
}
same->info[i].fd = ret;
same->info[i].logical_offset = atoll(argv[5 + (i * 2)]);
printf("(%llu, %llu): %s\n",
(unsigned long long)same->info[i].logical_offset,
(unsigned long long)same->length, destf);
}
ret = ioctl(src_fd, BTRFS_IOC_FILE_EXTENT_SAME, same);
if (ret < 0) {
ret = errno;
fprintf(stderr, "btrfs_same returned error: (%d) %s\n", ret,
strerror(ret));
return -ret;
}
printf("%u of %u files deduped\n", same->files_deduped, same->total_files);
for (i = 0; i < same->total_files; i++) {
info = &same->info[i];
printf("i: %d, status: %d, bytes_deduped: %llu\n", i,
info->status, (unsigned long long)info->bytes_deduped);
bytes += info->bytes_deduped;
}
printf("%llu total bytes deduped in this operation\n", bytes);
return ret;
}
#ifndef __BTRFS_IOCTL_H__
#define __BTRFS_IOCTL_H__
#include <stdint.h>
#include <stddef.h>
#include <sys/ioctl.h>
#define BTRFS_IOCTL_MAGIC 0x94
#define BTRFS_IOC_FILE_EXTENT_SAME _IOWR(BTRFS_IOCTL_MAGIC, 54, \
struct btrfs_ioctl_same_args)
#define BTRFS_SAME_DATA_DIFFERS 1
/* For extent-same ioctl */
struct btrfs_ioctl_same_extent_info {
int64_t fd; /* in - destination file */
uint64_t logical_offset; /* in - start of extent in destination */
uint64_t bytes_deduped; /* out - total # of bytes we
* were able to dedupe from
* this file */
/* status of this dedupe operation:
* 0 if dedup succeeds
* < 0 for error
* == BTRFS_SAME_DATA_DIFFERS if data differs
*/
int32_t status; /* out - see above description */
uint32_t reserved;
};
struct btrfs_ioctl_same_args {
uint64_t logical_offset; /* in - start of extent in source */
uint64_t length; /* in - length of extent */
uint16_t total_files; /* in - total elements in info array */
uint16_t files_deduped; /* out - number of files that got deduped */
uint32_t reserved;
struct btrfs_ioctl_same_extent_info info[0];
};
static inline int btrfs_extent_same(int fd,
struct btrfs_ioctl_same_args *same)
{
return ioctl(fd, BTRFS_IOC_FILE_EXTENT_SAME, same);
}
#endif /* __BTRFS_IOCTL_H__ */
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <mhash.h>
#include "csum.h"
static MHASH td;
#define HASH_FUNC MHASH_SHA256
unsigned int digest_len = 0;
void checksum_block(char *buf, int len, unsigned char *digest)
{
td = mhash_init(HASH_FUNC);
if (td == MHASH_FAILED)
abort();
mhash(td, buf, len);
mhash_deinit(td, digest);
}
int init_hash(void)
{
digest_len = mhash_get_block_size(HASH_FUNC);
if (!digest_len)
return 1;
if (digest_len == 0 || digest_len > DIGEST_LEN_MAX)
abort();
return 0;
}
void debug_print_digest(FILE *stream, unsigned char *digest)
{
int i;
for (i = 0; i < digest_len; i++)
fprintf(stream, "%.2x", digest[i]);
}
struct running_checksum {
MHASH td;
unsigned char digest[DIGEST_LEN_MAX];
};
struct running_checksum *start_running_checksum(void)
{
struct running_checksum *c = calloc(1, sizeof(struct running_checksum));
if (c)
c->td = mhash_init(HASH_FUNC);
return c;
}
void add_to_running_checksum(struct running_checksum *c,
unsigned int len, unsigned char *buf)
{
mhash(c->td, buf, len);
}
void finish_running_checksum(struct running_checksum *c, unsigned char *digest)
{
mhash_deinit(c->td, digest);
}
#ifndef __CSUM_H__
#define __CSUM_H__
#include <stdio.h>
#define DIGEST_LEN_MAX 32
extern unsigned int digest_len;
/* Init / debug */
int init_hash(void);
void debug_print_digest(FILE *stream, unsigned char *digest);
/* Checksums a single block in one go. */
void checksum_block(char *buf, int len, unsigned char *digest);
/* Keeping a 'running' checksum - we add data to it a bit at a time */
struct running_checksum;
struct running_checksum *start_running_checksum(void);
void add_to_running_checksum(struct running_checksum *c,
unsigned int len, unsigned char *buf);
void finish_running_checksum(struct running_checksum *c, unsigned char *digest);
#endif /* __CSUM_H__ */
#include <stdlib.h>
#include "dedupe.h"
void free_dedupe_ctxt(struct dedupe_ctxt *ctxt)
{
if (ctxt) {
if (ctxt->filerec_index)
free(ctxt->filerec_index);
if (ctxt->same)
free(ctxt->same);
free(ctxt);
}
}
struct dedupe_ctxt *new_dedupe_ctxt(unsigned int max_extents, uint64_t loff,
uint64_t elen, int fd,
unsigned int filerec_index)
{
struct dedupe_ctxt *ctxt = calloc(1, sizeof(*ctxt));
struct btrfs_ioctl_same_args *same;
unsigned int same_size;
if (ctxt == NULL)
return NULL;
ctxt->filerec_index = calloc(max_extents, sizeof(unsigned int));
if (ctxt->filerec_index == NULL) {
free(ctxt);
return NULL;
}
same_size = sizeof(*same) +
max_extents * sizeof(struct btrfs_ioctl_same_extent_info);
same = calloc(1, same_size);
if (same == NULL) {
free(ctxt->filerec_index);
free(ctxt);
return NULL;
}
ctxt->same = same;
ctxt->max_extents = max_extents;
ctxt->len = ctxt->same->length = elen;
ctxt->ioctl_fd = fd;
ctxt->ioctl_fd_index = filerec_index;
ctxt->ioctl_fd_off = same->logical_offset = loff;
return ctxt;
}
void add_extent_to_dedupe(struct dedupe_ctxt *ctxt, uint64_t loff, uint64_t len,
int fd, unsigned int filerec_index)
{
int i = ctxt->same->total_files;
struct btrfs_ioctl_same_args *same = ctxt->same;
if (ctxt->same->total_files >= ctxt->max_extents)
abort();
same->info[i].logical_offset = loff;
same->info[i].fd = fd;
ctxt->filerec_index[i] = filerec_index;
same->total_files++;
}
int dedupe_extents(struct dedupe_ctxt *ctxt)
{
return btrfs_extent_same(ctxt->ioctl_fd, ctxt->same);
}
void get_dedupe_result(struct dedupe_ctxt *ctxt, int idx, int *status,
uint64_t *off, uint64_t *bytes_deduped,
unsigned int *filerec_index)
{
struct btrfs_ioctl_same_extent_info *info = &ctxt->same->info[idx];
*status = info->status;
*off = info->logical_offset;
*bytes_deduped = info->bytes_deduped;
*filerec_index = ctxt->filerec_index[idx];
}
#ifndef __DEDUPE_H__
#define __DEDUPE_H__
#include "btrfs-ioctl.h"
struct dedupe_ctxt {
unsigned int max_extents;
uint64_t len;
int ioctl_fd;
unsigned int ioctl_fd_index;
uint64_t ioctl_fd_off;
unsigned int *filerec_index;
struct btrfs_ioctl_same_args *same;
};
struct dedupe_ctxt *new_dedupe_ctxt(unsigned int num_extents, uint64_t loff,
uint64_t elen, int fd,
unsigned int filerec_index);
void free_dedupe_ctxt(struct dedupe_ctxt *ctxt);
void add_extent_to_dedupe(struct dedupe_ctxt *ctxt, uint64_t loff, uint64_t len,
int fd, unsigned int filerec_index);
int dedupe_extents(struct dedupe_ctxt *ctxt);
void get_dedupe_result(struct dedupe_ctxt *ctxt, int idx, int *status,
uint64_t *off, uint64_t *bytes_deduped,
unsigned int *filerec_index);
static inline int num_dedupe_requests(struct dedupe_ctxt *ctxt)
{
return ctxt->same->total_files;
}
#endif /* __BTRFS_IOCTL_H__ */
This diff is collapsed.
#include <stdlib.h>
#include <errno.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include "kernel.h"
#include "rbtree.h"
#include "list.h"
#include "csum.h" /* for digest_len variable and DIGEST_LEN_MAX */
#include "hash-tree.h"
static void insert_block_list(struct hash_tree *tree,
struct dupe_blocks_list *list)
{
struct rb_node **p = &tree->root.rb_node;
struct rb_node *parent = NULL;
struct dupe_blocks_list *tmp;
int cmp;
while (*p) {
parent = *p;
tmp = rb_entry(parent, struct dupe_blocks_list, dl_node);
cmp = memcmp(list->dl_hash, tmp->dl_hash, digest_len);
if (cmp < 0)
p = &(*p)->rb_left;
else if (cmp > 0)
p = &(*p)->rb_right;
else abort(); /* We should never find a duplicate */
}
rb_link_node(&list->dl_node, parent, p);
rb_insert_color(&list->dl_node, &tree->root);
tree->num_hashes++;
return;
}
static struct dupe_blocks_list *find_block_list(struct hash_tree *tree,
unsigned char *digest)
{
struct rb_node *n = tree->root.rb_node;
struct dupe_blocks_list *list;
int cmp;
while (n) {
list = rb_entry(n, struct dupe_blocks_list, dl_node);
cmp = memcmp(digest, list->dl_hash, digest_len);
if (cmp < 0)
n = n->rb_left;
else if (cmp > 0)
n = n->rb_right;
else return list;
}
return NULL;
}
struct file_block * insert_hashed_block(struct hash_tree *tree,
unsigned char *digest,
unsigned int fileid, uint64_t loff,
struct list_head *head)
{
struct file_block *e = malloc(sizeof(*e));
struct dupe_blocks_list *d;
if (!e)
return NULL;
d = find_block_list(tree, digest);
if (d == NULL) {
d = calloc(1, sizeof(*d));
if (!d)
return NULL;
memcpy(d->dl_hash, digest, digest_len);
rb_init_node(&d->dl_node);
INIT_LIST_HEAD(&d->dl_list);
insert_block_list(tree, d);
}
e->b_file = fileid;
e->b_loff = loff;
if (head)
list_add_tail(&e->b_file_next, head);
else
INIT_LIST_HEAD(&e->b_file_next);
e->b_parent = d;
d->dl_num_elem++;
list_add_tail(&e->b_list, &d->dl_list);
tree->num_blocks++;
return e;
}
void for_each_dupe(struct file_block *block, unsigned int fileid,
for_each_dupe_t func, void *priv)
{
struct dupe_blocks_list *parent = block->b_parent;
struct file_block *cur;
list_for_each_entry(cur, &parent->dl_list, b_list) {
/* Ignore self and any blocks from another fileid */
if (cur == block)
continue;
if (cur->b_file != fileid)
continue;
if (func(cur, priv))
break;
}
}
static unsigned int seen_counter = 1;
int block_seen(struct file_block *block)
{
return !!(block->b_seen == seen_counter);
}
void mark_block_seen(struct file_block *block)
{
block->b_seen = seen_counter;
}
void clear_all_seen_blocks(void)
{
seen_counter++;
}
void init_hash_tree(struct hash_tree *tree)
{
tree->root = RB_ROOT;
tree->num_blocks = tree->num_hashes = 0;
}
#ifndef __HASH_TREE__
#define __HASH_TREE__
struct hash_tree {
struct rb_root root;
unsigned int num_blocks;
unsigned int num_hashes;
};
struct dupe_blocks_list {
struct rb_node dl_node;
unsigned int dl_num_elem;
struct list_head dl_list;
unsigned char dl_hash[DIGEST_LEN_MAX];
};
struct file_block {
struct dupe_blocks_list *b_parent;
unsigned int b_file;
unsigned int b_seen;
uint64_t b_loff;
struct list_head b_list; /* For d_list, all blocks
* with this md5. */
struct list_head b_file_next; /* Points to the next logical
* extent for this file. */
};
struct file_block * insert_hashed_block(struct hash_tree *tree,
unsigned char *digest,
unsigned int fileid, uint64_t loff,
struct list_head *head);
typedef int (for_each_dupe_t)(struct file_block *, void *);
void for_each_dupe(struct file_block *block, unsigned int fileid,
for_each_dupe_t func, void *priv);
int block_seen(struct file_block *block);
void mark_block_seen(struct file_block *block);
void clear_all_seen_blocks(void);
void init_hash_tree(struct hash_tree *tree);
#endif /* __HASH_TREE__ */
#ifndef _LINUX_KERNEL_H
#define _LINUX_KERNEL_H
/**
* container_of - cast a member of a structure out to the containing structure
* @ptr: the pointer to the member.
* @type: the type of the container struct this is embedded in.
* @member: the name of the member within the struct.
*
*/
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
This diff is collapsed.
This diff is collapsed.
/*
Red Black Trees
(C) 1999 Andrea Arcangeli <andrea@suse.de>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
linux/include/linux/rbtree.h
To use rbtrees you'll have to implement your own insert and search cores.
This will avoid us to use callbacks and to drop drammatically performances.
I know it's not the cleaner way, but in C (not in C++) to get
performances and genericity...
Some example of insert and search follows here. The search is a plain
normal search over an ordered tree. The insert instead must be implemented
in two steps: First, the code must insert the element in order as a red leaf
in the tree, and then the support library function rb_insert_color() must
be called. Such function will do the not trivial work to rebalance the
rbtree, if necessary.
-----------------------------------------------------------------------
static inline struct page * rb_search_page_cache(struct inode * inode,
unsigned long offset)
{
struct rb_node * n = inode->i_rb_page_cache.rb_node;
struct page * page;
while (n)
{
page = rb_entry(n, struct page, rb_page_cache);
if (offset < page->offset)
n = n->rb_left;
else if (offset > page->offset)
n = n->rb_right;
else
return page;
}
return NULL;
}
static inline struct page * __rb_insert_page_cache(struct inode * inode,
unsigned long offset,
struct rb_node * node)
{
struct rb_node ** p = &inode->i_rb_page_cache.rb_node;
struct rb_node * parent = NULL;
struct page * page;
while (*p)
{
parent = *p;
page = rb_entry(parent, struct page, rb_page_cache);
if (offset < page->offset)
p = &(*p)->rb_left;
else if (offset > page->offset)
p = &(*p)->rb_right;
else
return page;
}
rb_link_node(node, parent, p);
return NULL;
}
static inline struct page * rb_insert_page_cache(struct inode * inode,
unsigned long offset,
struct rb_node * node)
{
struct page * ret;
if ((ret = __rb_insert_page_cache(inode, offset, node)))
goto out;
rb_insert_color(node, &inode->i_rb_page_cache);
out:
return ret;
}
-----------------------------------------------------------------------
*/
#ifndef _LINUX_RBTREE_H
#define _LINUX_RBTREE_H
#include "kernel.h"
struct rb_node
{
unsigned long rb_parent_color;
#define RB_RED 0
#define RB_BLACK 1
struct rb_node *rb_right;