Commit 59bef848 authored by Joao Eriberto Mota Filho's avatar Joao Eriberto Mota Filho

Imported Debian patch 1.5.1-1

parents 4538a0c5 b9cadbaa
jdupes 1.5.1
- Significant reduction in memory usage (with a bonus tiny speed boost)
- Improvements in string_malloc memory allocator code
- Bug fixes for output formatting inconsistencies
- Major BTRFS dedupe compilation and functionality fixes
- LOW_MEMORY compile option added for more size/speed tradeoff control
jdupes 1.5
- Invert -Z option: only "soft abort" if asked explicitly to do so
......
......@@ -20,16 +20,22 @@ make CFLAGS_EXTRA='-DYOUR_OPTION_ONE -DYOUR_OPTION_TWO'
This is a list of options that can be "turned on" this way:
ENABLE_BTRFS Enable '-B/--dedupe' for btrfs deduplication
HAVE_BTRFS_IOCTL_H Same as ENABLE_BTRFS
DEBUG * Turn on algorithm statistic reporting with '-D'
OMIT_GETOPT_LONG Do not use getopt_long() C library call
ON_WINDOWS Modify code to compile with MinGW on Windows
USE_TREE_REBALANCE * Use experimental tree rebalancing code
CONSIDER_IMBALANCE * Change tree rebalance to analyze weights first
LOW_MEMORY Build for lower memory usage instead of speed
* These options may slow down the program somewhat and are off by
default. Do not enable them unless you are experimenting.
The LOW_MEMORY option tweaks various knobs in the program to lower
total memory usage. It also disables some features to reduce the size
of certain data structures. The improvements in memory usage are not
very large, but if you're running in a very RAM-limited environment
or have a CPU with very small caches it may be the best choice.
You can turn on the -@ option for "loud" debugging with:
make LOUD=1
......
......@@ -10,10 +10,15 @@ PREFIX = /usr
# Certain platforms do not support long options (command line options).
# To disable long options, uncomment the following line.
#CFLAGS_CONFIG += -DOMIT_GETOPT_LONG
#CFLAGS += -DOMIT_GETOPT_LONG
# Uncomment if you have btrfs/ioctl.h. Needed for --dedupe.
#HAVE_BTRFS_IOCTL_H = -DHAVE_BTRFS_IOCTL_H
# Uncomment for Linux with BTRFS support. Needed for -B/--dedupe.
# This can also be enabled at build time: 'make ENABLE_BTRFS=1'
#CFLAGS += -DENABLE_BTRFS
# Uncomment for low memory usage at the expense of speed and features
# This can be enabled at build time: 'make LOW_MEMORY=1'
#LOW_MEMORY=1
#####################################################################
# Developer Configuration Section #
......@@ -45,6 +50,10 @@ CC ?= gcc
COMPILER_OPTIONS = -Wall -Wextra -Wwrite-strings -Wcast-align -Wstrict-aliasing -pedantic -Wstrict-overflow
COMPILER_OPTIONS += -std=gnu99 -O2 -g -D_FILE_OFFSET_BITS=64 -fstrict-aliasing -pipe
#####################################################################
# no need to modify anything beyond this point #
#####################################################################
# Debugging code inclusion
ifdef LOUD
DEBUG=1
......@@ -58,9 +67,24 @@ endif
ifeq ($(OS), Windows_NT)
COMPILER_OPTIONS += -D__USE_MINGW_ANSI_STDIO=1 -municode
OBJECT_FILES += win_stat.o
override undefine ENABLE_BTRFS
override undefine HAVE_BTRFS_IOCTL_H
endif
CFLAGS += $(COMPILER_OPTIONS) -I. $(CFLAGS_CONFIG) $(CFLAGS_EXTRA) $(HAVE_BTRFS_IOCTL_H)
# Remap old BTRFS support option to new name
ifdef HAVE_BTRFS_IOCTL_H
ENABLE_BTRFS=1
endif
# New BTRFS support option
ifdef ENABLE_BTRFS
COMPILER_OPTIONS += -DENABLE_BTRFS
endif
# Low memory mode
ifdef LOW_MEMORY
COMPILER_OPTIONS += -DLOW_MEMORY -DJODY_HASH_WIDTH=32 -DSMA_PAGE_SIZE=32768
endif
CFLAGS += $(COMPILER_OPTIONS) $(CFLAGS_EXTRA)
INSTALL_PROGRAM = $(INSTALL) -c -m 0755
INSTALL_DATA = $(INSTALL) -c -m 0644
......@@ -71,10 +95,6 @@ INSTALL_DATA = $(INSTALL) -c -m 0644
OBJECT_FILES += jdupes.o jody_hash.o string_malloc.o $(ADDITIONAL_OBJECTS)
#####################################################################
# no need to modify anything beyond this point #
#####################################################################
all: jdupes
jdupes: $(OBJECT_FILES)
......
......@@ -159,14 +159,14 @@ file. If more than 1023 links are created for a file, an error results."
Contact Information
--------------------------------------------------------------------------
For all jdupes inquiries, contact Jody Bruchon <jody@jodybruchon.com>
Please do NOT contact Adrian Lopez about issues with jdupes!
Please DO NOT contact Adrian Lopez about issues with jdupes.
Legal Information
--------------------------------------------------------------------------
jdupes is Copyright (C) 2015-2016 by Jody Bruchon
jdupes is Copyright (C) 2015-2016 by Jody Bruchon <jody@jodybruchon.com>
Derived from the original 'fdupes' (C) 1999-2016 by Adrian Lopez
Includes jody_hash (C) 2015-2016 Jody Bruchon <jody@jodybruchon.com>
Includes jody_hash and string_malloc, (C) 2015-2016 Jody Bruchon
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation files
......
jdupes (1.5.1-1) experimental; urgency=medium
* New upstream release.
* debian/watch: removed the opts line, no longer needed.
-- Joao Eriberto Mota Filho <eriberto@debian.org> Tue, 01 Nov 2016 15:19:36 -0200
jdupes (1.5+git20161027.bc13175-1) experimental; urgency=medium
* Initial release (Closes: #835554)
......
version=4
opts=dversionmangle=s/\+git20161027\.bc13175// \
https://github.com/jbruchon/jdupes/releases .*/archive/v?(\d\S+)\.tar\.(?:bz2|gz|xz)
This diff is collapsed.
......@@ -8,52 +8,95 @@
#include <stdlib.h>
#include <stdint.h>
#include "string_malloc.h"
/* Must be divisible by uintptr_t */
/* Size of pages to allocate at once. Must be divisible by uintptr_t.
* The maximum object size is this page size minus about 16 bytes! */
#ifndef SMA_PAGE_SIZE
#define SMA_PAGE_SIZE 262144
#endif
/* Max freed pointers to remember. Increasing this number allows storing
* more free objects but can slow down allocations. Don't increase it if
* the program's total reused freed alloc counter doesn't increase as a
* result or you're slowing allocs down to no benefit. */
#ifndef SMA_MAX_FREE
#define SMA_MAX_FREE 32
#endif
/* Minimum free object size to consider adding to free list */
#ifndef SMA_MIN_SLACK
#define SMA_MIN_SLACK 64
#endif
static void *sma_head = NULL;
static uintptr_t *sma_lastpage = NULL;
static unsigned int sma_pages = 0;
static size_t sma_lastfree = 0;
static void *sma_freelist[SMA_MAX_FREE];
static int sma_freelist_cnt = 0;
static size_t sma_nextfree = sizeof(uintptr_t);
#ifdef DEBUG
uintmax_t sma_allocs = 0;
uintmax_t sma_free_ignored = 0;
uintmax_t sma_free_good = 0;
uintmax_t sma_free_reclaimed = 0;
uintmax_t sma_free_scanned = 0;
uintmax_t sma_free_tails = 0;
#define DBG(a) a
#else
#define DBG(a)
#endif
/* This function is for debugging the string table only! */
#if 0
static void dump_string_table(void)
/* Scan the freed chunk list for a suitably sized object */
static inline void *scan_freelist(const size_t size)
{
char *p = sma_head;
unsigned int i = sizeof(uintptr_t);
int pg = sma_pages;
while (pg > 0) {
while (i < SMA_PAGE_SIZE && *(p+i) == '\0') i++;
printf("[%16p] (%jd) '%s'\n", p+i, strlen(p+i), p+i);
i += strlen(p+i);
if (pg <= 1 && i >= sma_nextfree) return;
if (i < SMA_PAGE_SIZE) i++;
else {
p = (char *)*(uintptr_t *)p;
pg--;
i = sizeof(uintptr_t);
char *min_p, *object;
size_t sz, min = 0;
int i, used = 0;
/* Don't bother scanning if the list is empty */
if (sma_freelist_cnt == 0) return NULL;
for (i = 0; i < SMA_MAX_FREE; i++) {
/* Stop scanning once we run out of valid entries */
if (used == sma_freelist_cnt) return NULL;
DBG(sma_free_scanned++;)
object = sma_freelist[i];
/* Skip empty entries */
if (object == NULL) continue;
sz = *(size_t *)object;
used++;
/* Skip smaller objects */
if (sz < size) continue;
/* Object is big enough; record if it's the new minimum */
if (min == 0 || sz < min) {
min = sz;
min_p = object;
/* Always stop scanning if exact sized object found */
if (sz == size) break;
}
if (p == NULL) return;
}
return;
/* Enhancement TODO: split the free item if it's big enough */
/* Return smallest object found and delete from free list */
if (min != 0) {
sma_freelist[i] = NULL;
sma_freelist_cnt--;
min_p += sizeof(size_t);
return min_p;
}
/* Fall through - free list search failed */
return NULL;
}
#endif
/* malloc() a new page for string_malloc to use */
static inline void *string_malloc_page(void)
{
uintptr_t * restrict pageptr;
......@@ -92,19 +135,51 @@ void *string_malloc(size_t len)
/* Make room for size prefix */
len += sizeof(size_t);
/* Refuse to allocate a space larger than we can store */
if (len > (unsigned int)(SMA_PAGE_SIZE - sizeof(uintptr_t))) return NULL;
/* Pass-through allocations larger than maximum object size to malloc() */
if (len > (SMA_PAGE_SIZE - sizeof(uintptr_t) - sizeof(size_t))) {
/* Allocate the space */
address = (char *)malloc(len + sizeof(size_t));
if (!address) return NULL;
/* Prefix object with its size */
*(size_t *)address = (size_t)len;
address += sizeof(size_t);
DBG(sma_allocs++;)
return (void *)address;
}
/* Initialize on first use */
if (sma_pages == 0) {
/* Initialize the freed object list */
for (int i = 0; i < SMA_MAX_FREE; i++) sma_freelist[i] = NULL;
/* Allocate first page and set up for first allocation */
sma_head = string_malloc_page();
if (!sma_head) return NULL;
sma_nextfree = (2 * sizeof(uintptr_t));
page = sma_head;
}
/* Allocate new pages when objects don't fit anymore */
/* Allocate objects from the free list first */
address = (char *)scan_freelist(len);
if (address != NULL) {
DBG(sma_free_reclaimed++;)
return address;
}
/* Allocate new page if this object won't fit */
if ((sma_nextfree + len) > SMA_PAGE_SIZE) {
size_t sz;
char *tailaddr;
/* See if remaining space is usable */
if (sma_freelist_cnt < SMA_MAX_FREE && sma_nextfree < SMA_PAGE_SIZE) {
/* Get total remaining space size */
sz = SMA_PAGE_SIZE - sma_nextfree;
if (sz >= (SMA_MIN_SLACK + sizeof(size_t))) {
tailaddr = (char *)page + sma_nextfree;
*(size_t *)tailaddr = (size_t)sz;
string_free(tailaddr + sizeof(size_t));
DBG(sma_free_tails++;)
}
}
page = string_malloc_page();
if (!page) return NULL;
sma_nextfree = (2 * sizeof(uintptr_t));
......@@ -115,53 +190,48 @@ void *string_malloc(size_t len)
/* Prefix object with its size */
*(size_t *)address = (size_t)len;
address += sizeof(size_t);
sma_lastfree = sma_nextfree;
sma_nextfree += len;
#ifdef DEBUG
sma_allocs++;
#endif
return address;
DBG(sma_allocs++;)
return (void *)address;
}
/* Roll back the last allocation */
void string_free(const void * restrict addr)
/* Free an object, adding to free list if possible */
void string_free(void * const restrict addr)
{
static const char * restrict p;
/* Do nothing on NULL address or no last length */
if ((addr == NULL) || (sma_lastfree < sizeof(uintptr_t))) {
#ifdef DEBUG
sma_free_ignored++;
#endif
return;
}
p = (char *)sma_lastpage + sma_lastfree;
/* Only take action on the last pointer in the page */
addr = (void *)((uintptr_t)addr - sizeof(size_t));
if ((uintptr_t)addr != (uintptr_t)p) {
#ifdef DEBUG
sma_free_ignored++;
#endif
return;
int i = 0;
/* Do nothing on NULL address or full free list */
if ((addr == NULL) || sma_freelist_cnt == SMA_MAX_FREE)
goto sf_failed;
/* Tiny objects keep big ones from being freed; ignore them */
if (*(size_t *)((char *)addr - sizeof(size_t)) < (SMA_MIN_SLACK + sizeof(size_t)))
goto sf_failed;
/* Add object to free list */
while (i < SMA_MAX_FREE) {
if (sma_freelist[i] == NULL) {
sma_freelist[i] = (char *)addr - sizeof(size_t);
sma_freelist_cnt++;
DBG(sma_free_good++;)
return;
}
i++;
}
sma_nextfree = sma_lastfree;
sma_lastfree = 0;
#ifdef DEBUG
sma_free_good++;
#endif
/* Fall through */
sf_failed:
DBG(sma_free_ignored++;)
return;
}
/* Destroy all allocated pages */
void string_malloc_destroy(void)
{
static void *cur;
static uintptr_t *next;
void *cur;
uintptr_t *next;
cur = (void *)sma_head;
while (sma_pages > 0) {
......
......@@ -17,10 +17,13 @@ extern "C" {
extern uintmax_t sma_allocs;
extern uintmax_t sma_free_ignored;
extern uintmax_t sma_free_good;
extern uintmax_t sma_free_scanned;
extern uintmax_t sma_free_reclaimed;
extern uintmax_t sma_free_tails;
#endif
extern void *string_malloc(size_t len);
extern void string_free(const void * restrict addr);
extern void string_free(void * const restrict addr);
extern void string_malloc_destroy(void);
#ifdef __cplusplus
......
......@@ -3,7 +3,7 @@
#ifndef JDUPES_VERSION_H
#define JDUPES_VERSION_H
#define VER "1.5"
#define VERDATE "2016-09-26"
#define VER "1.5.1"
#define VERDATE "2016-11-01"
#endif /* JDUPES_VERSION_H */
......@@ -13,7 +13,7 @@
#include <windows.h>
/* Convert NT epoch to UNIX epoch */
static time_t nttime_to_unixtime(uint64_t *timestamp)
static time_t nttime_to_unixtime(const uint64_t * const restrict timestamp)
{
uint64_t newstamp;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment