Skip to content
Commits on Source (13)
.git/
.idea/
bin/
build/
test/
cmake-build-debug/
cmake-build-release/
# Object files
*.o
*.ko
*.obj
*.elf
# Libraries
*.lib
*.a
# Shared objects (inc. Windows DLLs)
*.dll
*.so
*.so.*
*.dylib
# Executables
*.exe
*.out
*.app
*.i*86
*.x86_64
*.hex
# Build directory
/build
.*\.o
test_big
bin
lib
include
# Ubuntu 14.04 Trusty support
sudo: required
dist: trusty
addons:
apt:
packages:
- cmake
before_install:
- git submodule update --init --recursive
script:
- mkdir build
- cd build
- cmake .. && make
# - CTEST_OUTPUT_ON_FAILURE=TRUE make test
matrix:
include:
- compiler: gcc
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-4.9
- mpi-default-dev
env: CXX=g++-4.9
- compiler: gcc
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-5
- mpi-default-dev
env: CXX=g++-5
- compiler: gcc
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-6
- mpi-default-dev
env: CXX=g++-6
- compiler: clang
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.6
packages:
- clang-3.6
- mpi-default-dev
env: CXX=clang++-3.6
- compiler: clang
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.7
packages:
- clang-3.7
- mpi-default-dev
env: CXX=clang++-3.7
- compiler: clang
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-precise-3.8
packages:
- clang-3.8
- mpi-default-dev
env: CXX=clang++-3.8
cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
SET(HAVE_MPI 1 CACHE BOOL "Have MPI")
project(ffindex C)
include_directories(src)
include_directories(src/ext)
add_subdirectory(src)
FROM alpine:latest as ffindex-builder
RUN apk add --no-cache gcc cmake musl-dev ninja
WORKDIR /opt/ffindex
ADD . .
WORKDIR build
RUN cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..
RUN ninja && ninja install
FROM alpine:latest
MAINTAINER Milot Mirdita <milot@mirdita.de>
RUN apk add --no-cache gawk bash grep libstdc++
COPY --from=ffindex-builder /opt/ffindex/build/bin /usr/local/bin/
AUTHOR
FFindex was designed and implemented by Andreas Hauser <andy@splashground.de>.
FFindex was designed and implemented by Andreas Hauser <hauser@genzentrum.lmu.de>.
Please add your name here if you distribute modified versions.
This file must be distributed with any source or binary distribution including
parts of this work, excempt ffutil.h and ffutil.c which are also available
under MIT license.
parts of this work.
SHORT SUMMARY OF THE LICENSE
FFindex is provided under the Create Commons license "Attribution-ShareAlike
Ffindex is provided under the Create Commons license "Attribution-ShareAlike
3.0", which basically captures the spirit of the Gnu Public License (GPL).
See:
......
all:
$(MAKE) -C src $@
%:
$(MAKE) -C src $@
release:
git archive HEAD --prefix "ffindex-`cat VERSION`/" -o "ffindex-`cat VERSION`.tar.gz"
relnotes:
git log --pretty=format:" - %s%n%b"
* Copyright
# FFindex - A database wrapped around mmap
FFindex is a very simple index/database for huge amounts of small files. The
files are stored concatenated in one big data file, seperated by '\0'. A second
file contains a plain text index, giving name, offset and length of of the
small files. The lookup is currently done with a binary search on an array made
from the index file. The attatched binaries (see Usage below) and their source
code shall give an impression of how to use the functions supported by the library in C/C++ code.
## Copyright
FFindex was written by Andreas Hauser <hauser@genzentrum.lmu.de>.
FFindex was written by Andreas Hauser <andy@splpashground.de>.
Please add your name here if you distribute modified versions.
* Martin Steinegger <martin.steinegger@mpibpc.mpg.de>
* Markus Meier <markus.meier@mpibpc.mpg.de>
FFindex is provided under the Create Commons license "Attribution-ShareAlike 3.0",
which basically captures the spirit of the Gnu Public License (GPL).
......@@ -9,85 +21,86 @@ which basically captures the spirit of the Gnu Public License (GPL).
See:
http://creativecommons.org/licenses/by-sa/3.0/
* Thanks
[ ![Codeship Status for soedinglab/ffindex](https://codeship.com/projects/6c26cd00-2247-0133-1269-52bb0fef976f/status?branch=master)](https://codeship.com/projects/96084)
[ ![Build Status](https://travis-ci.org/soedinglab/hh-suite.svg?branch=master)](https://travis-ci.org/soedinglab/ffindex_soedinglab)
## Thanks
Thanks to Laszlo Kajan for creating and maintaining Debian packages
and many suggestions to improve the build and user experience.
* Overview
FFindex is a very simple index/database for huge amounts of small files. The
files are stored concatenated in one big data file, seperated by '\0'. A second
file contains a plain text index, giving name, offset and length of of the
small files. The lookup is currently done with a binary search on an array made
from the index file.
* Installation
$ cd src
$ make
$ make test
## Installation
If you have MPI and want to compile ffindex_apply_mpi:
$ make HAVE_MPI=1
### Compilation
With the sourcecode ready, simply run cmake with the default settings and libraries should be auto-detected:
On OS X use for the first make line:
$ make -f Makefile.osx
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -G "Unix Makefiles" -DCMAKE_INSTALL_PREFIX=${INSTALL_BASE_DIR} ..
make
make install
# Please use a sensible value for INSTALL_DIR, e.g. /usr/local or /opt/ffindex
# or $HOME/ffindex instead of "..".
$ make install INSTALL_DIR=..
and with MPI:
**Please use a sensible value for ${INSTALL_BASE_DIR}, e.g. /usr/local or /opt/ffindex or $HOME/ffindex**
$ make install INSTALL_DIR=.. HAVE_MPI=1
* Usage
### Setting environment variables
Please note that before querying or unlinking entries a ffindex must be
sorted, although you can add to it without. So either specify -s with
ffindex_build or sorted later with ffindex_modify -s.
Also the length of the entry names is restricted. See the usage output of
the ffindex_build program.
Setup environment:
$ export PATH="$INSTALL_DIR/bin:$PATH"
$ export LD_LIBRARY_PATH="$INSTALL_DIR/lib:$LD_LIBRARY_PATH"
export PATH="${INSTALL_BASE_DIR}/bin:${PATH}"
export LD_LIBRARY_PATH="${INSTALL_BASE_DIR}/lib:${LD_LIBRARY_PATH}"
On OS X set DYLD_LIBRARY_PATH instead of LD_LIBRARY_PATH.
## Usage
Build index from files in test/data and test/data2.
$ ffindex_build -s /tmp/test.data /tmp/test.ffindex test/data test/data2
ffindex_build -s /tmp/test.data /tmp/test.ffindex test/data test/data2
Retrieve three entries:
$ ffindex_get /tmp/test.data /tmp/test.ffindex a b foo
ffindex_get /tmp/test.data /tmp/test.ffindex a b foo
Unlink (Remove reference from index) an entry:
$ ffindex_modify -u /tmp/test.ffindex b
ffindex_modify -u /tmp/test.ffindex b
Retrieve three entries, "b" should now be missing:
$ ffindex_get /tmp/test.data /tmp/test.ffindex a b foo
ffindex_get /tmp/test.data /tmp/test.ffindex a b foo
Convert a Fasta file to ffindex, entry names are incerental IDs starting from 1:
$ ffindex_from_fasta -s fasta.ffdata fasta.ffindex NC_007779.ffn
ffindex_from_fasta -s fasta.ffdata fasta.ffindex NC_007779.ffn
Get first entry by name:
$ ffindex_get fasta.ffdata fasta.ffindex 1
ffindex_get fasta.ffdata fasta.ffindex 1
Get first and third entry by entry index, this a little faster:
$ ffindex_get fasta.ffdata fasta.ffindex -n 1 3
ffindex_get fasta.ffdata fasta.ffindex -n 1 3
Count the characters including header in each entry:
$ ffindex_apply fasta.ffdata fasta.ffindex wc -c
ffindex_apply fasta.ffdata fasta.ffindex wc -c
Count the number of characters in each sequence, without the header:
$ ffindex_apply fasta.ffdata fasta.ffindex perl -ne '$x += length unless(/^>/); END{print "$x\n"}'
ffindex_apply fasta.ffdata fasta.ffindex perl -ne '$x += length unless(/^>/); END{print "$x\n"}'
Parallel version for counting the characters including header in each entry:
$ mpirun -np 4 ffindex_apply_mpi fasta.ffdata fasta.ffindex -- wc -c
mpirun -np 4 ffindex_apply_mpi fasta.ffdata fasta.ffindex -- wc -c
Parallel version for counting the characters including header in each entry and
saving the output to a new ffindex:
$ mpirun -np 4 ffindex_apply_mpi fasta.ffdata fasta.ffindex -i out-wc.ffindex -d out-wc.ffdata -- wc -c
mpirun -np 4 ffindex_apply_mpi fasta.ffdata fasta.ffindex -i out-wc.ffindex -o out-wc.ffdata -- wc -c
0.9.9.6
- Fix OS X builds and integrate Makefile.osx
- Drop OpenMP from compiler flags
- Add ffindex_posix_search.o to make clean
- Avoid non-standard nested function, even so it's the cleanest solution,
since the posix twalk interface is broken. Factor out functions using posix
tree. Use TLS variables to stay threadsafe.
- Fix ffnchomp check
0.9.9.5
- Make use of our parse_long for higher startup performance.
- Add RELEASE_NOTES
- Roll our own parse_ulong instead strtol for speed reasons.
- Fix usage formatting
- Make more clear that tree is already sorted ffindex_modify.c
- Remove unused code in ffindex_modify.c
0.9.9.4 and before
- ffindex_apply is now ffindex_apply_mpi without MPI
- Add src/ffindex_apply_mpi_spawn.c, using spawn istead of fork & exec
- Fix example
- Add collect_data
- Add Markus Meier
- perl5/FFindex.pm: Add copyright. Fix entry reading. Add get_entry* methods.
- Add perl5/test.pl
- Add perl5
- Add each_data
- Return Qnil if not found
- Add method_ffindex_get_data_by_name
- Add first ruby wrapper.
- Add ffindex_db
- Add nginx server module
- Fix memory leak on error
- ffindex_get: Add -f FILE argument to let the keys to retrieve be specified by a file.
- Add ffindex_from_tsv to Makefile
- Add ffindex_from_tsv
- Update RPM spec
- bump version
- Update copyrigths and version.
- Set higher limits. Update version.
- Handle the case where filenames are actually longer than FFINDEX_MAX_ENTRY_NAME_LENTH. This was also security vulnerability.
- fix fprintf
- Add log_file option to ffindex_apply_mpi and rework error handling.
- fix getopt
- merge
- Add ffindex_optimize
- bumb version
- Add -q for quiet
- merge
- Better usage info.
- Make sort work independant from unlink.
- Add ffindex_insert_ffindex().
- Make use of ffindex_index_open in ffindex_from_fasta
- Add ffindex_index_open() and use it in ffindex_build.
- Add -D_GNU_SOURCE=1, probably better here than in source.
- Bump version
- Export FFINDEX_ENTRY_NAME and unlink() temporary ffindexes in ffindex_apply_mpi.
- Print entry and return status of ffindex_apply_mpi to stderr. This allows for further processing based on return status.
- Add MPI to README
- Don't assume string data, make binary data work too.
- Add get_entry_by_name as alias for bsearch
- Fix build for some compilers
- Ma
- Another fix for noncapture and cleanup
- Fix no capture mode
- First beta of apply_mpi output catching to another ffindex
- wip
- wip
- apply_mpi now works with capturing output to another ffindex, but I/O can block.
- Better error message
- More clever root Makefile contributed by Laszlo.
- Add thanks
- Reduce FFINDEX_MAX_ENTRY_NAME_LENTH from 56 to 32 so that by default the malloced space for the index is <2GB. Suggested by Laszlo.
- Add message to indicated number of failed bytes.
- Bump version to 0.980
- Add ffindex_unpack to install and spec and bump version to 0.9.8
- More improvements to usage info for ffindex_build.
- Add better usage info to ffindex_build, proposed by Johannes.
- remove modifies -t from usage
- merge
- merge
- wip ffindex_apply_mpi.c
- Add ffindex_unpack to unpack all entries/files from an ffindex
- Bump version to 0.97
- Check index out of bounds for ffindex_get_entry_by_index
- Add some more usage examples to README
- Add option -n to ffindex_get to use entry index instead of entry name
- Add ifndef wrapper as suggested by Markus.
- Add return check for 1 byte fwrite.
- Don't access index before NULL check. Reporter: Laszlo Karjan.
- Bump version
- Diy OSX build
- Bump version to 0.9.6
- Fix bug data vs. filedata.
- Use optimized interface
- Fix comment
- Conditionalize MPI
- Bump version to 0.9.5.
- Refactor to handle entry lookup outside of ffindex_apply.
- Handling of left over.
- Add first implementation of ffindex_apply_mpi.c.
- Handle EPIPE.
- Add a first version of ffindex_apply, that executes a program for each entry in the ffindex.
- Reflect ffindex_fopen rename.
- Rename ffindex_fopen to ffindex_fopen_by_name. Add ffindex_fopen_by_entry and ffindex_get_data_by_entry.
- Add hgignore
- Clean also .o
- Add it also to osx
- Add ffindex_from_fasta to clean
- Install ffindex_from_fasta
- Improve error handling, especially with empty Index file.
- Add ffindex_get_data_by_index. And rename other ffindex_get functions to be more uniform.
- Use ffindex_get() instead of ffindex_fopen() in ffindex_get program.
- Add ffindex_get to get entry as string.
- Raise FFINDEX_MAX_INDEX_ENTRIES_DEFAULT from 6 million to 40 million
- Make sure ">" is after newline. Better code.
- Add -O2.
- Complete implementation of ffindex_from_fasta.
- Bump version to 0.94
- Sart implementation of ffindex_from_fasta
- Add ffindex_insert_memory().
- Improve README
- Bump version to 0.9.3
- merge
- Better make chdir
- Make usage of tree default in ffindex_modify.
- Make OSX compile again.
- basename() is in libgen.h on BSD.
- Clean ffutil.o too.
- Add deinstall target
- Fix spec libdir
- Fix libdir creation
- bump version
- Fix spec
- Respect lib64
- nicer
- Add a SPEC file for building an RPM package.
- Rework build a little. Build a static library libffindex.a too.
- Add top-level Makefile.
- Fix remaining printf.
- Fix size_t printfs. Use length modifier z.
- Bump version.
- Move textfiles to top.
- Add inclusion clause to LICENSE.
- Add advertisement.
- Fix fclose with ffindex_add_filestream.
- ffindex_build now can incorporate (merge) other ffindexes.
- Add -t to ffindex_modify, which tells ffindex to speedup unlinking using a tree.
- unlink: walk to lists implmentation
- Fix message
- Fix unlink.
- Fix OS X build
- Fix test
- Remove more strnlen. Introduce ffnchomp in ffutil that was renamed from fferror.
- Use strlen instead of strnlen. Sae case and BSD/OS X do not always have strnlen.
- fix includes
- little things
- better documentation
- Add possibility to add single files not only directories. TODO: Decide what should be the name in the index, that is should the whole path be part of the entry name.
- Make INSTALL_DIR usable.
- Add
- Add ffindex_modify that oprates on an existing index. E.g. for unlinking.
- Guard against overwriting existing index or data file
- Add ffindex_unlink
- better tests
- Better error handling
- comment
- Better error handling
- nicer error handling
- nicer error handling
- spell
- Better error handling and messages, especially if malloc failes.
- ffindex_index_parse now takes a second argument specifying the maximum number of entries. A default is used if 0 is specified.
- Add -v for version. Add -f for list_files.
- Small code improvement.
- Add fmemopen for Mac OS X.
- fix warnings
- Add install target.
- - Add license. - Add append and sort.
- Faster parsing. ffindex_mmap interface changed.
- Convert index to struct. Implement bsearch.
- Rest of data type conversion. Ffindex_build now takes offset return value, which is needed e.g. for multiple directory indexing.
- Change *data to from type void* to char* to avoid problems with pointer arithmetics.
- Add test for multiple directory indexing
- Add second test dir.
- Make Makefile more robust.
- Add more functions to header.
- ffindex_get now support multiple entry retrieves.
- Remoove debug extra newlines from ffindex_get output.
- Remove debug print.
- Add clean.
- Add a program to retrieve data from index.
- Fix parameter mixup. Add length to index.
- Add test
- ffindex_fopen
- wip
- primitive ffindex_get_offset
- use read
- init
ffindex for Debian
==================
The official development of ffindex is discontinued. However, a fork
is maintained by
https://github.com/soedinglab/ffindex_soedinglab
which is used in hhsuite. Since the main focus of the Debian packaging
is hhsuite[1] this fork is replacing the original code in the packaging.
[1] https://lists.debian.org/debian-med/2019/01/msg00013.html
-- Andreas Tille <tille@debian.org> Wed, 09 Jan 2019 11:37:23 +0100
ffindex (0.9.9.7.soedinglab+git20171201.74550c8-1) unstable; urgency=medium
* Revert latest change and base packaging again on soedinglab
(see README.source)
* Standards-Version: 4.3.0
* Respect DEB_BUILD_OPTIONS in override_dh_auto_test target
-- Andreas Tille <tille@debian.org> Wed, 09 Jan 2019 16:45:16 +0100
ffindex (0.9.9.7+sog+git20160415.14274c9-1) unstable; urgency=medium
* The previous location on Github was an improperly chosen fork
......
......@@ -8,11 +8,12 @@ Build-Depends: debhelper (>= 11~),
help2man,
libopenmpi-dev,
openmpi-bin,
cmake (>= 3.11),
chrpath
Standards-Version: 4.2.1
Standards-Version: 4.3.0
Vcs-Browser: https://salsa.debian.org/med-team/ffindex
Vcs-Git: https://salsa.debian.org/med-team/ffindex.git
Homepage: http://www.splashground.de/~andy/programs/FFindex/
Homepage: https://github.com/soedinglab/ffindex_soedinglab
Package: ffindex
Architecture: any
......
usr/include/*
usr/lib/libffindex.a
usr/lib/libffindex.so
src/ffindex_posix_search.h usr/include
Author: Steffen Moeller
Last-Update: 2018-08-02 19:14:12 +0200
Description: Fix building of shared lib
Index: ffindex/src/CMakeLists.txt
===================================================================
--- ffindex.orig/src/CMakeLists.txt
+++ ffindex/src/CMakeLists.txt
@@ -1,17 +1,32 @@
+cmake_minimum_required(VERSION 3.11)
set(CMAKE_C_FLAGS "-std=c99 ${CMAKE_C_FLAGS}")
# sets HAVE_FMEMOPEN
add_subdirectory(ext)
-add_library (ffindex ffindex.c ffutil.c)
+# Adapted from https://stackoverflow.com/questions/2152077/is-it-possible-to-get-cmake-to-build-both-a-static-and-shared-version-of-the-sam
-target_include_directories (ffindex PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+# this is the "object library" target: compiles the sources only once
+add_library(objlib OBJECT ffindex.c ffutil.c)
+target_include_directories (objlib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+
+# shared libraries need PIC
+set_property(TARGET objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
+
+# shared and static libraries built from the same object files
+add_library(ffindex-static STATIC $<TARGET_OBJECTS:objlib>)
+add_library(ffindex-shared SHARED $<TARGET_OBJECTS:objlib>)
+
+set_target_properties(ffindex-static PROPERTIES OUTPUT_NAME ffindex)
+
+set_target_properties(ffindex-shared PROPERTIES VERSION 0.9.9.7
+ SOVERSION 0
+ OUTPUT_NAME ffindex)
-add_library (ffindex_shared SHARED ffindex.c ffutil.c)
if(NOT HAVE_FMEMOPEN)
- target_link_libraries(ffindex ext)
- target_link_libraries(ffindex_shared ext)
+# target_link_libraries(ffindex ext)
+ target_link_libraries(ffindex-shared ext)
endif()
@@ -19,7 +34,7 @@ endif()
add_executable(ffindex_apply
ffindex_apply_mpi.c
)
-target_link_libraries (ffindex_apply ffindex)
+target_link_libraries (ffindex_apply ffindex-shared)
set_property(TARGET ffindex_apply PROPERTY COMPILE_FLAGS "-UHAVE_MPI")
find_package(MPI)
@@ -27,7 +42,7 @@ if(MPI_C_FOUND AND HAVE_MPI)
add_executable(ffindex_apply_mpi
ffindex_apply_mpi.c
)
- target_link_libraries (ffindex_apply_mpi ffindex)
+ target_link_libraries (ffindex_apply_mpi ffindex-shared)
add_subdirectory(mpq)
set_property(TARGET ffindex_apply_mpi PROPERTY COMPILE_FLAGS "-DHAVE_MPI=1 ${MPI_C_COMPILE_FLAGS}")
@@ -45,48 +60,48 @@ endif()
add_executable(ffindex_reduce
ffindex_reduce.c
)
-target_link_libraries (ffindex_reduce ffindex)
+target_link_libraries (ffindex_reduce ffindex-shared)
add_executable(ffindex_build
ffindex_build.c
)
-target_link_libraries (ffindex_build ffindex)
+target_link_libraries (ffindex_build ffindex-shared)
add_executable(ffindex_from_fasta
ffindex_from_fasta.c
)
-target_link_libraries (ffindex_from_fasta ffindex)
+target_link_libraries (ffindex_from_fasta ffindex-shared)
add_executable(ffindex_get
ffindex_get.c
)
-target_link_libraries (ffindex_get ffindex)
+target_link_libraries (ffindex_get ffindex-shared)
add_executable(ffindex_modify
ffindex_modify.c
)
-target_link_libraries (ffindex_modify ffindex)
+target_link_libraries (ffindex_modify ffindex-shared)
add_executable(ffindex_unpack
ffindex_unpack.c
)
-target_link_libraries (ffindex_unpack ffindex)
+target_link_libraries (ffindex_unpack ffindex-shared)
add_executable(ffindex_order
ffindex_order.c
)
-target_link_libraries (ffindex_order ffindex)
+target_link_libraries (ffindex_order ffindex-shared)
add_executable(ffindex_from_fasta_with_split
ffindex_from_fasta_with_split.c
)
-target_link_libraries (ffindex_from_fasta_with_split ffindex)
+target_link_libraries (ffindex_from_fasta_with_split ffindex-shared)
install(PROGRAMS
@@ -95,15 +110,15 @@ install(PROGRAMS
DESTINATION include
)
-install(TARGETS ffindex
- LIBRARY DESTINATION lib64 COMPONENT libraries
- ARCHIVE DESTINATION lib64 COMPONENT libraries
+install(TARGETS ffindex-static
+ LIBRARY DESTINATION lib COMPONENT libraries
+ ARCHIVE DESTINATION lib COMPONENT libraries
)
-install(TARGETS ffindex_shared
- LIBRARY DESTINATION lib64 COMPONENT libraries
- ARCHIVE DESTINATION lib64 COMPONENT libraries
+install(TARGETS ffindex-shared
+ LIBRARY DESTINATION lib COMPONENT libraries
+ ARCHIVE DESTINATION lib COMPONENT libraries
)
Description: Makefile fixes
* make Makefile DESTDIR-aware
* other changes to allow help2man to work better.
* deinstall -> uninstall
* pass CPPFLAGS and LDFLAGS for hardening
.
Upstream is aware but not interested in DESTDIR.
Forwarded: yes
Author: Laszlo Kajan <lkajan@debian.org>
Author: Graham Inggs <ginggs@debian.org>
Last-Update: 2017-06-29
--- a/src/Makefile
+++ b/src/Makefile
@@ -1,17 +1,22 @@
-INSTALL_DIR?=..
-libdir=`([ -d /usr/lib64 ] && echo lib64) || echo lib`
+PREFIX?=/usr
+INSTALL_DIR=$(DESTDIR)$(PREFIX)
+libdir=lib
LANG=C
#CC=musl-gcc
#OMPI_CC=musl-gcc
#CC=gcc
#CC=icc
#CC=clang
-CFLAGS+=-std=c99 -O2 -g -fPIC -Wall -pedantic -D_GNU_SOURCE=1 -I.
-LDFLAGS=-lffindex -L.
+CFLAGS+=-std=c99 -O2 -g -fPIC -Wall -pedantic -D_GNU_SOURCE=1 -I. $(CPPFLAGS)
+SOCURRENT=2
+SOREVISION=0
+SOAGE=2
+SOVERSION=$(SOCURRENT).$(SOREVISION).$(SOAGE)
+SOMAJORV=$(shell echo $$(( $(SOCURRENT)-$(SOAGE) )) )
OS:= $(shell uname)
-FFINDEX_LIBS=libffindex.so libffindex.so.0.1 libffindex.a
+FFINDEX_LIBS=libffindex.so libffindex.so.$(SOVERSION) libffindex.a
FFINDEX_HEADERS=ffindex.h ffindex_posix_search.h ffutil.h
FFINDEX_OBJECTS=$(FFINDEX_HEADERS:.h=.o)
@@ -34,30 +39,30 @@ libffindex.a: $(FFINDEX_OBJECTS)
ar rcs libffindex.a $^
ifeq ($(OS), Darwin)
-libffindex.so.0.1: $(FFINDEX_OBJECTS) ext/fmemopen.o
- $(CC) -shared -o libffindex.so.0.1 $^
+libffindex.so.$(SOVERSION): $(FFINDEX_OBJECTS) ext/fmemopen.o
+ $(CC) -shared -o libffindex.so.$(SOVERSION) $^
else
ifdef FFINDEX_STATIC
STATIC=-static
endif
-libffindex.so.0.1: $(FFINDEX_OBJECTS)
- $(CC) -Wl,-z,defs -Wl,-soname,libffindex.so.0.1 -shared -o libffindex.so.0.1 $^
+libffindex.so.$(SOVERSION): $(FFINDEX_OBJECTS)
+ $(CC) $(LDFLAGS) -Wl,-z,defs -Wl,-soname,libffindex.so.$(SOVERSION) -shared -o libffindex.so.$(SOVERSION) $^
endif
-libffindex.so: libffindex.so.0.1
- ln -sf libffindex.so.0.1 libffindex.so
+libffindex.so: libffindex.so.$(SOVERSION)
+ ln -sf libffindex.so.$(SOVERSION) libffindex.so
ffindex_apply.o: ffindex_apply_mpi.c $(FFINDEX_LIBS)
$(CC) $(CFLAGS) -c ffindex_apply_mpi.c -o $@
ffindex_apply: ffindex_apply.o $(FFINDEX_LIBS)
- $(CC) $(CFLAGS) -o $@ ffindex_apply.o $(LDFLAGS) $(STATIC)
+ $(CC) $(CFLAGS) -o $@ ffindex_apply.o $(LDFLAGS) -L. -lffindex
ffindex_apply_mpi: ffindex_apply_mpi.o $(FFINDEX_LIBS)
- mpicc $(CFLAGS) -DHAVE_MPI -o $@ ffindex_apply_mpi.o $(LDFLAGS)
+ mpicc $(CFLAGS) -DHAVE_MPI -o $@ ffindex_apply_mpi.o $(LDFLAGS) -L. -lffindex
ffindex_apply_mpi.o: ffindex_apply_mpi.c $(FFINDEX_HEADERS)
mpicc $(CFLAGS) -DHAVE_MPI -c ffindex_apply_mpi.c -o $@
@@ -69,24 +74,24 @@ ffindex_apply_mpi_spawn.o: ffindex_apply
mpicc $(CFLAGS) -c ffindex_apply_mpi_spawn.c -o $@
ffindex_from_fasta: ffindex_from_fasta.o $(FFINDEX_LIBS)
- $(CC) -o $@ ffindex_from_fasta.o $(LDFLAGS) $(STATIC)
+ $(CC) -o $@ ffindex_from_fasta.o $(LDFLAGS) -L. -lffindex
ffindex_from_tsv: ffindex_from_tsv.o $(FFINDEX_LIBS)
- $(CC) -o $@ ffindex_from_tsv.o $(LDFLAGS) $(STATIC)
+ $(CC) -o $@ ffindex_from_tsv.o $(LDFLAGS) -L. -lffindex
ffindex_get: ffindex_get.o $(FFINDEX_LIBS)
- $(CC) -o $@ ffindex_get.o $(LDFLAGS) $(STATIC)
+ $(CC) -o $@ ffindex_get.o $(LDFLAGS) -L. -lffindex
ffindex_build: ffindex_build.o $(FFINDEX_LIBS)
- $(CC) -o $@ ffindex_build.o $(LDFLAGS) $(STATIC)
+ $(CC) -o $@ ffindex_build.o $(LDFLAGS) -L. -lffindex
ffindex_modify: ffindex_modify.o $(FFINDEX_LIBS)
- $(CC) -o $@ ffindex_modify.o $(LDFLAGS) $(STATIC)
+ $(CC) -o $@ ffindex_modify.o $(LDFLAGS) -L. -lffindex
ffindex_unpack: ffindex_unpack.o $(FFINDEX_LIBS)
- $(CC) -o $@ ffindex_unpack.o $(LDFLAGS) $(STATIC)
+ $(CC) -o $@ ffindex_unpack.o $(LDFLAGS) -L. -lffindex
-test:
+check:
rm -f /tmp/test.data /tmp/test.ffindex
LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" ./ffindex_build -s /tmp/test.data /tmp/test.ffindex ../test/data ../test/data2
LD_LIBRARY_PATH=".:$(LD_LIBRARY_PATH)" ./ffindex_get /tmp/test.data /tmp/test.ffindex a b foo | tee /tmp/test.out
@@ -107,9 +112,10 @@ time:
clean:
rm -f $(FFINDEX_PROGRAMS) $(FFINDEX_OBJECTS) $(FFINDEX_LIBS) \
- libffindex.so libffindex.so.0.1 libffindex.a \
+ libffindex.so libffindex.so.$(SOVERSION) libffindex.a \
ffindex_build.o ffindex_get.o ffindex_modify.o ffindex_from_fasta.o ffindex_apply_mpi.o \
- ffindex_apply.o ffindex_from_tsv.o ffindex_unpack.o ffindex_posix_search.o ffindex_apply_mpi_spawn.o
+ ffindex_apply.o ffindex_from_tsv.o ffindex_unpack.o ffindex_posix_search.o ffindex_apply_mpi_spawn.o \
+ $(OBJECTS) $(OBJECTS:%=%.o)
install:
mkdir -p $(INSTALL_DIR)/bin
@@ -122,9 +128,10 @@ install:
install ffindex_from_tsv $(INSTALL_DIR)/bin/ffindex_from_tsv
install ffindex_apply $(INSTALL_DIR)/bin/ffindex_apply
install ffindex_unpack $(INSTALL_DIR)/bin/ffindex_unpack
- install libffindex.so.0.1 $(INSTALL_DIR)/$(libdir)/libffindex.so.0.1
+ install libffindex.so.$(SOVERSION) $(INSTALL_DIR)/$(libdir)/libffindex.so.$(SOVERSION)
install libffindex.a $(INSTALL_DIR)/$(libdir)/libffindex.a
- ln -sf libffindex.so.0.1 $(INSTALL_DIR)/$(libdir)/libffindex.so
+ ln -sf libffindex.so.$(SOVERSION) $(INSTALL_DIR)/$(libdir)/libffindex.so.$(SOMAJORV)
+ ln -sf libffindex.so.$(SOVERSION) $(INSTALL_DIR)/$(libdir)/libffindex.so
install ffindex.h $(INSTALL_DIR)/include/ffindex.h
install ffutil.h $(INSTALL_DIR)/include/ffutil.h
install ffindex_posix_search.h $(INSTALL_DIR)/include/ffindex_posix_search.h
@@ -134,7 +141,7 @@ endif
deinstall:
rm -f $(INSTALL_DIR)/bin/ffindex_get $(INSTALL_DIR)/bin/ffindex_build $(INSTALL_DIR)/bin/ffindex_modify \
- $(INSTALL_DIR)/$(libdir)/libffindex.so.0.1 $(INSTALL_DIR)/$(libdir)/libffindex.a \
+ $(INSTALL_DIR)/$(libdir)/libffindex.so.$(SOVERSION) $(INSTALL_DIR)/$(libdir)/libffindex.a \
$(INSTALL_DIR)/$(libdir)/libffindex.so $(INSTALL_DIR)/include/ffindex.h $(INSTALL_DIR)/include/ffutil.h\
$(INSTALL_DIR)/include/ffindex_posix_search.h $(INSTALL_DIR)/bin/ffindex_from_fasta $(INSTALL_DIR)/bin/ffindex_from_tsv
rmdir $(INSTALL_DIR)/bin || true
Author: Laszlo Kajan <lkajan@rostlab.org>
Author: Elso Andras <elso.andras@gmail.com>
Subject: 40000000 bug and basename
[Laszlo Kajan]
The default value 40000000 for FFINDEX_MAX_INDEX_ENTRIES_DEFAULT is too high for a machine with 2G (or less) RAM.
Bug fixed by introducing an environment variable FFINDEX_MAX_INDEX_ENTRIES to control the value.
[Elso Andras]
using basename for usage print, this helps help2man if a packager is using it
Forwarded: yes
--- a/src/ffindex_build.c
+++ b/src/ffindex_build.c
@@ -42,6 +42,8 @@ void usage(char *program_name)
"\t\t\tAnother append operations can be done without sorting.\n"
"\t-v\t\tprint version and other info then exit\n"
"\t-h\t\tdisplay this help message\n"
+ "ENVIRONMENT\n"
+ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
"\nEXAMPLES:\n"
"\tCreate a new ffindex containing all files from the \"bar/\" directory containing\n"
"\tsay myfile1.txt, myfile2.txt and sort (-s) it so that e.g. ffindex_get can use it.\n"
@@ -55,7 +57,7 @@ void usage(char *program_name)
"\tdefault maximum entries %d.\n"
"\tThis can be changed in the sources.\n"
FFINDEX_COPYRIGHT,
- program_name, MAX_FILENAME_LIST_FILES, FFINDEX_MAX_ENTRY_NAME_LENTH, FFINDEX_MAX_INDEX_ENTRIES_DEFAULT);
+ basename(program_name), MAX_FILENAME_LIST_FILES, FFINDEX_MAX_ENTRY_NAME_LENTH, FFINDEX_MAX_INDEX_ENTRIES_DEFAULT);
}
int main(int argn, char **argv)
--- a/src/ffindex_from_fasta.c
+++ b/src/ffindex_from_fasta.c
@@ -36,8 +36,10 @@ void usage(char *program_name)
"\t-i\tuse identifier as name (you probably want -s too)\n"
"\t-n\tuse incremental number as id (default)\n"
"\t-v\tprint version\n"
+ "ENVIRONMENT\n"
+ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
FFINDEX_COPYRIGHT,
- program_name);
+ basename(program_name));
}
int main(int argn, char **argv)
@@ -140,4 +142,4 @@ int main(int argn, char **argv)
return err;
}
-/* vim: ts=2 sw=2 et: */
+// vim: ts=2 sw=2 et:
--- a/src/ffindex_unpack.c
+++ b/src/ffindex_unpack.c
@@ -37,7 +37,7 @@ int main(int argn, char **argv)
fprintf(stderr, "USAGE: %s DATA_FILENAME INDEX_FILENAME OUT_DIR\n"
"\n\tPlease note that a huge number of files might be created!\n"
FFINDEX_COPYRIGHT,
- argv[0]);
+ basename(argv[0]));
return -1;
}
char *data_filename = argv[1];
--- a/src/ffindex.h
+++ b/src/ffindex.h
@@ -24,6 +24,8 @@
#define FFINDEX_VERSION 0.997
#define FFINDEX_COPYRIGHT "\nDesigned and implemented by Andreas Hauser <andy@splashground.de>.\n"
+// lkajan: the below makes a default allocation over 2G. malloc in ffindex.c:225 fails on a machine with 'only' 2G of RAM.
+// lkajan: let the default remain 80000000, and let the actual value be controlled by the env var FFINDEX_MAX_INDEX_ENTRIES wherever FFINDEX_MAX_INDEX_ENTRIES_DEFAULT is used.
#define FFINDEX_MAX_INDEX_ENTRIES_DEFAULT 80000000
#define FFINDEX_MAX_ENTRY_NAME_LENTH 63
@@ -76,6 +78,8 @@ int ffindex_index_close(ffindex_index_t*
/* return *out_data_file, *out_index_file, out_offset. */
int ffindex_index_open(char *data_filename, char *index_filename, char* mode, FILE **out_data_file, FILE **out_index_file, size_t *out_offset);
+inline size_t ffindex_max_index_entries(){ char *FMIE = getenv("FFINDEX_MAX_INDEX_ENTRIES"); if( FMIE ) { return atol( FMIE ); } else { return FFINDEX_MAX_INDEX_ENTRIES_DEFAULT; } }
+
int ffindex_insert_memory(FILE *data_file, FILE *index_file, size_t *offset, char *from_start, size_t from_length, char *name);
int ffindex_insert_file(FILE *data_file, FILE *index_file, size_t *offset, const char *path, char *name);
--- a/src/ffindex_apply_mpi.c
+++ b/src/ffindex_apply_mpi.c
@@ -214,8 +214,8 @@ int main(int argn, char **argv)
"\t-d\tFFDATA output file, stdout of program will be captured\n"
"\t-i\tFFINDEX output file, entries get the same name as input entry\n"
FFINDEX_COPYRIGHT,
- argv[0],
- argv[0]);
+ basename(argv[0]),
+ basename(argv[0]));
errno = EINVAL;
goto EXCEPTION;
}
--- a/src/ffindex_get.c
+++ b/src/ffindex_get.c
@@ -29,8 +29,10 @@ void usage(char* program_name)
"\t-f FILE\t\tfile containing a list of file names, one per line\n"
"\t-h\t\tdisplay this help message\n"
"\t-n\t\tuse index of entry instead of entry name\n"
+ "ENVIRONMENT\n"
+ "\tFFINDEX_MAX_INDEX_ENTRIES - allocate memory for this number of entries\n"
FFINDEX_COPYRIGHT,
- program_name);
+ basename(program_name));
}
int main(int argn, char **argv)
--- a/src/ffindex_modify.c
+++ b/src/ffindex_modify.c
@@ -41,7 +41,7 @@ void usage(char *program_name)
"\tTo clean up the data file after unlinking insert it to an new one:\n"
"\tffindex_build -as cleaned.ffdata cleaned.ffindex -d foo.ffdata -i foo.ffindex\n"
FFINDEX_COPYRIGHT,
- program_name, MAX_FILENAME_LIST_FILES, program_name);
+ basename(program_name), MAX_FILENAME_LIST_FILES, basename(program_name));
}
int main(int argn, char **argv)
--- a/src/ffindex.c
+++ b/src/ffindex.c
@@ -373,7 +373,7 @@ ffindex_entry_t* ffindex_bsearch_get_ent
ffindex_index_t* ffindex_index_parse(FILE *index_file, size_t num_max_entries)
{
if(num_max_entries == 0)
- num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT;
+ num_max_entries = ffindex_max_index_entries();
size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries);
ffindex_index_t *index = (ffindex_index_t *)mmap(0, nbytes, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
Description: Debugging code used for #907624
--- a/src/ffindex.c
+++ b/src/ffindex.c
@@ -165,6 +165,8 @@ int ffindex_insert_list_file(FILE *data_
char path[PATH_MAX];
while(fgets(path, PATH_MAX, list_file) != NULL) {
char *name = strdup(basename(path));
+ printf("DEBUG: %s (%i): name = %s\n", __FILE__, __LINE__, name);
+ fflush(stdout);
ffindex_insert_file(data_file, index_file, &offset, ffnchomp(path, strlen(path)), ffnchomp(name, strlen(name)));
free(name);
}
@@ -322,6 +324,8 @@ ffindex_index_t* ffindex_index_parse(FIL
if(num_max_entries == 0)
num_max_entries = FFINDEX_MAX_INDEX_ENTRIES_DEFAULT;
size_t nbytes = sizeof(ffindex_index_t) + (sizeof(ffindex_entry_t) * num_max_entries);
+ printf("DEBUG: %s (%i): ffindex_index_parse(): nbytes = %li\n", __FILE__, __LINE__, nbytes);
+
ffindex_index_t *index = (ffindex_index_t *)malloc(nbytes);
if(index == NULL)
{
@@ -350,9 +354,15 @@ ffindex_index_t* ffindex_index_parse(FIL
for(i = 0; d < (index->index_data + index->index_data_size); i++)
{
int p;
+ printf("DEBUG: %s (%i): ffindex_index_parse(): i=%li, d=%s, index->index_data=%li, index->index_data_size=%li\n", __FILE__, __LINE__, i, d, index->index_data, index->index_data_size);
for(p = 0; *d != '\t'; d++)
+{
+printf("DEBUG: %s (%i): ffindex_index_parse(): p=%i, *d=%c\n", __FILE__, __LINE__, p, *d);
index->entries[i].name[p++] = *d;
+}
index->entries[i].name[p] = '\0';
+ printf("DEBUG: %s (%i): ffindex_index_parse(): index->entries[%i].name = %s\n", __FILE__, __LINE__, i, index->entries[i].name);
+ fflush(stdout);
index->entries[i].offset = strtoull(d, &end, 10);
d = end;
index->entries[i].length = strtoull(d, &end, 10);
@@ -439,11 +449,15 @@ void ffindex_sort_index_file(ffindex_ind
int ffindex_write(ffindex_index_t* index, FILE* index_file)
{
/* Use tree if available */
+ printf("DEBUG: %s (%i): ffindex_write()\n", __FILE__, __LINE__);
+ fflush(stdout);
if(index->type == TREE)
return ffindex_tree_write(index, index_file);
for(size_t i = 0; i < index->n_entries; i++)
{
+ printf("DEBUG: %s (%i): ffindex_write(), i = %li\n", __FILE__, __LINE__, i);
+ fflush(stdout);
ffindex_entry_t ffindex_entry = index->entries[i];
if(fprintf(index_file, "%s\t%zd\t%zd\n", ffindex_entry.name, ffindex_entry.offset, ffindex_entry.length) < 0)
return EXIT_FAILURE;
@@ -551,6 +565,32 @@ void action(const void *node, const VISI
case postorder:
case leaf:
entry = *(ffindex_entry_t **) node;
+ printf("DEBUG: %s(%i): %p\n", __FILE__, __LINE__, entry);
+ fflush(stdout);
+ if ( !entry ) {
+ fprintf(stderr, "No entry found in %s line %i\n", __FILE__, __LINE__-2);
+ break;
+ }
+ printf("DEBUG: %s(%i): before checking entry->offset\n", __FILE__, __LINE__);
+ fflush(stdout);
+ if ( !entry->offset ) {
+ fprintf(stderr, "Entry has no offset in %s line %i\n", __FILE__, __LINE__-10);
+ break;
+ } else {
+ printf("DEBUG: %s(%i): entry->offset = %li\n", __FILE__, __LINE__, entry->offset);
+ }
+ printf("DEBUG: %s(%i): before checking entry->name\n", __FILE__, __LINE__);
+ fflush(stdout);
+ if ( !entry->name ) {
+ fprintf(stderr, "Entry has no name in %s line %i\n", __FILE__, __LINE__-6);
+ break;
+ } else {
+ printf("DEBUG: %s(%i): entry->name = %s\n", __FILE__, __LINE__, entry->name);
+ }
+ printf("DEBUG: %s(%i): before printing both\n", __FILE__, __LINE__);
+ fflush(stdout);
+ printf("DEBUG: %s(%i): entry->name = %s, entry->offset = %li\n", __FILE__, __LINE__, entry->name, entry->offset);
+ fflush(stdout);
if(fprintf(index_file, "%s\t%zd\t%zd\n", entry->name, entry->offset, entry->length) < 0)
break;
}
@@ -560,7 +600,9 @@ int ffindex_tree_write(ffindex_index_t*
{
int ret = EXIT_SUCCESS;
+ printf("DEBUG: %s(%i): in ffindex_tree_write(): index = %p\n", __FILE__, __LINE__, index);
twalkmisc(index->tree_root, action, (void *) index_file);
+ printf("DEBUG: %s(%i): after twalkmisc in ffindex_tree_write()\n", __FILE__, __LINE__);
return ret;
}
@@ -568,6 +610,8 @@ void ffsort_index(const char* index_file
FILE* index_fh = fopen(index_filename, "r");
size_t lines = ffcount_lines(index_filename);
+ printf("DEBUG: %s (%i): name = %s, lines = %li\n", __FILE__, __LINE__, index_filename, lines);
+ fflush(stdout);
ffindex_index_t* index = ffindex_index_parse(index_fh, lines);
fclose(index_fh);
--- a/src/ffindex_modify.c
+++ b/src/ffindex_modify.c
@@ -106,10 +106,14 @@ int main(int argn, char **argv)
index_file = fopen(index_filename, "r+");
if(index_file == NULL) { perror(index_filename); return EXIT_FAILURE; }
+ printf("DEBUG: %s (%i): index_file = %s\n", __FILE__, __LINE__, index_filename);
+ fflush(stdout);
size_t entries = ffcount_lines(index_filename);
ffindex_index_t* index = ffindex_index_parse(index_file, entries);
if(index == NULL) { perror("ffindex_index_parse failed"); return (EXIT_FAILURE); }
+ printf("DEBUG: %s (%i): index_file = %s, entries = %li\n", __FILE__, __LINE__, index_filename, entries);
+ fflush(stdout);
fclose(index_file);
@@ -118,11 +122,14 @@ int main(int argn, char **argv)
{
if(use_tree)
{
+ printf("DEBUG: %s (%i): unlink + use_tree\n", __FILE__, __LINE__);
+
/* Build tree */
index = ffindex_index_as_tree(index);
/* For each list_file unlink all entries */
if(list_filenames_index > 0)
+ printf("DEBUG: %s (%i): list_filenames_index = %i\n", __FILE__, __LINE__, list_filenames_index);
for(int i = 0; i < list_filenames_index; i++)
{
printf("Unlinking entries from '%s'\n", list_filenames[i]);
@@ -137,7 +144,10 @@ int main(int argn, char **argv)
/* unlink entries specified by args */
for(int i = optind; i < argn; i++)
+{
+ printf("DEBUG: %s (%i): i = %i\n", __FILE__, __LINE__, i);
index = ffindex_unlink(index, argv[i]);
+}
}
else
{
@@ -178,8 +188,14 @@ int main(int argn, char **argv)
/* Write index back */
index_file = fopen(index_filename, "w");
+ printf("DEBUG: %s (%i): index_file = %s, entries = %li\n", __FILE__, __LINE__, index_filename, entries);
+ fflush(stdout);
+
if(index_file == NULL) { perror(index_filename); return EXIT_FAILURE; }
err += ffindex_write(index, index_file);
+ printf("DEBUG: %s (%i): err = %i\n", __FILE__, __LINE__, err);
+ fflush(stdout);
+
return err;
}
Description: Switch off optimisation - does not help
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.11)
-set(CMAKE_C_FLAGS "-std=c99 ${CMAKE_C_FLAGS}")
+set(CMAKE_C_FLAGS "-std=c99 -O0")
# sets HAVE_FMEMOPEN
add_subdirectory(ext)