.travis.yml

+language: c
+
+env:
+  - SQC_NONZERO_EXIT=1
+
+script:
+  - git clone -b develop https://github.com/${TRAVIS_REPO_SLUG/hmmer/easel}.git
+  - ln -s easel/aclocal.m4 aclocal.m4
+  - autoconf
+  - ./configure
+  - make
+  - make dev
+  - make check
+

INSTALL

 Brief installation instructions 
-HMMER 3.2.1 (June 2018)
+HMMER 3.3 (Nov 2019)
 
-Starting from a source distribution, hmmer-3.2.1.tar.gz:
-   uncompress hmmer-3.2.1.tar.gz  
-   tar xf hmmer-3.2.1.tar
-   cd hmmer-3.2.1
+Starting from a source distribution, hmmer-3.3.tar.gz:
+   uncompress hmmer-3.3.tar.gz  
+   tar xf hmmer-3.3.tar
+   cd hmmer-3.3
    ./configure
    make
    make check                        # optional: automated tests

LICENSE

 HMMER - Biological sequence analysis with profile hidden Markov models
-Copyright (C) 1992-2018 Sean R. Eddy
-Copyright (C) 2000-2018 Howard Hughes Medical Institute
-Copyright (C) 2015-2018 President and Fellows of Harvard College
+Copyright (C) 1992-2019 Sean R. Eddy
+Copyright (C) 2000-2019 Howard Hughes Medical Institute
+Copyright (C) 2015-2019 President and Fellows of Harvard College
 Copyright (C) 1992-2004 Washington University School of Medicine
 Copyright (C) 1992-1994 MRC Laboratory of Molecular Biology
 -----------------------------------------------------------------------
@@ -39,12 +39,12 @@ incorporated from other sources, including:
    X Consortium                    
 
 HMMER uses the Easel software library, which has its own license and
-copyright information. See easel/COPYRIGHT and easel/LICENSE.
+copyright information. See easel/LICENSE.
 
 HMMER includes patent-pending SIMD technology under a nonexclusive
-license from Michael Farrar. You are sublicensed to use this
-technology specifically for the use, modification, and redistribution
-of HMMER.
+license from the estate of Michael Farrar. You are sublicensed to use
+this technology specifically for the use, modification, and
+redistribution of HMMER.
 
 HMMER development is supported in part by the National Human Genome
 Research Institute of the US National Institutes of Health under grant

Makefile.in

@@ -109,6 +109,7 @@ tests:
 # check: Run test suites.
 #
 check:
+	@command -v python3 >/dev/null 2>&1 || { echo >&2 "python3 is required for 'make check', but is not in your PATH. Aborting."; exit 1; }
 	${QUIET_SUBDIR0}${ESLDIR}  ${QUIET_SUBDIR1} tests
 	${QUIET_SUBDIR0}${SADIR}   ${QUIET_SUBDIR1} all	
 	${QUIET_SUBDIR0}src        ${QUIET_SUBDIR1} tests
@@ -128,7 +129,7 @@ pdf:
 #          building packages.
 #
 #          Does not use quiet beautification magic; always verbose.
-install: 
+install: all
 	${INSTALL} -d ${DESTDIR}${bindir}
 	${INSTALL} -d ${DESTDIR}${man1dir}
 	${MAKE} -C src install

README.md

 ## HMMER - biological sequence analysis using profile HMMs
 
+[![](https://travis-ci.org/EddyRivasLab/hmmer.svg?branch=develop)](https://travis-ci.org/EddyRivasLab/hmmer)
+![](http://img.shields.io/badge/license-BSD-brightgreen.svg)
+
 [HMMER](http://hmmer.org) searches biological sequence databases for
 homologous sequences, using either single sequences or multiple
 sequence alignments as queries. HMMER implements a technology called
@@ -21,7 +24,7 @@ depends on the Easel library, also at
 ```
    % wget http://eddylab.org/software/hmmer/hmmer.tar.gz
    % tar zxf hmmer.tar.gz
-   % cd hmmer-3.2.1
+   % cd hmmer-3.3
    % ./configure --prefix /your/install/path
    % make
    % make check                 # optional: run automated tests
@@ -74,7 +77,9 @@ includes three main branches:
 
 To build the most recent official release, leave both HMMER and Easel
 on their default **master** branch.  To contribute to HMMER3
-development, you want to be on the **develop** branches.
+development, you want to be on the **develop** branches. If you want
+to send us a pull request on GitHub, please base your changes on our
+**develop** branches.
 
 
 ### to report a problem:

RELEASE-3.2.1

-HMMER 3.2.1 release notes    (Jun 2018)
-http://hmmer.org
-
-Release 3.2.1 fixes:
-----------------------
-* iss #142: compilation fails ungracefully on unsupported ppc64le
-* iss #140: jackhmmer --fast segfault. Removes --fast 
-  and --symfrac options from jackhmmer and its documentation.
-* iss #141, #120: compilation failures on i586, ppc64be.
-* iss #132: hmmbuild miscounts transitions in final node.
-
-
-
-Release notes for 3.2:
------------------------
-Larger changes:
-
-* HMMER is now distributed under the BSD license, instead of the GPL.
-
-* The 'make install' has been streamlined. Now we install only
-  programs and man pages, for 18 HMMER programs and 22 Easel miniapps.
-  We don't install development .h headers or .a libraries.
-
-* We changed our default policy on the number of threads/cores that
-  the search programs use. Previously multithreaded programs would use
-  all available cores by default; now we use two worker threads by
-  default (~2 cores), if multiple cores are available. HMMER3 search
-  programs do not scale much beyond 2 cores anyway, due to input
-  saturation and memory use.
-
-* H3 now strictly requires either SSE2 or Altivec/VMX vector
-  support. The portable 'dummy' implementation has been removed. This
-  was an non-vectorized portable pure C implementation, much slower
-  than HMMER3 on x86 and PowerPC platforms with SSE2 or Altivec/VMX
-  vector instructions. We had included it because we could -- HMMER3
-  implements "generic" non-vectorized versions of its core algorithms
-  for unit testing purposes anyway. We thought maybe it'd be useful.
-  As far as I can tell, it was never useful, but several packagers on
-  non-x86, non-PowerPC platforms packaged and distributed it, ignoring
-  the prominent health warnings we put on it.
-
-
-Smaller new features include:
-
-* improved documentation of --mxfile and score matrix file format
-* adds support for EBI's "uncheck all hits" on jackhmmer web site
-* removed experimental hmmbuild --seq_weights_r and --seq_weights_e options
-* appeased new gcc6 --Wmisleading-indentation warning
-* SSI indexing is now robust against duplicate keys
-* autoconf now robust against someone running gnu `autoheader`
-* improved autoconf of SIMD vector support
-* improved autoconf of DAZ,FTZ support on x86 processors
-* improved autodetection of phylip interleaved vs sequential format
-* improved autodetection of A2M vs. aligned FASTA format
-* clarified A2M format parse error when file contains 'O' residue 
-* improved portability to Intel/Solaris
-* FASTA parser now ignores description line after a ctrl-A
-* MSAs output with -A option now include name, acc, desc, author markup.
-* In Easel miniapps, esl-translate replaces previous (buggy) esl-stranslate.
-* Easel miniapps include esl-alirev, reverse complementing RNA/DNA alignment
-* nhmmer/nhmmscan now allow alignments as target dbs
-* target relentropy for DNA models changed from 0.45 to 0.62
-* updated the User Guide
-
-Fixed bugs include:
-
-* nhmmscan faulted on old model files because it expected MAXL field
-* --cut-ga wasn't checking that GA line was actually present in model
-* GA thresholds weren't being captured from DNA/RNA Stockholm alignments
-* A2M alignment parsing was corrupted on lines ending in insertions
-* alphabet-guessing failed for FASTA seq >4096 bytes and <4000 residues
-* genbank to fasta format conversion was leaving extra \n in accession
-* NCBI db format: multithreaded nhmmer was failing to read 
-* NCBI db format: 1st database sequence was skipped for second query (#e6)
-* unit test failures in esl_normal, esl_random due to floating point math
-* nhmmer was corrupted by * symbols in input sequence (iss#118)
-* hmmsearch -A crashed if top-ranked hit has no domains (iss#131)
-* new variety of "backconverted subseq" hmmsearch crash on *'s (iss#135)
-* qsort() callbacks now return -1/0/1 as they should. (Infernal iss#11)
-* esl_buffer hangs when input ends with \r (Easel iss#23) 
-
-For details, you can view and search our issue tracking logs at github:
-  https://github.com/EddyRivasLab/hmmer/issues/   for HMMER itself
-  https://github.com/EddyRivasLab/easel/issues/   for the underlying Easel library
-
-

RELEASE-3.3.md

+# HMMER 3.3 release notes (Nov 2019)
+
+
+## most important changes:
+
+* We improved the `hmmpgmd` search daemon. (`hmmpgmd` is the compute
+  server used at EBI to support their HMMER web servers.)  
+  Now `hmmpgmd` handles large target sequence databases more
+  efficiently by using "sharding". Instead of loading the entire
+  target sequence database into memory on every node, a target
+  database can be sharded across multiple nodes. The `hmmpgmd` daemon
+  also now has its own user guide.
+
+* We improved how we calculate our default sequence weights (Henikoff
+  position-based weights), especially on deep alignments of 10-100K+
+  sequences. Now we calculate PB weights only on consensus columns,
+  not all columns. This avoids some cases of artifactually extreme
+  weights on very gappy alignments. We also changed to a better rule
+  for defining sequence fragments.  These changes give a small
+  improvement in sensitivity/specificity benchmarking of HMMER/Pfam
+  searches, and substantial speed improvements in building profiles on
+  deep alignments. Because of these changes, profile HMMs built with
+  version 3.3 give slightly different scores compared to previous
+  HMMER3 versions.
+
+* Fixed a bug in the `hmmstat` "compKL" (composition KL divergence)
+  calculation, which was off by a constant. Now it is reported as a
+  standard KL divergence in bits.
+
+
+## bug fixes:
+
+* fixed a bug where in some rare and complicated situations, `nhmmer`
+  could report overlapping envelopes on reverse strand. [iss#159]
+
+* Several bugs were fixed in MPI mode, including iss#157 (`hmmsim
+  --mpi` was always segfaulting, even on simple examples) and iss#154
+  (problems in `hmmscan` that were corrupting output data).
+
+* Fixed some 32-bit integer overflow bugs in `hmmpgmd`. 
+
+* Fixed a bug in the `hmmstat` "compKL" (composition KL divergence)
+  calculation, which was off by a constant. Now it is reported as a
+  standard KL divergence in bits.
+
+* Fixed a bug where `hmmconvert --outfmt` wasn't recognizing `3/f` as
+  a format.
+
+
+## Smaller changes
+
+* Our fasta format parser now detects aligned FASTA format (.afa
+  files) more robustly, and will not attempt to read a .afa file as an
+  unaligned sequence file. [iss#153]
+
+* Our `make check` tests depend on Python >= 3.5. Added checks in
+  `./configure` and `make` to fail gracefully if python3 isn't available.
+
+* `./configure` now always calls `AC_PROG_CC_STDC` to add compiler flags
+  for C99 (even with icc).
+
+* Removed undocumented `--{tmm,tmi,tmd,tim,tii,tdm,tdd}` options from
+  hmmbuild.
+
+* Data serialization routines (used in `hmmpgmd`) were rewritten and
+  improved.
+
+
+For even more information, you can peruse the
+[git log for our develop branch](https://github.com/EddyRivasLab/hmmer/commits/develop).
+
+

configure.ac

@@ -61,7 +61,6 @@ AC_PREREQ(2.63)
 # It's sort of standard to expect them in m4/, don't 
 # be confused that they're down in easel.
 #
-m4_include([easel/m4/ax_compiler_vendor.m4])
 m4_include([easel/m4/ax_gcc_func_attribute.m4])
 
 m4_include([easel/m4/esl_sse.m4])
@@ -70,12 +69,13 @@ m4_include([easel/m4/esl_vmx.m4])
 m4_include([easel/m4/ax_mpi.m4])
 m4_include([easel/m4/ax_pthread.m4])
 
+m4_include([easel/m4/esl_pic_flags.m4])
 
 ################################################################
 # 2. AC_INIT
 ################################################################
 
-AC_INIT(HMMER, 3.2.1, sean@eddylab.org, hmmer)
+AC_INIT(HMMER, 3.3, sean@eddylab.org, hmmer)
 AC_MSG_NOTICE([Configuring HMMER3 for your system.])
 
 # remember if the user is overriding CFLAGS
@@ -107,8 +107,8 @@ fi
 # to use HMMER as a library inside other packages, with no name clashes.
 ################################################################
 
-HMMER_DATE="June 2018"
-HMMER_COPYRIGHT="Copyright (C) 2018 Howard Hughes Medical Institute."
+HMMER_DATE="Nov 2019"
+HMMER_COPYRIGHT="Copyright (C) 2019 Howard Hughes Medical Institute."
 HMMER_LICENSE="Freely distributed under the BSD open source license."
 HMMER_VERSION=$PACKAGE_VERSION
 HMMER_URL="http://hmmer.org/"
@@ -116,10 +116,10 @@ HMMER_URL="http://hmmer.org/"
 HMMER_ESLDIR="easel"
 HMMER_SADIR="libdivsufsort"
 
-EASEL_DATE="June 2018"
-EASEL_COPYRIGHT="Copyright (C) 2018 Howard Hughes Medical Institute."
+EASEL_DATE="Nov 2019"
+EASEL_COPYRIGHT="Copyright (C) 2019 Howard Hughes Medical Institute."
 EASEL_LICENSE="Freely distributed under the BSD open source license."
-EASEL_VERSION="0.45h"
+EASEL_VERSION="0.46"
 EASEL_URL="http://bioeasel.org/"
 
 AC_SUBST(HMMER_DATE)
@@ -147,6 +147,7 @@ AC_DEFINE_UNQUOTED([EASEL_DATE],      ["$EASEL_DATE"],      [Easel release date]
 AC_DEFINE_UNQUOTED([EASEL_COPYRIGHT], ["$EASEL_COPYRIGHT"], [Easel copyright])
 AC_DEFINE_UNQUOTED([EASEL_LICENSE],   ["$EASEL_LICENSE"],   [Easel license])
 AC_DEFINE_UNQUOTED([EASEL_VERSION],   ["$EASEL_VERSION"],   [Easel version])
+AC_DEFINE_UNQUOTED([EASEL_URL],       ["$EASEL_URL"],       [Easel web URL])
 
 
 # Figure out what host we're compiling on.
@@ -199,10 +200,13 @@ AC_ARG_ENABLE(vmx,     [AS_HELP_STRING([--enable-vmx],     [enable our Altivec/V
 AC_ARG_ENABLE(threads, [AS_HELP_STRING([--enable-threads], [enable POSIX threads parallelization])],     enable_threads=$enableval, enable_threads=check)
 AC_ARG_ENABLE(mpi,     [AS_HELP_STRING([--enable-mpi],     [enable MPI parallelization])],               enable_mpi=$enableval,     enable_mpi=no)
 
+AC_ARG_ENABLE(pic,     [AS_HELP_STRING([--enable-pic],     [enable position-independent code])],         enable_pic=$enableval,     enable_pic=no)
+
 AC_ARG_WITH(gsl,       [AS_HELP_STRING([--with-gsl],       [use the GSL, GNU Scientific Library])],      with_gsl=$withval,         with_gsl=no)
 
 
 
+
 # If a vector implementation is force-selected, make sure only one is,
 # and turn off checking for the others.
 vecsel=0
@@ -221,19 +225,13 @@ fi
 ################################################################
 
 AC_PROG_CC
+AC_PROG_CC_STDC
 AC_PROG_CPP
 AC_PROG_INSTALL
 AC_PROG_RANLIB
 AC_PATH_PROG([AR], [ar], [:], [$PATH:/usr/ccs/bin:/usr/xpg4/bin])
 AC_PROG_LN_S
 
-# AC_PROG_CC_STDC wants to append -c99 to icc; icc doesn't like this.
-AX_COMPILER_VENDOR
-if test "$ax_cv_c_compiler_vendor" != "intel"; then
-  AC_PROG_CC_STDC
-fi
-
-
 # Select our default optimization flags in CFLAGS.
 #  --enable-gcov, --enable-gprof, and --enable-debugging are mutually exclusive.
 #
@@ -276,6 +274,12 @@ if test "$enable_mpi" = "yes"; then
 fi
 
 
+# PIC (position-independent code) for shared library support
+#
+if test "$enable_pic" = "yes"; then
+   ESL_PIC_FLAGS
+fi
+
 
 # Support for POSIX multithreading (we should generally have this)
 #
@@ -409,6 +413,13 @@ else
   AC_DEFINE(HAVE_GZIP, 1, [Support external gzip decompression])
 fi
 
+# We need python 3 for 'make check' and some dev tools. sqc checks
+# too, as does the Makefile, but we also check in ./configure, so we
+# don't recommend 'make check' to the user if they can't use it.
+# 
+AC_CHECK_PROG(HAVE_PYTHON3, python3, yes)
+
+
 ################################################################
 # 6. Checks for libraries
 #################################################################
@@ -584,6 +595,7 @@ AC_CONFIG_FILES([ \
   documentation/Makefile                       \
   documentation/man/Makefile                   \
   documentation/userguide/Makefile             \
+  documentation/userguide/inclusions/Makefile  \
   libdivsufsort/Makefile                       \
   ])
 
@@ -599,6 +611,7 @@ AC_CONFIG_FILES([ \
 # HMMER_VERSION, HMMER_DATE, HMMER_COPYRIGHT
 AC_CONFIG_FILES([ \
   documentation/userguide/titlepage.tex  \
+  documentation/userguide/titlepage_daemon.tex  \
   documentation/userguide/copyright.tex  \
   ])
 
@@ -608,12 +621,14 @@ AC_CONFIG_FILES([ \
   documentation/man/alimask.man     \
   documentation/man/hmmalign.man    \
   documentation/man/hmmbuild.man    \
+  documentation/man/hmmc2.man       \
   documentation/man/hmmconvert.man  \
   documentation/man/hmmemit.man     \
   documentation/man/hmmer.man       \
   documentation/man/hmmfetch.man    \
   documentation/man/hmmlogo.man     \
   documentation/man/hmmpgmd.man     \
+  documentation/man/hmmpgmd_shard.man     \
   documentation/man/hmmpress.man    \
   documentation/man/hmmscan.man     \
   documentation/man/hmmsearch.man   \
@@ -642,6 +657,7 @@ AC_CONFIG_FILES([ \
   easel/miniapps/esl-construct.man   \
   easel/miniapps/esl-histplot.man    \
   easel/miniapps/esl-mask.man        \
+  easel/miniapps/esl-mixdchlet.man   \
   easel/miniapps/esl-reformat.man    \
   easel/miniapps/esl-selectn.man     \
   easel/miniapps/esl-seqrange.man    \
@@ -673,17 +689,25 @@ AC_OUTPUT
 echo "
 
 HMMER configuration:
-     compiler:             ${CC} ${CFLAGS} ${SSE_CFLAGS} ${VMX_CFLAGS} ${PTHREAD_CFLAGS} 
+   compiler:             ${CC} ${CFLAGS} ${SSE_CFLAGS} ${VMX_CFLAGS} ${PTHREAD_CFLAGS} ${PIC_CFLAGS}
    host:                 $host
    linker:               ${LDFLAGS}
    libraries:            ${LIBS} ${LIBGSL} ${PTHREAD_LIBS}
-     DP implementation:    ${impl_choice}
+   DP implementation:    ${impl_choice}"
+
 
-Do 'make'  to build HMMER, and optionally:
+if test x"$HAVE_PYTHON3" = x"yes"; then echo "
+Now do 'make'  to build HMMER, and optionally:
        'make check'  to run self tests,
        'make install'  to install programs and man pages,
        '(cd easel; make install)'  to install Easel tools.
-"
-
+";
+else echo "
+   (No python3 found, so 'make check' is disabled.)
 
+Now do 'make'  to build HMMER, and optionally:
+       'make install'  to install programs and man pages,
+       '(cd easel; make install)'  to install Easel tools.
+";
+fi
 

documentation/man/Makefile.in

@@ -23,11 +23,13 @@ INSTALL     = @INSTALL@
 MANS =  hmmer\
 	hmmalign\
 	hmmbuild\
+	hmmc2\
 	hmmconvert\
 	hmmemit\
 	hmmfetch\
 	hmmlogo\
 	hmmpgmd\
+	hmmpgmd_shard\
 	hmmpress\
 	hmmscan\
 	hmmsearch\

documentation/man/hmmc2.man.in

+.TH "hmmc2" 1 "@HMMER_DATE@" "HMMER @HMMER_VERSION@" "HMMER Manual"
+
+.SH NAME
+hmmc2 \- example client for the HMMER daemon
+
+
+.SH SYNOPSIS
+.B hmmc2
+[\fIoptions\fR]
+
+
+.SH DESCRIPTION
+
+.PP
+.B Hmmc2
+is a text client for the hmmpgmd or hmmpgmd_shard daemons.  When run, it opens a connection to a daemon at the specified
+IP address and port, and then enters an interactive loop waiting for the user to input commands to be sent to the daemon.
+See the User's Guide for the HMMER Daemon for a discussion of hmmpgmd's command format.
+ 
+
+.SH OPTIONS
+
+.TP
+.B \-i <IP address>
+Specify the IP address of the daemon that hmmc2 should connect to.  Defaults to 127.0.0.1 if not provided
+
+
+.TP
+.B \-p <port number>
+Specify the port number that the daemon is listening on.  Defaults to 51371 if not provided
+
+
+.TP
+.B \-S
+Print the scores of any hits found during searches.
+
+
+.TP
+.B \-A
+Print the alignment of any hits found during searches.  This is a superset of the "-S" flag, so providing both is redundant.
+
+
+
+.SH SEE ALSO 
+
+See 
+.BR hmmer (1)
+for a master man page with a list of all the individual man pages
+for programs in the HMMER package.
+
+.PP
+For complete documentation, see the user guide that came with your
+HMMER distribution (Userguide.pdf); or see the HMMER web page
+(@HMMER_URL@).
+
+
+
+.SH COPYRIGHT
+
+.nf
+@HMMER_COPYRIGHT@
+@HMMER_LICENSE@
+.fi
+
+For additional information on copyright and licensing, see the file
+called COPYRIGHT in your HMMER source distribution, or see the HMMER
+web page 
+(@HMMER_URL@).
+
+
+.SH AUTHOR
+
+.nf
+http://eddylab.org
+.fi
+
+
+

documentation/man/hmmfetch.man.in

@@ -146,10 +146,7 @@ instead of to standard output.
 .B \-O
 Output HMM(s) to individual file(s) named
 .I key
-instead of standard output. With the
-.B \-f 
-option, this can result in many files
-being created.
+instead of standard output.
 
 .TP
 .B \-\-index

documentation/man/hmmpgmd.man.in

@@ -48,7 +48,7 @@ prints the line:
 
 
 .PP
-Only after master is ready, one or more instances of hmmpgmd may
+After the master is ready, one or more instances of hmmpgmd may
 be started as workers. These workers may be (and typically are) on
 different machines from the master, but must have access to the 
 same database file(s) provided to the master, with the same path. As 
@@ -57,7 +57,7 @@ indicates completion by printing: "Data loaded into memory. Worker is ready."
 
 
 .PP
-The master server and workers are expected to remain running.
+The master process and workers are expected to remain running.
 One or more clients then connect to the master and submit possibly
 many queries. The master distributes the work of a query among the
 workers, collects results, and merges them before responding to the
@@ -121,11 +121,6 @@ can be associated with one or more sub-databases. The
 flag indicates which of these sub-databases will be queried. 
 The HMM database format does not support sub-databases.
 
-.PP
-The result of each query is an undocumented data structure in 
-binary format. In the future the data will be returned in a proper
-serialized structure, but for now, it requires meticulous unpacking
-within the client. The example clients show how this is done.
 
  
 
@@ -136,10 +131,6 @@ within the client. The example clients show how this is done.
 Help; print a brief reminder of command line usage and all available
 options.
 
-
-
-.SH EXPERT OPTIONS
-
 .TP 
 .BI \-\-master
 Run as the master server.
@@ -150,10 +141,6 @@ Run as a worker, connecting to the master server that is running on IP
 address
 .IR <s> .
 
-.TP 
-.BI \-\-daemon
-Run as a daemon using config file: /etc/hmmpgmd.conf
-
 .TP 
 .BI \-\-cport " <n>"
 Port to use for communication between clients and the master server. 

documentation/man/hmmpgmd_shard.man.in

+.TH "hmmpgmd_shard" 1 "@HMMER_DATE@" "HMMER @HMMER_VERSION@" "HMMER Manual"
+
+.SH NAME
+hmmpgmd_shard \- sharded daemon for database search web services 
+
+
+.SH SYNOPSIS
+.B hmmpgmd_shard
+[\fIoptions\fR]
+
+
+.SH DESCRIPTION
+
+.PP
+The 
+.B hmmpgmd_shard 
+program provides a sharded version of the 
+.B hmmpgmd 
+program that we use internally to implement high-performance HMMER services that can be accessed via the internet.  See the 
+.B hmmpgmd
+man page for a discussion of how the base 
+.B hmmpgmd
+program is used.  This man page discusses differences between 
+.B hmmpgmd_shard
+and 
+.B hmmpgmd.   
+The base 
+.B hmmpgmd
+program loads the entirety of its database file into RAM on every worker node, in spite of the fact that each worker node searches a predictable fraction of the database(s) contained in that file when performing searches.  This wastes RAM, particularly when many worker nodes are used to accelerate searches of large databases.
+
+.PP
+.B Hmmpgmd_shard 
+addresses this by dividing protein sequence database files into shards.  Each worker node loads only 1/Nth of the database file, where N is the number of worker nodes attached to the master.  HMM database files are not sharded, meaning that every worker node will load the entire database file into RAM.  Current HMM databases are much smaller than current protein sequence databases, and easily fit into the RAM of modern servers even without sharding.
+
+.PP
+.B Hmmpgmd_shard 
+is used in the same manner as 
+.B hmmpgmd
+, except that it takes one additional argument: 
+.BI \-\-num_shards " <n>"
+, which specifies the number of shards that protein databases will be divided into, and defaults to 1 if unspecified.  This argument is only valid for the master node of a 
+.B hmmpgmd
+system (i.e., when 
+.BI \-\-master
+is passed to the 
+.B hmmpgmd
+program), and must be equal to the number of worker nodes that will connect to the master node.  
+.B Hmmpgmd_shard 
+will signal an error if more than 
+.BI num_shards
+worker nodes attempt to connect to the master node or if a search is started when fewer than 
+.BI num_shards 
+workers are connected to the master.
+
+.SH OPTIONS
+
+.TP
+.B \-h
+Help; print a brief reminder of command line usage and all available
+options.
+
+.TP 
+.BI \-\-master
+Run as the master server.
+
+.TP
+.BI \-\-worker " <s>"
+Run as a worker, connecting to the master server that is running on IP
+address
+.IR <s> .
+
+.TP 
+.BI \-\-cport " <n>"
+Port to use for communication between clients and the master server. 
+The default is 51371.
+
+.TP 
+.BI \-\-wport " <n>"
+Port to use for communication between workers and the master server. 
+The default is 51372.
+
+.TP 
+.BI \-\-ccncts " <n>"
+Maximum number of client connections to accept. The default is 16.
+
+.TP 
+.BI \-\-wcncts " <n>"
+Maximum number of worker connections to accept. The default is 32.
+
+.TP 
+.BI \-\-pid " <f>"
+Name of file into which the process id will be written. 
+
+.TP 
+.BI \-\-seqdb " <f>"
+Name of the file (in
+.B hmmpgmd
+format) containing protein sequences.
+The contents of this file will be cached for searches. 
+
+.TP 
+.BI \-\-hmmdb " <f>"
+Name of the file containing protein HMMs. The contents of this file 
+will be cached for searches.
+
+.TP 
+.BI \-\-cpu " <n>"
+Number of parallel threads to use (for 
+.B \-\-worker
+).
+
+.TP 
+.BI \-\-num_shards " <n>"
+Number of shards to divide cached sequence database(s) into.  HMM databases are not sharded, due to their small size.
+This option is only valid when the 
+.B \-\-master 
+option is present, and defaults to 1 if not specified.
+.B Hmmpgmd_shard 
+requires that the number of shards be equal to the number of worker nodes, and will give errors if more than 
+.BI num_shards 
+workers attempt to connect to the master node or if a search is started with fewer than 
+.BI num_shards 
+workers connected to the master.
+
+.SH SEE ALSO 
+
+See 
+.BR hmmmpgmd (1)
+for a description of the base hmmpgmd command and how the daemon should be used.
+
+.BR hmmer (1)
+for a master man page with a list of all the individual man pages
+for programs in the HMMER package.
+
+.PP
+For complete documentation, see the user guide that came with your
+HMMER distribution (Userguide.pdf); or see the HMMER web page
+(@HMMER_URL@).
+
+
+
+.SH COPYRIGHT
+
+.nf
+@HMMER_COPYRIGHT@
+@HMMER_LICENSE@
+.fi
+
+For additional information on copyright and licensing, see the file
+called COPYRIGHT in your HMMER source distribution, or see the HMMER
+web page 
+(@HMMER_URL@).
+
+
+.SH AUTHOR
+
+.nf
+http://eddylab.org
+.fi
+
+
+

documentation/man/hmmstat.man.in

@@ -31,7 +31,7 @@ The columns are:
 
 .TP
 .B idx
-The index of this profile, numbering each on in the file starting from 1.
+The index of this profile, numbering each profile in the file starting from 1.
 
 .TP
 .B name
@@ -79,8 +79,9 @@ per model consensus position.
 
 .TP
 .B compKL
-Kullback-Leibler distance between the model's overall average
-residue composition and the default background frequency distribution.
+Kullback-Leibler divergence from
+the default background frequency distribution to
+the average composition of the profile's consensus match states, in bits.
 The higher this number, the more biased the residue composition of the
 profile is. Highly biased profiles can slow the HMMER3 acceleration
 pipeline, by causing too many nonhomologous sequences to pass the filters.

documentation/userguide/Makefile.in

@@ -26,16 +26,27 @@ INSTALL       = @INSTALL@
 RMAN          = rman
 RMANPROCESS   = ${top_srcdir}/easel/devkit/rmanprocess.py
 
+# beautification magic stolen from git
+QUIET_SUBDIR0 = +${MAKE} -C #space separator after -c
+QUIET_SUBDIR1 = 
+ifndef V
+	QUIET_SUBDIR0 = +@subdir=
+	QUIET_SUBDIR1 = ; echo '    ' SUBDIR $$subdir; \
+		        ${MAKE} -s -C $$subdir
+endif
+
 # ./configure puts HMMER .man pages in ${top_builddir}/documentation/man
 MANPAGES = \
 	alimask.man     \
 	hmmalign.man    \
 	hmmbuild.man    \
+	hmmc2.man       \
 	hmmconvert.man  \
 	hmmemit.man     \
 	hmmfetch.man    \
 	hmmlogo.man     \
 	hmmpgmd.man     \
+	hmmpgmd_shard.man \
 	hmmpress.man    \
 	hmmscan.man     \
 	hmmsearch.man   \
@@ -47,6 +58,11 @@ MANPAGES = \
 	nhmmscan.man    \
 	phmmer.man      
 
+MANPAGES_DAEMON = \
+	hmmc2.man       \
+	hmmpgmd.man     \
+	hmmpgmd_shard.man
+
 # ./configure puts Easel .man pages in ${top_builddir}/easel/miniapps
 EASEL_MANPAGES = \
 	esl-afetch.man     \
@@ -62,6 +78,7 @@ EASEL_MANPAGES = \
 	esl-construct.man  \
 	esl-histplot.man   \
 	esl-mask.man       \
+	esl-mixdchlet.man  \
 	esl-reformat.man   \
 	esl-selectn.man    \
 	esl-seqrange.man   \
@@ -75,6 +92,7 @@ EASEL_MANPAGES = \
 TEXFILES =\
 	ack.tex          \
 	copyright.tex    \
+	daemon.tex       \
 	formats.tex      \
 	glossary.tex     \
 	install.tex      \
@@ -84,11 +102,12 @@ TEXFILES =\
 	pipeline.tex     \
 	tabular.tex      \
 	titlepage.tex    \
+	titlepage_daemon.tex \
 	tutorial.tex
 
 .PHONY: pdf manpages clean distclean TAGS
 
-pdf:    Userguide.pdf
+pdf:    Userguide.pdf Daemon_userguide.pdf
 
 # We add ${srcdir} to TEXINPUTS, etc so we can build userguide in a build tree, with input files in source tree.
 Userguide.pdf: manpages ${TEXFILES}
@@ -102,11 +121,23 @@ Userguide.pdf: manpages ${TEXFILES}
 	@TEXINPUTS=${TEXINPUTS}:${srcdir} pdflatex main                                >> latex.log 2>&1 
 	@mv main.pdf Userguide.pdf
 
+Daemon_userguide.pdf: manpages ${TEXFILES}
+	@for prog in pdflatex bibtex; do \
+	  command -v $$prog >/dev/null 2>&1 || { echo >&2 "$$prog is required to build Userguide, but it's not installed. Aborting."; exit 1; } \
+	done
+	@echo "     LATEX Daemon_userguide.pdf  (see latex.log for output)"
+	@TEXINPUTS=${TEXINPUTS}:${srcdir} pdflatex daemon                                >  latex.log 2>&1 
+	@BIBINPUTS=${BIBINPUTS}:${srcdir} BSTINPUTS=${BSTINPUTS}:${srcdir} bibtex main >> latex.log 2>&1 
+	@TEXINPUTS=${TEXINPUTS}:${srcdir} pdflatex daemon                                >> latex.log 2>&1 
+	@TEXINPUTS=${TEXINPUTS}:${srcdir} pdflatex daemon                                >> latex.log 2>&1 
+	@mv daemon.pdf Daemon_userguide.pdf
+
 # manpages: convert man pages to LaTeX chapter in User Guide.
 #    uses PolyglotMan 3.2 "rman", and rmanprocess.py script in easel's devkit
 manpages:
 	@command -v ${RMAN} >/dev/null 2>&1 || { echo >&2 "${RMAN} is required to build Userguide, but it's not installed. Aborting."; exit 1; }
 	@echo '%% Manual pages chapter automatically generated. Do not edit.'           > manpages.tex
+	@echo '%% Manual pages chapter automatically generated. Do not edit.'           > manpages_daemon.tex
 	@echo '%% Easel miniapps manpage chapter automatically generated. Do not edit.' > easel_manpages.tex
 	@for file in ${MANPAGES}; do\
 	   ${RMAN} -f latex2e ../man/$$file 2>/dev/null | ${RMANPROCESS} >> manpages.tex ;\
@@ -114,22 +145,30 @@ manpages:
 	@for file in ${EASEL_MANPAGES}; do\
 	   ${RMAN} -f latex2e ../../easel/miniapps/$$file 2>/dev/null | ${RMANPROCESS} >> easel_manpages.tex ;\
 	done
+	@for file in ${MANPAGES_DAEMON}; do\
+	   ${RMAN} -f latex2e ../man/$$file 2>/dev/null | ${RMANPROCESS} >> manpages_daemon.tex ;\
+	done
 
 clean:
+	${QUIET_SUBDIR0}inclusions   ${QUIET_SUBDIR1} clean
 	-rm -f latex.log
 	-rm -f main.aux main.bbl main.blg main.log main.toc main.brf main.out x.log *~
+	-rm -f daemon.aux daemon.log daemon.out daemon.toc 
 ifndef V
 	@echo '     ' CLEAN userguide
 endif
 
 
 distclean: clean
+	${QUIET_SUBDIR0}inclusions   ${QUIET_SUBDIR1} distclean
 	if test "x${srcdir}" != "x."; then \
 	   for texfile in ${TEXFILES}; do \
 	      rm -f $$texfile ;\
 	   done ;\
 	fi
 	-rm -f titlepage.tex copyright.tex manpages.tex easel_manpages.tex
+	-rm -f manpages_daemon.tex titlepage_daemon.tex
 	-rm -f Userguide.pdf
+	-rm -f Daemon_userguide.pdf
 	-rm -f Makefile