.travis.yml

@@ -7,13 +7,15 @@ env:
   - PYTHON_VERSION=3.4 USE_CYTHON=True
   - PYTHON_VERSION=3.5 USE_H5PY=True
   - PYTHON_VERSION=3.5 USE_CYTHON=True
+  - PYTHON_VERSION=3.6 USE_H5PY=True
+  - PYTHON_VERSION=3.6 USE_CYTHON=True
 before_install:
   - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   - chmod +x miniconda.sh
   - ./miniconda.sh -b
   - export PATH=/home/travis/miniconda3/bin:$PATH
 install:
-  - conda create --yes -n env_name python=$PYTHON_VERSION pip click numpy scipy nose pep8 flake8 coverage future
+  - conda create --yes -n env_name python=$PYTHON_VERSION pip click numpy scipy nose pep8 flake8 coverage future six pandas
   - if [ ${USE_CYTHON} ]; then conda install --yes -n env_name cython; fi
   - if [ ${USE_H5PY} ]; then conda install --yes -n env_name h5py>=2.2.0; fi
   - if [ ${PYTHON_VERSION} = "2.7" ]; then conda install --yes -n env_name Sphinx=1.2.2; fi
@@ -30,5 +32,10 @@ script:
   - if [ ${USE_H5PY} ]; then for table in examples/*hdf5.biom; do echo ${table}; biom validate-table -i ${table}; done; fi
   # validate JSON formatted tables
   - for table in examples/*table.biom; do echo ${table}; biom validate-table -i ${table}; done;
+  - pushd biom/assets
+  - if [ ${USE_H5PY} ]; then python exercise_api.py ../../examples/rich_sparse_otu_table_hdf5.biom sample; fi
+  - if [ ${USE_H5PY} ]; then python exercise_api.py ../../examples/rich_sparse_otu_table_hdf5.biom observation; fi
+  - if [ ${USE_H5PY} ]; then sh exercise_cli.sh; fi
+  - popd
 after_success:
   - coveralls

ChangeLog.md

 BIOM-Format ChangeLog
 =====================
 
+biom 2.1.6
+----------
+
+New features and bug fixes, released on 27 April 2017.
+
+New Features:
+
+* `Table.from_hdf5` now supports a rapid subset in the event that metadata is
+   not needed. In benchmarking against the Earth Microbiome Project BIOM table,
+   the reduction in runtime was multiple orders of magnitude while additionally
+   preserving substantial memory. 
+* `Table.rankdata` has been added to convert values to ranked abundances on
+  either axis. See [issue #645](https://github.com/biocore/biom-format/issues/639).
+* Format of numbers in ``biom summarize-table`` output is now more readable and localized. See [issue #679](https://github.com/biocore/biom-format/issues/679).
+* `Table.concat` has been added to the API and allows for concatenating multiple tables in which the IDs of one of the axes are known to be disjoint. This has substantial performance benefits over `Table.merge`.
+* `Table.sort_order` was performing an implicit cast to dense, and not leveraging fancy indexing. A substantial performance gain was acheived. See [PR #720](https://github.com/biocore/biom-format/pull/720).
+* `biom subset-table` now accepts a QIIME-like mapping file when subsetting by IDs [Issue #587](https://github.com/biocore/biom-format/issues/587)
+* `Table.del_metadata` was added to support the removal of metadata entries from the table [Issue #708](https://github.com/biocore/biom-format/issues/708).
+* `Table.to_dataframe` was added to cast the internal matrix data to a Pandas `SparseDataFrame` [Issue #622](https://github.com/biocore/biom-format/issues/622).
+* `Table.metadata_to_dataframe` was added to cast axis metadata to a Pandas `DataFrame` [Issue #622](https://github.com/biocore/biom-format/issues/622).
+* `test_table.py` and `test_util.py` now use a stable random seed. See issue [#728](https://github.com/biocore/biom-format/issues/728)
+* Failure to cast a value when parsing a TSV will now print the associated line number which had the bad value. See [#284](https://github.com/biocore/biom-format/issues/284).
+* `Table.remove_empty` has been added to remove zero'd samples, observations or both. See [#721](https://github.com/biocore/biom-format/issues/721).
+* A subcommand of the command line interface was added to obtain a table's IDs: `table-ids`.
+
+Bug fixes:
+
+* ``-o`` is now a required parameter of ``biom from-uc``. This was not the case previously, which resulted in a cryptic error message if ``-o`` was not provided. See [issue #683](https://github.com/biocore/biom-format/issues/683).
+* Matrices are now cast to csr on `Table` construction if the data evaluate as `isspmatrix`. This fixes [#717](https://github.com/biocore/biom-format/issues/717) where some API methods assumed the data were csc or csr.
+* `Table.concat` was not handling tables without metadata, resulting in an exception due to mismatches metadata shape. See [#724](https://github.com/biocore/biom-format/issues/724).
+* When validating a BIOM-Format 1.0.0 table, specifying the version string would trigger an error. See [#664](https://github.com/biocore/biom-format/issues/664). An explicit regression test was not added as this stemmed from an integration, and there currently is not support for script usage tests; see [#656](https://github.com/biocore/biom-format/issues/656).
+* `Table.nnz` was not calling `eliminate_zeros()` on the underlying sparse matrix, resulting in the possibility of counting explicitly set zero values. See [#727](https://github.com/biocore/biom-format/issues/727).
+* `Table.from_hdf5` was not properly turning `bytes` into `str` for the `table_id` and the `type` HDF5 attributes. See [#731](https://github.com/biocore/biom-format/issues/731).
+* `Table.__init__` now always performs an `astype(float)` on the contained `spmatrix`. This type normalization is beneficial for underlying Cython code on the filtering and transform operations. It is possible this will introduce some performance overhead, however in _most_ cases the data should already be float. See [#718](https://github.com/biocore/biom-format/issues/718).
+* `Table.to_hdf5` was not handling lists of str appropriately in the general case. Ssee [#638](https://github.com/biocore/biom-format/issues/638).
+* `Table.to_hdf5` was not handling taxonomy as flat strings, which was a common mistake that was outside of expectations for the formatter. The formatter now attempts to split on semicolon if this scenario is encountered, and errors with a more informative error if a problem occurs. See [#530](https://github.com/biocore/biom-format/issues/530).
+
 biom 2.1.5
 ----------
 
@@ -31,6 +68,10 @@ Bug fixes:
  [issue #633](https://github.com/biocore/biom-format/issues/633).
 * `biom --version` now prints the software version (previously the individual
   commands did this, but not the base command).
+* `Table.vlen_list_of_str_formatter` was considering a `str` to be valid for
+  formatting resulting in an obscure error when a `str`, as opposed to a 
+  `list` of `str`, was used for taxonomy. See 
+  [issue #709](https://github.com/biocore/biom-format/issues/709).
 
 biom 2.1.4
 ----------

biom/__init__.py

@@ -41,7 +41,7 @@ either in TSV, HDF5, JSON, gzip'd JSON or gzip'd TSV and parse accordingly:
 
 """
 # ----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
@@ -53,7 +53,7 @@ from .parse import parse_biom_table as parse_table, load_table
 from .util import __format_version__, __version__
 
 __author__ = "Daniel McDonald"
-__copyright__ = "Copyright 2011-2013, The BIOM Format Development Team"
+__copyright__ = "Copyright 2011-2017, The BIOM Format Development Team"
 __credits__ = ["Daniel McDonald", "Jai Ram Rideout", "Greg Caporaso",
                "Jose Clemente", "Justin Kuczynski", "Antonio Gonzalez",
                "Yoshiki Vazquez Baeza", "Jose Navas", "Adam Robbins-Pianka",

biom/_filter.pyx

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2013, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

biom/_subsample.pyx

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2013, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

biom/_transform.pyx

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2013, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
@@ -29,7 +29,7 @@ def _transform(arr, ids, metadata, function, axis):
         metadata along the given axis.
     function : function
         A function that takes three values: an array of nonzero values
-        for each colum or row of `arr`, an id string and a metadata
+        for each column or row of `arr`, an id string and a metadata
         dictionary. It must return an array of transformed values.
     axis : int
         Transform rows of `arr` if 0, columns if 1.

biom/assets/exercise_api.py

+# ----------------------------------------------------------------------------
+# Copyright (c) 2011-2015, The BIOM Format Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# ----------------------------------------------------------------------------
+
+import sys
+import biom
+import numpy as np
+import tempfile
+import h5py
+
+if '://' in sys.argv[1]:
+    from urllib import request
+    fp, _ = request.urlretrieve(sys.argv[1])
+else:
+    fp = sys.argv[1]
+axis = sys.argv[2]
+table = biom.load_table(fp)
+
+# drop some samples, verify what gets dropped
+sample_sums = table.sum(axis=axis)
+idx = np.argsort(sample_sums)[:int(len(table.ids()) / 2)]
+dropped_ids = table.ids(axis=axis)[idx]
+obs_by_id = table.filter(lambda v, i, md: i not in set(dropped_ids),
+                         axis=axis, inplace=False)
+assert set(obs_by_id.ids()).intersection(set(dropped_ids)) == set()
+obs_by_id = table.filter(set(dropped_ids), invert=True,
+                         axis=axis, inplace=False)
+assert set(obs_by_id.ids()).intersection(set(dropped_ids)) == set()
+
+obs_without_top = table.filter(lambda v, i, md: v.sum() < sample_sums.max(),
+                               inplace=False, axis=axis)
+exp = table.ids(axis=axis)[sample_sums != sample_sums.max()]
+assert set(obs_without_top.ids(axis=axis)) == set(exp)
+
+# arbitrary partitioning and collapsing
+md = {i: {'partition': True if idx % 2 else False}
+      for idx, i in enumerate(table.ids(axis=axis))}
+table.add_metadata(md, axis=axis)
+parts = list(table.partition(lambda i, m: m['partition'], axis=axis))
+collapsed = table.collapse(lambda i, m: m['partition'], axis=axis, norm=False)
+
+assert len(parts) == 2
+assert (len(parts[0][1].ids(axis=axis)) + len(parts[1][1].ids(axis=axis))) == \
+        len(table.ids(axis=axis))
+collapsed_sums = collapsed.sum(axis=axis)
+parts_as_dict = dict(parts)
+for name, obs in zip(collapsed.ids(axis=axis), collapsed_sums):
+    assert parts_as_dict[name].sum() == obs
+
+regrouped = parts[0][1].concat([parts[1][1]], axis=axis)
+regrouped = regrouped.sort_order(table.ids(axis=axis), axis=axis)
+invaxis = 'sample' if axis == 'observation' else 'observation'
+regrouped = regrouped.sort_order(table.ids(axis=invaxis), axis=invaxis)
+assert regrouped == table
+
+regrouped.del_metadata(keys=['partition'], axis=axis)
+assert regrouped != table
+table.del_metadata(keys=['partition'], axis=axis)
+assert regrouped == table
+
+# transforms
+pa = table.pa(inplace=False)
+assert pa.sum() == table.matrix_data.nnz
+
+
+class inc:
+    def __init__(self):
+        self.scaler = 0
+
+    def __call__(self, v, i, md):
+        new_v = v + self.scaler
+        self.scaler += 1
+        return new_v
+
+
+scaled = pa.transform(inc(), inplace=False, axis=axis)
+for idx, v in enumerate(scaled.iter_data(axis=axis, dense=False)):
+    assert np.allclose(v.data, np.ones(1) + idx)
+
+# roundtrips
+_, filepath = tempfile.mkstemp()
+with h5py.File(filepath, 'w') as out:
+    table.to_hdf5(out, 'rc-testing')
+obs = biom.load_table(filepath)
+assert obs == table
+
+with open(filepath, 'w') as out:
+    out.write(table.to_json('rc-testing'))
+obs = biom.load_table(filepath)
+assert obs == table
+
+with open(filepath, 'w') as out:
+    out.write(table.to_tsv())
+obs = biom.load_table(filepath)
+exp = table.copy()
+
+# TSV metadata are a nightmare especially as there are custom parsing
+# is necessary for things like taxonomy.
+exp.del_metadata()
+exp.type = None  # a TSV does not have a table type
+assert obs == exp

biom/assets/exercise_cli.sh

+#!/bin/bash
+set -xe
+
+table=../../examples/min_sparse_otu_table_hdf5.biom
+obsmd=../../examples/obs_md.txt
+if [[ ! -f ${table} ]];
+then
+    echo "This script expects to operate in the biom/assets directory"
+    exit 1
+fi
+
+biom show-install-info
+biom validate-table -i $table
+biom add-metadata -i $table -o test_add_metadata.biom --observation-metadata-fp $obsmd --sc-separated taxonomy
+biom convert -i $table -o test_json.biom --to-json
+biom convert -i $table -o test_tsv.txt --to-tsv
+biom validate-table -i test_json.biom
+biom convert -i test_json.biom -o test_hdf5.biom --to-hdf5
+biom validate-table -i test_hdf5.biom
+biom convert -i test_json.biom -o test_tsv.txt --to-tsv
+biom convert -i test_tsv.txt -o test_hdf5.biom --to-hdf5 --table-type "OTU table"
+biom validate-table -i test_hdf5.biom
+biom convert -i test_tsv.txt -o test_json.biom --to-json --table-type "OTU table"
+biom validate-table -i test_json.biom
+biom head -i $table
+biom table-ids -i $table
+biom normalize-table -i $table -o test_norm.biom -p
+cat << EOF >> ids.txt
+GG_OTU_3
+GG_OTU_5
+EOF
+biom subset-table -i $table -o test_subset.biom -a observation -s ids.txt
+biom summarize-table -i $table
+
+cat << EOF >> uctest
+# uclust --input /var/folders/xq/0kh93ng53bs6zzk091w_bbsr0000gn/T/UclustExactMatchFilterrW47Ju.fasta --id 0.97 --tmpdir /var/folders/xq/0kh93ng53bs6zzk091w_bbsr0000gn/T --w 8 --stepwords 8 --usersort --maxaccepts 1 --stable_sort --maxrejects 8 --uc dn-otus/uclust_picked_otus/seqs_clusters.uc
+# version=1.2.22
+# Tab-separated fields:
+# 1=Type, 2=ClusterNr, 3=SeqLength or ClusterSize, 4=PctId, 5=Strand, 6=QueryStart, 7=SeedStart, 8=Alignment, 9=QueryLabel, 10=TargetLabel
+# Record types (field 1): L=LibSeed, S=NewSeed, H=Hit, R=Reject, D=LibCluster, C=NewCluster, N=NoHit
+# For C and D types, PctId is average id with seed.
+# QueryStart and SeedStart are zero-based relative to start of sequence.
+# If minus strand, SeedStart is relative to reverse-complemented seed.
+S   0   133 *   *   *   *   *   f2_1539 *
+S   0   133 *   *   *   *   *   f3_1540 *
+H   0   141 100.0   +   0   0   133M8D  f3_42   f2_1539
+EOF
+
+biom from-uc -i uctest -o test_uc.biom
+
+echo "If this message was shown, then all commands executed without failure"

biom/cli/__init__.py

 # ----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
@@ -14,10 +14,18 @@ import click
 import biom
 
 
-@click.group()
+def _terribly_handle_brokenpipeerror():
+    # based off http://stackoverflow.com/a/34299346
+    import os
+    import sys
+    sys.stdout = os.fdopen(1, 'w')
+
+
+@click.group(context_settings=dict(help_option_names=['-h', '--help']))
 @click.version_option(version=biom.__version__)
-def cli():
-    pass
+@click.pass_context
+def cli(ctx):
+    ctx.call_on_close(_terribly_handle_brokenpipeerror)
 
 
 import_module('biom.cli.table_summarizer')
@@ -27,5 +35,6 @@ import_module('biom.cli.installation_informer')
 import_module('biom.cli.table_subsetter')
 import_module('biom.cli.table_normalizer')
 import_module('biom.cli.table_head')
+import_module('biom.cli.table_ids')
 import_module('biom.cli.table_validator')
 import_module('biom.cli.uc_processor')

biom/cli/installation_informer.py

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

biom/cli/metadata_adder.py

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

biom/cli/table_converter.py

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

biom/cli/table_head.py

 # ----------------------------------------------------------------------------
-# Copyright (c) 2011-2013, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

biom/cli/table_ids.py

+# -----------------------------------------------------------------------------
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+
+import click
+
+from biom.cli import cli
+from biom import load_table
+
+
+@cli.command(name='table-ids')
+@click.option('-i', '--input-fp', required=True,
+              type=click.Path(exists=True, dir_okay=False),
+              help='The input BIOM table')
+@click.option('--observations', default=False, is_flag=True,
+              help="Grab observation IDs")
+def summarize_table(input_fp, observations):
+    """Dump IDs in a table.
+
+    Dump out the IDs found within a table:
+
+    Example usage:
+
+    Get the sample IDs within a table:
+
+    $ biom table-ids -i table.biom
+
+    Get the observation IDs within a table:
+
+    $ biom table-ids -i table.biom --observations
+    """
+    tab = load_table(input_fp)
+    for id_ in tab.ids(axis='observation' if observations else 'sample'):
+        click.echo(id_)

biom/cli/table_normalizer.py

 #!/usr/bin/env python
 
 # ----------------------------------------------------------------------------
-# Copyright (c) 2011-2013, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #

biom/cli/table_subsetter.py

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
@@ -64,7 +64,10 @@ def subset_table(input_hdf5_fp, input_json_fp, axis, ids, output_fp):
             input_json_fp = f.read()
 
     with open(ids, 'U') as f:
-        ids = [line.strip() for line in f]
+        ids = []
+        for line in f:
+            if not line.startswith('#'):
+                ids.append(line.strip().split('\t')[0])
 
     table, format_ = _subset_table(input_hdf5_fp, input_json_fp, axis, ids)
 

biom/cli/table_summarizer.py

 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
@@ -9,6 +9,7 @@
 from __future__ import division
 
 from operator import itemgetter
+import locale
 
 import click
 from numpy import std
@@ -55,6 +56,7 @@ def summarize_table(input_fp, output_fp, qualitative, observations):
 
 def _summarize_table(table, qualitative=False, observations=False):
     lines = []
+    locale.setlocale(locale.LC_ALL, '')
 
     if observations:
         table = table.transpose()
@@ -79,15 +81,20 @@ def _summarize_table(table, qualitative=False, observations=False):
 
     if observations:
         # as this is a transpose of the original table...
-        lines.append('Num samples: %d' % num_observations)
-        lines.append('Num observations: %d' % num_samples)
+        lines.append('Num samples: ' + locale.format('%d', num_observations,
+                                                     grouping=True))
+        lines.append('Num observations: ' + locale.format('%d', num_samples,
+                                                          grouping=True))
     else:
-        lines.append('Num samples: %d' % num_samples)
-        lines.append('Num observations: %d' % num_observations)
+        lines.append('Num samples: ' + locale.format('%d', num_samples,
+                                                     grouping=True))
+        lines.append('Num observations: ' + locale.format('%d',
+                     num_observations, grouping=True))
 
     if not qualitative:
         total_count = sum(counts_per_sample_values)
-        lines.append('Total count: %d' % total_count)
+        lines.append('Total count: ' + locale.format('%d', total_count,
+                                                     grouping=True))
         lines.append('Table density (fraction of non-zero values): %1.3f' %
                      table.get_table_density())
 
@@ -101,11 +108,14 @@ def _summarize_table(table, qualitative=False, observations=False):
     else:
         lines.append('Counts/sample summary:')
 
-    lines.append(' Min: %r' % min_counts)
-    lines.append(' Max: %r' % max_counts)
-    lines.append(' Median: %1.3f' % median_counts)
-    lines.append(' Mean: %1.3f' % mean_counts)
-    lines.append(' Std. dev.: %1.3f' % std(counts_per_sample_values))
+    lines.append(' Min: ' + locale.format('%1.3f', min_counts, grouping=True))
+    lines.append(' Max: ' + locale.format('%1.3f', max_counts, grouping=True))
+    lines.append(' Median: ' + locale.format('%1.3f', median_counts,
+                                             grouping=True))
+    lines.append(' Mean: ' + locale.format('%1.3f', mean_counts,
+                                           grouping=True))
+    lines.append(' Std. dev.: ' + locale.format('%1.3f',
+                 std(counts_per_sample_values), grouping=True))
 
     if observations:
         # since this is a transpose...
@@ -131,6 +141,6 @@ def _summarize_table(table, qualitative=False, observations=False):
         lines.append('Counts/sample detail:')
 
     for k, v in sorted(counts_per_samp.items(), key=itemgetter(1)):
-        lines.append('%s: %r' % (k, v))
+        lines.append('%s: ' % k + locale.format('%1.3f', v, grouping=True))
 
     return "\n".join(lines)

biom/cli/table_validator.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # -----------------------------------------------------------------------------
-# Copyright (c) 2011-2013, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
@@ -82,10 +82,10 @@ class TableValidator(object):
                 kwargs['format_version'] = '1.0.0'
             else:
                 kwargs['format_version'] = '2.1'
-        else:
-            if is_json:
+        elif is_json:
+            if kwargs['format_version'] != "1.0.0":
                 raise ValueError("Only format 1.0.0 is valid for JSON")
-
+        else:
             fmt_ver = [int(v) for v in kwargs['format_version'].split('.')]
             if tuple(fmt_ver) not in self.HDF5FormatVersions:
                 raise ValueError("Unrecognized format version: %s" %
@@ -387,7 +387,7 @@ class TableValidator(object):
         else:
             return ''
 
-    def _check_date(self, val):
+    def _valid_date(self, val):
         valid_times = ["%Y-%m-%d",
                        "%Y-%m-%dT%H:%M",
                        "%Y-%m-%dT%H:%M:%S",
@@ -416,7 +416,7 @@ class TableValidator(object):
                                           note that a 'T' separates the date
                                           and time)
         """
-        return self._check_date(table.attrs['creation-date'])
+        return self._valid_date(table.attrs['creation-date'])
 
     def _valid_datetime(self, table):
         """Verify datetime can be parsed
@@ -425,7 +425,7 @@ class TableValidator(object):
                                           note that a 'T' separates the date
                                           and time)
         """
-        return self._check_date(table['date'])
+        return self._valid_date(table['date'])
 
     def _valid_sparse_data(self, table_json):
         """All index positions must be integers and values are of dtype"""
@@ -493,6 +493,8 @@ class TableValidator(object):
         """Table must be a known table type"""
         key = self._json_or_hdf5_key(table, 'type')
         value = self._json_or_hdf5_get(table, key)
+        if value is None or value == "":
+            return "Unknown table type, however that is likely okay."
         if value.lower() not in self.TableTypes:
             return "Unknown BIOM type: %s" % value
         else:
@@ -534,8 +536,11 @@ class TableValidator(object):
         required_keys = [('id', self._valid_id),
                          ('metadata', self._valid_metadata)]
         required_by_type = {}
+        ttype = table_json.get('type')
+        if ttype is None:
+            ttype = ""
         required_keys.extend(
-            required_by_type.get(table_json['type'].lower(), []))
+            required_by_type.get(ttype.lower(), []))
 
         for idx, row in enumerate(table_json['rows']):
             for key, method in required_keys:
@@ -552,8 +557,11 @@ class TableValidator(object):
         required_keys = [('id', self._valid_id),
                          ('metadata', self._valid_metadata)]
         required_by_type = {}
+        ttype = table_json.get('type')
+        if ttype is None:
+            ttype = ""
         required_keys.extend(
-            required_by_type.get(table_json['type'].lower(), []))
+            required_by_type.get(ttype.lower(), []))
 
         for idx, col in enumerate(table_json['columns']):
             for key, method in required_keys:

biom/cli/uc_processor.py

 # ----------------------------------------------------------------------------
-# Copyright (c) 2011-2013, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
@@ -20,9 +20,9 @@ from biom.exception import TableException
 @click.option('-i', '--input-fp', required=True,
               type=click.Path(exists=True, dir_okay=False),
               help='The input uc filepath.')
-@click.option('-o', '--output-fp', default=None,
+@click.option('-o', '--output-fp', required=True,
               type=click.Path(writable=True),
-              help='The output BIOM filepath', required=False)
+              help='The output BIOM filepath')
 @click.option('--rep-set-fp', type=click.Path(exists=True, dir_okay=False),
               help="Fasta file containing representative sequences with "
                    "where sequences are labeled with OTU identifiers, and "

biom/cli/util.py

 # ----------------------------------------------------------------------------
-# Copyright (c) 2011-2015, The BIOM Format Development Team.
+# Copyright (c) 2011-2017, The BIOM Format Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #