ci/recipe/meta.yaml

@@ -21,8 +21,6 @@ requirements:
     - scikit-bio >=0.5.4
     - numpy
     - blas=*=openblas
-    # AVX 512 extensions are broken in 0.3.5 and 0.3.6
-    - openblas 0.3.3
     - pandas
     - biom-format >=2.1.5,<2.2.0
     - ijson

debian/changelog

+q2-types (2019.7.0-1) unstable; urgency=medium
+
+  * Team upload.
+  * New upstream version
+  * debhelper-compat 12
+  * Standards-Version: 4.4.0
+  * Respect DEB_BUILD_OPTIONS in override_dh_auto_test target
+
+ -- Andreas Tille <tille@debian.org>  Thu, 05 Sep 2019 10:38:27 +0200
+
 q2-types (2019.4.1-1) unstable; urgency=medium
 
   * New upstream version

debian/compat

-12

debian/control

 Source: q2-types
-Section: science
-Priority: optional
 Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.org>
 Uploaders: Liubov Chuprikova <chuprikovalv@gmail.com>
-Build-Depends: debhelper (>= 12~),
+Section: science
+Priority: optional
+Build-Depends: debhelper-compat (= 12),
                dh-python,
                python3,
                python3-setuptools,
@@ -13,7 +13,7 @@ Build-Depends: debhelper (>= 12~),
                python3-ijson,
                python3-h5py,
                qiime
-Standards-Version: 4.3.0
+Standards-Version: 4.4.0
 Vcs-Browser: https://salsa.debian.org/med-team/q2-types
 Vcs-Git: https://salsa.debian.org/med-team/q2-types.git
 Homepage: https://qiime2.org/

debian/rules

@@ -14,7 +14,9 @@ export PYBUILD_BEFORE_INSTALL=rm -rvf {build_dir}/q2-types.egg-* {build_dir}/sit
 	dh $@ --with python3 --buildsystem=pybuild
 
 override_dh_auto_test:
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
 	dh_auto_test -- -s custom --test-args="cd {build_dir}; py.test-3 --cov=q2_types"
+endif
 
 override_dh_install:
 	dh_install

q2_types/_version.py

@@ -23,9 +23,9 @@ def get_keywords():
     # setup.py/versioneer.py will grep for the variable names, so they must
     # each be defined on a line of their own. _version.py will just call
     # get_keywords().
-    git_refnames = " (tag: 2019.4.1)"
-    git_full = "39051b2475fccf77526778589b67efcbf00314cf"
-    git_date = "2019-05-08 16:49:19 -0700"
+    git_refnames = " (tag: 2019.7.0)"
+    git_full = "9f31de0c81510fbe6be8b16f95e23b4c974ca002"
+    git_date = "2019-07-30 18:15:54 +0000"
     keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
     return keywords
 

q2_types/feature_data/__init__.py

@@ -13,9 +13,11 @@ from ._format import (
     HeaderlessTSVTaxonomyDirectoryFormat, TSVTaxonomyFormat,
     TSVTaxonomyDirectoryFormat, DNAFASTAFormat, DNASequencesDirectoryFormat,
     PairedDNASequencesDirectoryFormat, AlignedDNAFASTAFormat,
-    AlignedDNASequencesDirectoryFormat)
+    AlignedDNASequencesDirectoryFormat, DifferentialFormat,
+    DifferentialDirectoryFormat)
 from ._type import (
-    FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence)
+    FeatureData, Taxonomy, Sequence, PairedEndSequence, AlignedSequence,
+    Differential)
 
 # TODO remove these imports when tests are rewritten. Remove from __all__ too
 from ._transformer import DNAIterator, PairedDNAIterator, AlignedDNAIterator
@@ -23,11 +25,11 @@ from ._transformer import DNAIterator, PairedDNAIterator, AlignedDNAIterator
 __all__ = [
     'TaxonomyFormat', 'TaxonomyDirectoryFormat', 'HeaderlessTSVTaxonomyFormat',
     'HeaderlessTSVTaxonomyDirectoryFormat', 'TSVTaxonomyFormat',
-    'TSVTaxonomyDirectoryFormat', 'DNAFASTAFormat',
+    'TSVTaxonomyDirectoryFormat', 'DNAFASTAFormat', 'DifferentialFormat',
     'DNASequencesDirectoryFormat', 'PairedDNASequencesDirectoryFormat',
     'AlignedDNAFASTAFormat', 'AlignedDNASequencesDirectoryFormat',
     'FeatureData', 'Taxonomy', 'Sequence', 'PairedEndSequence',
     'AlignedSequence', 'DNAIterator', 'PairedDNAIterator',
-    'AlignedDNAIterator']
+    'AlignedDNAIterator', 'Differential', 'DifferentialDirectoryFormat']
 
 importlib.import_module('q2_types.feature_data._transformer')

q2_types/feature_data/_format.py

@@ -6,8 +6,12 @@
 # The full license is in the file LICENSE, distributed with this software.
 # ----------------------------------------------------------------------------
 
+import re
 import skbio.io
+
 import qiime2.plugin.model as model
+from qiime2.plugin import ValidationError
+import qiime2
 
 from ..plugin_setup import plugin
 
@@ -131,23 +135,47 @@ TSVTaxonomyDirectoryFormat = model.SingleFileDirectoryFormat(
 
 
 class DNAFASTAFormat(model.TextFileFormat):
-    def sniff(self):
-        filepath = str(self)
-        sniffer = skbio.io.io_registry.get_sniffer('fasta')
-        if sniffer(filepath)[0]:
-            generator = skbio.io.read(filepath, constructor=skbio.DNA,
-                                      format='fasta', verify=False)
+    def _validate_lines(self, max_lines):
+        FASTADNAValidator = re.compile(r'[ACGTURYKMSWBDHVN]+\r?\n?')
+        last_line_was_ID = False
+
+        with open(str(self), 'rb') as fh:
             try:
-                for seq, _ in zip(generator, range(5)):
-                    pass
-                return True
-            # ValueError raised by skbio if there are invalid DNA chars.
-            except ValueError:
-                pass
+                first = fh.read(6)
+                if first[:3] == b'\xEF\xBB\xBF':
+                    first = first[3:]
+                # Empty files should validate
+                if first.strip() == b'':
+                    return
+                if first[0] != ord(b'>'):
+                    raise ValidationError("First line of file is not a valid "
+                                          "FASTA ID. FASTA IDs must start "
+                                          "with '>'")
+                fh.seek(0)
+                for line_number, line in enumerate(fh, 1):
+                    if line_number >= max_lines:
+                        return
+                    line = line.decode('utf-8-sig')
+                    if line.startswith('>'):
+                        if last_line_was_ID:
+                            raise ValidationError('Multiple consecutive IDs '
+                                                  'starting on line '
+                                                  f'{line_number-1!r}')
+                        last_line_was_ID = True
+                    elif re.fullmatch(FASTADNAValidator, line):
+                        last_line_was_ID = False
+                    else:
+                        raise ValidationError('Invalid characters on line '
+                                              f'{line_number} (does not match '
+                                              'IUPAC characters for a DNA '
+                                              'sequence).')
+            except UnicodeDecodeError as e:
+                raise ValidationError(f'utf-8 cannot decode byte on line '
+                                      f'{line_number}') from e
 
-        # Empty files are ok also
-        empty_sniffer = skbio.io.io_registry.get_sniffer('<emptyfile>')
-        return empty_sniffer(filepath)[0]
+    def _validate_(self, max_lines):
+        level_map = {'min': 100, 'max': float('inf')}
+        self._validate_lines(level_map[max_lines])
 
 
 DNASequencesDirectoryFormat = model.SingleFileDirectoryFormat(
@@ -185,10 +213,30 @@ AlignedDNASequencesDirectoryFormat = model.SingleFileDirectoryFormat(
     AlignedDNAFASTAFormat)
 
 
+class DifferentialFormat(model.TextFileFormat):
+    def validate(self, *args):
+        try:
+            md = qiime2.Metadata.load(str(self))
+        except qiime2.metadata.MetadataFileError as md_exc:
+            raise ValidationError(md_exc) from md_exc
+
+        if md.column_count == 0:
+            raise ValidationError('Format must contain at least 1 column')
+
+        filtered_md = md.filter_columns(column_type='numeric')
+        if filtered_md.column_count != md.column_count:
+            raise ValidationError('Must only contain numeric values.')
+
+
+DifferentialDirectoryFormat = model.SingleFileDirectoryFormat(
+    'DifferentialDirectoryFormat', 'differentials.tsv', DifferentialFormat)
+
+
 plugin.register_formats(
     TSVTaxonomyFormat, TSVTaxonomyDirectoryFormat,
     HeaderlessTSVTaxonomyFormat, HeaderlessTSVTaxonomyDirectoryFormat,
     TaxonomyFormat, TaxonomyDirectoryFormat, DNAFASTAFormat,
     DNASequencesDirectoryFormat, PairedDNASequencesDirectoryFormat,
-    AlignedDNAFASTAFormat, AlignedDNASequencesDirectoryFormat
+    AlignedDNAFASTAFormat, AlignedDNASequencesDirectoryFormat,
+    DifferentialFormat, DifferentialDirectoryFormat
 )

q2_types/feature_data/_transformer.py

@@ -18,7 +18,7 @@ from ..plugin_setup import plugin
 from ..feature_table import BIOMV210Format
 from . import (TaxonomyFormat, HeaderlessTSVTaxonomyFormat, TSVTaxonomyFormat,
                DNAFASTAFormat, PairedDNASequencesDirectoryFormat,
-               AlignedDNAFASTAFormat)
+               AlignedDNAFASTAFormat, DifferentialFormat)
 
 
 # Taxonomy format transformers
@@ -359,3 +359,21 @@ def _19(data: AlignedDNAIterator) -> AlignedDNAFASTAFormat:
 @plugin.register_transformer
 def _33(ff: AlignedDNAFASTAFormat) -> qiime2.Metadata:
     return _dnafastaformats_to_metadata(ff)
+
+
+# differential types
+@plugin.register_transformer
+def _222(ff: DifferentialFormat) -> pd.DataFrame:
+    return qiime2.Metadata.load(str(ff)).to_dataframe()
+
+
+@plugin.register_transformer
+def _223(ff: DifferentialFormat) -> qiime2.Metadata:
+    return qiime2.Metadata.load(str(ff))
+
+
+@plugin.register_transformer
+def _224(data: pd.DataFrame) -> DifferentialFormat:
+    ff = DifferentialFormat()
+    qiime2.Metadata(data).save(str(ff))
+    return ff

q2_types/feature_data/_type.py

@@ -11,7 +11,8 @@ from qiime2.plugin import SemanticType
 from ..plugin_setup import plugin
 from . import (TSVTaxonomyDirectoryFormat, DNASequencesDirectoryFormat,
                PairedDNASequencesDirectoryFormat,
-               AlignedDNASequencesDirectoryFormat)
+               AlignedDNASequencesDirectoryFormat,
+               DifferentialDirectoryFormat)
 
 
 FeatureData = SemanticType('FeatureData', field_names='type')
@@ -26,8 +27,13 @@ PairedEndSequence = SemanticType('PairedEndSequence',
 AlignedSequence = SemanticType('AlignedSequence',
                                variant_of=FeatureData.field['type'])
 
+Differential = SemanticType('Differential',
+                            variant_of=FeatureData.field['type'])
+
 plugin.register_semantic_types(FeatureData, Taxonomy, Sequence,
-                               PairedEndSequence, AlignedSequence)
+                               PairedEndSequence, AlignedSequence,
+                               Differential)
+
 
 plugin.register_semantic_type_to_format(
     FeatureData[Taxonomy],
@@ -41,3 +47,5 @@ plugin.register_semantic_type_to_format(
 plugin.register_semantic_type_to_format(
     FeatureData[AlignedSequence],
     artifact_format=AlignedDNASequencesDirectoryFormat)
+plugin.register_semantic_type_to_format(
+    FeatureData[Differential], DifferentialDirectoryFormat)

q2_types/feature_data/tests/data/bad_differential.tsv

+featureid	effect	bad_effect
+F0	-0.910182258821735	a
+F1	1.01418002973925	0
+F2	1.02456128258909	0
+F3	-0.74363992043225	0
+F4	1.29823896534823	0
+F5	-1.12965055281585	0
+F6	-0.522401797448688	0
+F7	0.327560711072239	0
+F8	-1.3738693465664802	0
+F9	-0.7847891526325621	0
+F10	-0.280063201878434	0
+F11	-0.251269847578052	0
+F12	1.24602780723028	0
+F13	0.665734866338239	0
+F14	-0.889042985114811	0
+F15	-0.811956802515126	0
+F16	0.11591764582945001	0
+F17	-0.156195990858492	0
+F18	-0.965770633683909	0
+F19	0.8035240337800391	0
+F20	0.680306950765235	0
+F21	-0.688388077896823	0
+F22	0.7949653982850671	0
+F23	-1.11912925367142	0
+F24	-1.1059356352774599	0
+F25	0.678934047810573	0
+F26	-0.937189288219405	0
+F27	-0.9997301931164799	0
+F28	-0.31799033232181	0
+F29	-0.66141741897569	0
+F30	0.550511528315366	0

q2_types/feature_data/tests/data/differentials.tsv

+featureid	effect
+F0	-0.910182258821735
+F1	1.01418002973925
+F2	1.02456128258909
+F3	-0.74363992043225
+F4	1.29823896534823
+F5	-1.12965055281585
+F6	-0.522401797448688
+F7	0.327560711072239
+F8	-1.3738693465664802
+F9	-0.7847891526325621
+F10	-0.280063201878434
+F11	-0.251269847578052
+F12	1.24602780723028
+F13	0.665734866338239
+F14	-0.889042985114811
+F15	-0.811956802515126
+F16	0.11591764582945001
+F17	-0.156195990858492
+F18	-0.965770633683909
+F19	0.8035240337800391
+F20	0.680306950765235
+F21	-0.688388077896823
+F22	0.7949653982850671
+F23	-1.11912925367142
+F24	-1.1059356352774599
+F25	0.678934047810573
+F26	-0.937189288219405
+F27	-0.9997301931164799
+F28	-0.31799033232181
+F29	-0.66141741897569
+F30	0.550511528315366

q2_types/feature_data/tests/data/dna-sequences-consecutive-ids.fasta

+>This is an ID
+>This is another ID

q2_types/feature_data/tests/data/dna-sequences-corrupt-characters.fasta

+>This data is corrupt
+
\ No newline at end of file

q2_types/feature_data/tests/data/dna-sequences-first-line-not-id.fasta

+This is not an id

q2_types/feature_data/tests/data/dna-with-bom-fails.fasta

+Not a valid id

q2_types/feature_data/tests/data/dna-with-bom-passes.fasta

+>Some kinda DNA
+ACGTACGTACGT

q2_types/feature_data/tests/data/empty_differential.tsv

+featureid
+F0
+F1
+F2
+F3
+F4
+F5
+F6
+F7
+F8
+F9
+F10
+F11
+F12
+F13
+F14
+F15
+F16
+F17
+F18
+F19
+F20
+F21
+F22
+F23
+F24
+F25
+F26
+F27
+F28
+F29
+F30

q2_types/feature_data/tests/data/inf_differential.tsv

+featureid	effect
+F0	inf
+F1	1.01418002973925
+F2	1.02456128258909
+F3	-0.74363992043225
+F4	1.29823896534823
+F5	-1.12965055281585
+F6	-0.522401797448688
+F7	0.327560711072239
+F8	-1.3738693465664802
+F9	-0.7847891526325621
+F10	-0.280063201878434
+F11	-0.251269847578052
+F12	1.24602780723028
+F13	0.665734866338239
+F14	-0.889042985114811
+F15	-0.811956802515126
+F16	0.11591764582945
+F17	-0.156195990858492
+F18	-0.965770633683909
+F19	0.8035240337800391
+F20	0.680306950765235
+F21	-0.688388077896823
+F22	0.7949653982850671
+F23	-1.11912925367142
+F24	-1.10593563527746
+F25	0.678934047810573
+F26	-0.937189288219405
+F27	-0.99973019311648
+F28	-0.31799033232181
+F29	-0.66141741897569
+F30	0.550511528315366

q2_types/feature_data/tests/data/not_differential.tsv

+asdfasdfasdfasdf asdfasdfa asdfasdf asdf asdfas dfa
\ No newline at end of file