.travis.yml

 language: python
 sudo: false
 python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
+- '2.7'
+- '3.4'
+- '3.5'
+- '3.6'
 services:
 - redis
-# Apply only on main branches
 branches:
   except:
-    - /^feature.*$/
-# command to install dependencies
-#before_install:
-#  - "sudo apt-get update -qq"
-#  - "sudo apt-get install -qq libldap2-dev libsasl2-dev"
+  - "/^feature.*$/"
 install:
-  - "pip install -r requirements.txt"
-  - "pip install coverage"
-  - "pip install python-coveralls"
-  - "python setup.py -q install"
-#  - "echo data_file=$TRAVIS_BUILD_DIR/.coverage >> .coveragerc"
-# command to run tests
+- pip install -r requirements.txt
+- pip install coverage nose
+- pip install python-coveralls
+- python setup.py -q install
 script:
-   - python setup.py test
+- nosetests -a '!network'
 - flake8 --ignore E501 biomaj_download/*.py biomaj_download/download
-#after_success:
-#  - coveralls
-
+deploy:
+  provider: pypi
+  skip_existing: true
+  user: osallou
+  password:
+    secure: fMEDYld7N2uRbICa++b7dTUXxt6VxSD1ZE+ZaTHQWxHHSK5fxbdwvsVHicdgKH3erMOXb7foumQ6uxBAtQTVmY+ZJbidydT/BEeoLjIH0NxVoTLCOllWKHOKm8LzgigVjbSVqXi7ZZ6rzigpQbg7NCdiD4wQr6xLO+gB3LrWovIgcPIlWDLWiMIp0P3b9nmoOdiSBBpLXI7i9W7ksL25Mopx/p4OoeWFEbg7aSkRVKufWLCoyLnYnWCnmC0MK7PF7V7mgiw+Gutv3XcOgfaZ/MU088+OrpvnLxfKplkwukcMBc4cuMV8mtWmDaQ9u2X9MLoeyRoQ6jHyIvftW+fn4Ck3wzyKZA3Pbz5+UVUG6GVpnwm3NIYvPl0m62brVVqs/wVaI8HB/7u5qn60ER9u9l9/YGIG5rsPrb2w12QFmcjwrge2T9h+aDW3+fQuzbjVGJMBCLqu9hUoiz9WHWqa74Ka8OfP+fTA3vpASGfuYhzdlJ5tDO6M1kjIe23ga5DhUMKHyl4j7uA/YVxc5numYU9naD/PyZw1qqms8zhJdVmIsBMtBP/MeH0YVrhmQS7p4ni7BB545JBYv8UyUfVXt2DcQHG9EAkMT69QfJhSl9KICe4SmlHVYy2Uldw/LE/FwYmngfDzZUYtndsn2xS+NVi3FwMNa1+6q0Sqqmzfmqc=
+  distributions: sdist bdist_wheel
+  on:
+    tags: true

CHANGES.txt

+3.0.21:
+  Fix traefik labels
+3.0.20:
+  Update pika dependency release
+  Add tags for traefik support
 3.0.19:
   Check archives after download
   Fix python regexps syntax (deprecation)

biomaj_download/biomaj_download_web.py

@@ -41,7 +41,19 @@ with open(config_file, 'r') as ymlfile:
 def consul_declare(config):
     if config['consul']['host']:
         consul_agent = consul.Consul(host=config['consul']['host'])
-        consul_agent.agent.service.register('biomaj-download', service_id=config['consul']['id'], address=config['web']['hostname'], port=config['web']['port'], tags=['biomaj'])
+        consul_agent.agent.service.register(
+            'biomaj-download',
+            service_id=config['consul']['id'],
+            address=config['web']['hostname'],
+            port=config['web']['port'],
+            tags=[
+                'biomaj',
+                'api',
+                'traefik-int.backend=biomaj-download',
+                'traefik-int.frontend.rule=PathPrefix:/api/download',
+                'traefik-int.enable=true'
+            ]
+        )
         check = consul.Check.http(url='http://' + config['web']['hostname'] + ':' + str(config['web']['port']) + '/api/download', interval=20)
         consul_agent.agent.check.register(config['consul']['id'] + '_check', check=check, service_id=config['consul']['id'])
 

debian/changelog

+biomaj3-download (3.0.21-1) unstable; urgency=medium
+
+  * New upstream release 
+  * d/rules: use DEB_BUILD_MAINT_OPTIONS
+  * d/patches: remove irods protocol support as python lib is not
+    available in Debian.
+
+ -- Olivier Sallou <osallou@debian.org>  Sat, 09 Mar 2019 09:48:16 +0000
+
 biomaj3-download (3.0.19-1) unstable; urgency=medium
 
   * Team upload.

debian/patches/remove_irods.patch

+Subject: python irods not available, remove it from supported protocols
+Description: biomaj supports irods as download protocol but irods is not
+ available in Debian. In the meanwhile remove support for this protocol
+Author: Olivier Sallou <osallou@debian.org>
+Last-Updated: 2019-03-09
+Forwarded: no
+--- a/requirements.txt
++++ b/requirements.txt
+@@ -14,4 +14,3 @@
+ biomaj_zipkin
+ flake8
+ humanfriendly
+-python-irodsclient
+--- a/setup.py
++++ b/setup.py
+@@ -54,8 +54,7 @@
+                          'prometheus_client>=0.0.18',
+                          'protobuf',
+                          'requests',
+-                         'humanfriendly',
+-                         'python-irodsclient'
++                         'humanfriendly'
+                         ],
+     'tests_require': ['nose', 'mock'],
+     'test_suite': 'nose.collector',
+--- a/biomaj_download/download/protocolirods.py
++++ b/biomaj_download/download/protocolirods.py
+@@ -5,8 +5,6 @@
+ 
+ from biomaj_core.utils import Utils
+ from biomaj_download.download.interface import DownloadInterface
+-from irods.session import iRODSSession
+-from irods.models import Collection, DataObject, User
+ 
+ 
+ class IRODSDownload(DownloadInterface):
+@@ -31,27 +29,9 @@
+         self.zone = str(param['zone'])
+ 
+     def list(self, directory=''):
+-        session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
+         rfiles = []
+         rdirs = []
+-        rfile = {}
+-        date = None
+-        for result in session.query(Collection.name, DataObject.name, DataObject.size, DataObject.owner_name, DataObject.modify_time).filter(User.name == self.user).get_results():
+-            # if the user is biomaj : he will have access to all the irods data (biomaj ressource) : drwxr-xr-x
+-            # Avoid duplication
+-            if rfile != {} and rfile['name'] == str(result[DataObject.name]) and date == str(result[DataObject.modify_time]).split(" ")[0].split('-'):
+-                continue
+-            rfile = {}
+-            date = str(result[DataObject.modify_time]).split(" ")[0].split('-')
+-            rfile['permissions'] = "-rwxr-xr-x"
+-            rfile['size'] = int(result[DataObject.size])
+-            rfile['month'] = int(date[1])
+-            rfile['day'] = int(date[2])
+-            rfile['year'] = int(date[0])
+-            rfile['name'] = str(result[DataObject.name])
+-            rfile['download_path'] = str(result[Collection.name])
+-            rfiles.append(rfile)
+-        session.cleanup()
++        raise Exception("IRODS:NotSupported")
+         return (rfiles, rdirs)
+ 
+     def download(self, local_dir, keep_dirs=True):
+@@ -65,67 +45,10 @@
+         :return: list of downloaded files
+         '''
+         logging.debug('IRODS:Download')
+-        try:
+-            os.chdir(local_dir)
+-        except TypeError:
+-            logging.error("IRODS:list:Could not find offline_dir")
+-        nb_files = len(self.files_to_download)
+-        cur_files = 1
+-        # give a working directory to copy the file from irods
+-        remote_dir = self.remote_dir
+-        for rfile in self.files_to_download:
+-            if self.kill_received:
+-                raise Exception('Kill request received, exiting')
+-            file_dir = local_dir
+-            if 'save_as' not in rfile or rfile['save_as'] is None:
+-                rfile['save_as'] = rfile['name']
+-            if keep_dirs:
+-                file_dir = local_dir + os.path.dirname(rfile['save_as'])
+-            file_path = file_dir + '/' + os.path.basename(rfile['save_as'])
+-            # For unit tests only, workflow will take in charge directory creation before to avoid thread multi access
+-            if not os.path.exists(file_dir):
+-                os.makedirs(file_dir)
+-
+-            logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' downloading file ' + rfile['name'])
+-            logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' save as ' + rfile['save_as'])
+-            cur_files += 1
+-            start_time = datetime.now()
+-            start_time = time.mktime(start_time.timetuple())
+-            self.remote_dir = rfile['root']
+-            error = self.irods_download(file_dir, str(self.remote_dir), str(rfile['name']))
+-            if error:
+-                rfile['download_time'] = 0
+-                rfile['error'] = True
+-                raise Exception("IRODS:Download:Error:" + rfile['root'] + '/' + rfile['name'])
+-            else:
+-                archive_status = Utils.archive_check(file_path)
+-                if not archive_status:
+-                    self.logger.error('Archive is invalid or corrupted, deleting file')
+-                    rfile['error'] = True
+-                    if os.path.exists(file_path):
+-                        os.remove(file_path)
+-                    raise Exception("IRODS:Download:Error:" + rfile['root'] + '/' + rfile['name'])
+-
+-            end_time = datetime.now()
+-            end_time = time.mktime(end_time.timetuple())
+-            rfile['download_time'] = end_time - start_time
+-            self.set_permissions(file_path, rfile)
+-        self.remote_dir = remote_dir
+-        return(self.files_to_download)
++        raise Exception("IRODS:NotSupported")
+ 
+     def irods_download(self, file_dir, file_path, file_to_download):
+-        error = False
+-        logging.debug('IRODS:IRODS DOWNLOAD')
+-        session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
+-        try:
+-            file_to_get = str(file_path) + str(file_to_download)
+-            # Write the file to download in the wanted file_dir : with the python-irods iget
+-            obj = session.data_objects.get(file_to_get, file_dir)
+-        except ExceptionIRODS as e:
+-            logging.error("RsyncError:" + str(e))
+-            logging.error("RsyncError: irods object" + str(obj))
+-        session.cleanup()
+-        return(error)
++        return("irods not supported")
+ 
+ 
+ class ExceptionIRODS(Exception):

debian/patches/series

+remove_irods.patch

requirements.txt

@@ -2,7 +2,7 @@ mock
 nose
 pycurl
 py-bcrypt
-pika==0.11.2
+pika==0.13.0
 redis
 PyYAML
 protobuf
@@ -10,7 +10,7 @@ flask
 python-consul
 prometheus_client>=0.0.18
 requests
-biomaj_core>=3.0.12
+biomaj_core>=3.0.16
 biomaj_zipkin
 flake8
 humanfriendly

setup.py

@@ -21,7 +21,7 @@ config = {
     'url': 'http://biomaj.genouest.org',
     'download_url': 'http://biomaj.genouest.org',
     'author_email': 'olivier.sallou@irisa.fr',
-    'version': '3.0.19',
+    'version': '3.0.21',
      'classifiers': [
         # How mature is this project? Common values are
         #   3 - Alpha
@@ -46,7 +46,7 @@ config = {
                          'biomaj_zipkin',
                          'pycurl',
                          'py-bcrypt',
-                         'pika==0.11.2',
+                         'pika==0.13.0',
                          'redis',
                          'PyYAML',
                          'flask',

tests/biomaj_tests.py

@@ -320,21 +320,21 @@ class TestBiomajDirectFTPDownload(unittest.TestCase):
     self.utils.clean()
 
   def test_ftp_list(self):
-    file_list = ['/blast/db/FASTA/alu.n.gz.md5']
-    ftpd = DirectFTPDownload('ftp', 'ftp.ncbi.nih.gov', '')
+    file_list = ['/debian/doc/mailing-lists.txt']
+    ftpd = DirectFTPDownload('ftp', 'ftp.fr.debian.org', '')
     ftpd.set_files_to_download(file_list)
     (file_list, dir_list) = ftpd.list()
     ftpd.close()
     self.assertTrue(len(file_list) == 1)
 
   def test_download(self):
-    file_list = ['/blast/db/FASTA/alu.n.gz.md5']
-    ftpd = DirectFTPDownload('ftp', 'ftp.ncbi.nih.gov', '')
+    file_list = ['/debian/doc/mailing-lists.txt']
+    ftpd = DirectFTPDownload('ftp', 'ftp.fr.debian.org', '')
     ftpd.set_files_to_download(file_list)
     (file_list, dir_list) = ftpd.list()
     ftpd.download(self.utils.data_dir, False)
     ftpd.close()
-    self.assertTrue(os.path.exists(os.path.join(self.utils.data_dir,'alu.n.gz.md5')))
+    self.assertTrue(os.path.exists(os.path.join(self.utils.data_dir,'mailing-lists.txt')))
 
 
 @attr('directhttp')
@@ -431,17 +431,14 @@ class TestBiomajFTPDownload(unittest.TestCase):
 
   def test_ftp_list(self):
     ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
-    # ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
     (file_list, dir_list) = ftpd.list()
     ftpd.close()
     self.assertTrue(len(file_list) > 1)
 
   @attr('test')
   def test_download(self):
-    # ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
     ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
     (file_list, dir_list) = ftpd.list()
-    # ftpd.match([r'^alu.*\.gz$'], file_list, dir_list)
     ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
     try:
         ftpd.download(self.utils.data_dir)
@@ -453,11 +450,9 @@ class TestBiomajFTPDownload(unittest.TestCase):
     # self.assertTrue(len(ftpd.files_to_download) == 2)
 
   def test_download_skip_uncompress_checks(self):
-    # ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/db/FASTA/')
     os.environ['UNCOMPRESS_SKIP_CHECK'] = "1"
     ftpd = FTPDownload('ftp', 'speedtest.tele2.net', '/')
     (file_list, dir_list) = ftpd.list()
-    # ftpd.match([r'^alu.*\.gz$'], file_list, dir_list)
     ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
     ftpd.download(self.utils.data_dir)
     ftpd.close()
@@ -465,19 +460,19 @@ class TestBiomajFTPDownload(unittest.TestCase):
     del os.environ['UNCOMPRESS_SKIP_CHECK']
 
   def test_download_in_subdir(self):
-    ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
+    ftpd = FTPDownload('ftp', 'ftp.fr.debian.org', '/debian/')
     (file_list, dir_list) = ftpd.list()
     try:
-        ftpd.match([r'^db/FASTA/alu.*\.gz$'], file_list, dir_list)
+        ftpd.match([r'^doc/mailing-lists.txt$'], file_list, dir_list)
     except Exception as e:
         print("Error: " + str(e))
         self.skipTest("Skipping test due to remote server error")
     ftpd.download(self.utils.data_dir)
     ftpd.close()
-    self.assertTrue(len(ftpd.files_to_download) == 2)
+    self.assertTrue(len(ftpd.files_to_download) == 1)
 
   def test_download_or_copy(self):
-    ftpd = FTPDownload('ftp', 'ftp.ncbi.nih.gov', '/blast/')
+    ftpd = FTPDownload('ftp', 'ftp.fr.debian.org', '/debian/')
     ftpd.files_to_download = [
           {'name':'/test1', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},
           {'name':'/test2', 'year': '2013', 'month': '11', 'day': '10', 'size': 10},