New upstream version 3.0.27 (86037752) · Commits · Debian Med / biomaj3-download

.travis.yml

+4 −0

Original line number	Diff line number	Diff line
		@@ -10,6 +10,10 @@ services:
		branches:
		except:
		- "/^feature.*$/"
		addons:
		apt:
		packages:
		- libgnutls-dev
		install:
		- pip install -r requirements.txt
		- pip install coverage nose

CHANGES.txt

+14 −0

Original line number	Diff line number	Diff line
		3.0.27:
		Fix previous release broken with a bug in direct protocols
		3.0.26:
		Change default download timeout to 1h
		#12 Allow FTPS protocol
		#14 Add mechanism for protocol specific options
		3.0.25:
		Allow to use hardlinks in LocalDownload
		3.0.24:
		Remove debug logs
		3.0.23:
		Support spaces in remote file names
		3.0.22:
		Fix */ remote.files parsing
		3.0.21:
		Fix traefik labels
		3.0.20:

README.md

+3 −1

Original line number	Diff line number	Diff line
		# About

		[![PyPI version](https://badge.fury.io/py/biomaj-download.svg)](https://badge.fury.io/py/biomaj-download)

		Microservice to manage the downloads of biomaj.

		A protobuf interface is available in biomaj_download/message/message_pb2.py to exchange messages between BioMAJ and the download service.
		@@ -9,7 +11,7 @@ Messages go through RabbitMQ (to be installed).

		To compile protobuf, in biomaj_download/message:

		protoc --python_out=. message.proto
		protoc --python_out=. downmessage.proto

		# Development

biomaj_download/download/ftp.py

+149 −43

Original line number	Diff line number	Diff line
		import pycurl
		import re
		import os
		from datetime import datetime
		import time
		from datetime import datetime
		import stat
		import hashlib
		import ftputil

		from biomaj_core.utils import Utils
		from biomaj_download.download.interface import DownloadInterface
		@@ -13,6 +15,48 @@ try:
		except ImportError:
		from StringIO import StringIO as BytesIO

		# We use stat.filemode to convert from mode octal value to string.
		# In python < 3.3, stat.filmode is not defined.
		# This code is copied from the current implementation of stat.filemode.
		if 'filemode' not in stat.__dict__:
		_filemode_table = (
		((stat.S_IFLNK, "l"), # noqa: E241
		(stat.S_IFREG, "-"), # noqa: E241
		(stat.S_IFBLK, "b"), # noqa: E241
		(stat.S_IFDIR, "d"), # noqa: E241
		(stat.S_IFCHR, "c"), # noqa: E241
		(stat.S_IFIFO, "p")), # noqa: E241
		((stat.S_IRUSR, "r"),), # noqa: E241
		((stat.S_IWUSR, "w"),), # noqa: E241
		((stat.S_IXUSR \| stat.S_ISUID, "s"), # noqa: E241
		(stat.S_ISUID, "S"), # noqa: E241
		(stat.S_IXUSR, "x")), # noqa: E241
		((stat.S_IRGRP, "r"),), # noqa: E241
		((stat.S_IWGRP, "w"),), # noqa: E241
		((stat.S_IXGRP \| stat.S_ISGID, "s"), # noqa: E241
		(stat.S_ISGID, "S"), # noqa: E241
		(stat.S_IXGRP, "x")), # noqa: E241
		((stat.S_IROTH, "r"),), # noqa: E241
		((stat.S_IWOTH, "w"),), # noqa: E241
		((stat.S_IXOTH \| stat.S_ISVTX, "t"), # noqa: E241
		(stat.S_ISVTX, "T"), # noqa: E241
		(stat.S_IXOTH, "x")) # noqa: E241
		)

		def _filemode(mode):
		"""Convert a file's mode to a string of the form '-rwxrwxrwx'."""
		perm = []
		for table in _filemode_table:
		for bit, char in table:
		if mode & bit == bit:
		perm.append(char)
		break
		else:
		perm.append("-")
		return "".join(perm)

		stat.filemode = _filemode


		class FTPDownload(DownloadInterface):
		'''
		@@ -25,6 +69,12 @@ class FTPDownload(DownloadInterface):
		remote.files=^alu.*\\.gz$

		'''
		# Utilities to parse ftp listings: UnixParser is the more common hence we
		# put it first
		ftp_listing_parsers = [
		ftputil.stat.UnixParser(),
		ftputil.stat.MSParser(),
		]

		def __init__(self, protocol, host, rootdir):
		DownloadInterface.__init__(self)
		@@ -34,6 +84,25 @@ class FTPDownload(DownloadInterface):
		self.rootdir = rootdir
		self.url = url
		self.headers = {}
		# Initialize options
		# Should we skip SSL verification (cURL -k/--insecure option)
		self.ssl_verifyhost = True
		self.ssl_verifypeer = True
		# Path to the certificate of the server (cURL --cacert option; PEM format)
		self.ssl_server_cert = None
		# Keep alive
		self.tcp_keepalive = 0

		def set_options(self, protocol_options):
		super(FTPDownload, self).set_options(protocol_options)
		if "ssl_verifyhost" in protocol_options:
		self.ssl_verifyhost = Utils.to_bool(protocol_options["ssl_verifyhost"])
		if "ssl_verifypeer" in protocol_options:
		self.ssl_verifypeer = Utils.to_bool(protocol_options["ssl_verifypeer"])
		if "ssl_server_cert" in protocol_options:
		self.ssl_server_cert = protocol_options["ssl_server_cert"]
		if "tcp_keepalive" in protocol_options:
		self.tcp_keepalive = Utils.to_int(protocol_options["tcp_keepalive"])

		def match(self, patterns, file_list, dir_list=None, prefix='', submatch=False):
		'''
		@@ -63,19 +132,21 @@ class FTPDownload(DownloadInterface):
		if subdir == '^':
		subdirs_pattern = subdirs_pattern[1:]
		subdir = subdirs_pattern[0]
		for direlt in dir_list:
		subdir = direlt['name']
		self.logger.debug('Download:File:Subdir:Check:' + subdir)
		# If getting all, get all files
		if pattern == '*/':
		(subfile_list, subdirs_list) = self.list(prefix + '/' + subdir + '/')
		self.match([pattern], subfile_list, subdirs_list, prefix + '/' + subdir, True)
		for rfile in file_list:
		if pattern == '*/' or re.match(pattern, rfile['name']):
		rfile['root'] = self.rootdir
		if prefix != '':
		rfile['name'] = prefix + '/' + rfile['name']
		self.files_to_download.append(rfile)
		self.logger.debug('Download:File:MatchRegExp:' + rfile['name'])
		for direlt in dir_list:
		subdir = direlt['name']
		self.logger.debug('Download:File:Subdir:Check:' + subdir)
		if pattern == '*/':
		(subfile_list, subdirs_list) = self.list(prefix + '/' + subdir + '/')
		self.match([pattern], subfile_list, subdirs_list, prefix + '/' + subdir, True)

		else:
		if re.match(subdirs_pattern[0], subdir):
		self.logger.debug('Download:File:Subdir:Match:' + subdir)
		@@ -101,6 +172,27 @@ class FTPDownload(DownloadInterface):
		while(error is True and nbtry < 3):
		fp = open(file_path, "wb")
		curl = pycurl.Curl()

		# Configure TCP keepalive
		if self.tcp_keepalive:
		curl.setopt(pycurl.TCP_KEEPALIVE, True)
		curl.setopt(pycurl.TCP_KEEPIDLE, self.tcp_keepalive * 2)
		curl.setopt(pycurl.TCP_KEEPINTVL, self.tcp_keepalive)

		# Configure SSL verification (on some platforms, disabling
		# SSL_VERIFYPEER implies disabling SSL_VERIFYHOST so we set
		# SSL_VERIFYPEER after)
		curl.setopt(pycurl.SSL_VERIFYHOST, 2 if self.ssl_verifyhost else 0)
		curl.setopt(pycurl.SSL_VERIFYPEER, 1 if self.ssl_verifypeer else 0)
		if self.ssl_server_cert:
		# cacert is the name of the option for the curl command. The
		# corresponding cURL option is CURLOPT_CAINFO.
		# See https://curl.haxx.se/libcurl/c/CURLOPT_CAINFO.html
		# This is inspired by that https://curl.haxx.se/docs/sslcerts.html
		# (section "Certificate Verification", option 2) but the option
		# CURLOPT_CAPATH is for a directory of certificates.
		curl.setopt(pycurl.CAINFO, self.ssl_server_cert)

		try:
		curl.setopt(pycurl.URL, file_to_download)
		except Exception:
		@@ -133,8 +225,7 @@ class FTPDownload(DownloadInterface):
		nbtry += 1
		curl.close()
		fp.close()
		skip_check_uncompress = os.environ.get('UNCOMPRESS_SKIP_CHECK', None)
		if not error and skip_check_uncompress is None:
		if not error and not self.skip_check_uncompress:
		archive_status = Utils.archive_check(file_path)
		if not archive_status:
		self.logger.error('Archive is invalid or corrupted, deleting file and retrying download')
		@@ -231,6 +322,18 @@ class FTPDownload(DownloadInterface):
		'''
		self.logger.debug('Download:List:' + self.url + self.rootdir + directory)

		# Configure TCP keepalive
		if self.tcp_keepalive:
		self.crl.setopt(pycurl.TCP_KEEPALIVE, True)
		self.crl.setopt(pycurl.TCP_KEEPIDLE, self.tcp_keepalive * 2)
		self.crl.setopt(pycurl.TCP_KEEPINTVL, self.tcp_keepalive)

		# See the corresponding lines in method:`curl_download`
		self.crl.setopt(pycurl.SSL_VERIFYHOST, 2 if self.ssl_verifyhost else 0)
		self.crl.setopt(pycurl.SSL_VERIFYPEER, 1 if self.ssl_verifypeer else 0)
		if self.ssl_server_cert:
		self.crl.setopt(pycurl.CAINFO, self.ssl_server_cert)

		try:
		self.crl.setopt(pycurl.URL, self.url + self.rootdir + directory)
		except Exception:
		@@ -252,6 +355,7 @@ class FTPDownload(DownloadInterface):
		# Download should not take more than 5minutes
		self.crl.setopt(pycurl.TIMEOUT, self.timeout)
		self.crl.setopt(pycurl.NOSIGNAL, 1)

		try:
		self.crl.perform()
		except Exception as e:
		@@ -282,40 +386,42 @@ class FTPDownload(DownloadInterface):
		rdirs = []

		for line in lines:
		rfile = {}
		# lets print each part separately
		parts = line.split()
		# the individual fields in this list of parts
		if not parts:
		# Skip empty lines (usually the last)
		if not line:
		continue
		rfile['permissions'] = parts[0]
		rfile['group'] = parts[2]
		rfile['user'] = parts[3]
		rfile['size'] = int(parts[4])
		rfile['month'] = Utils.month_to_num(parts[5])
		rfile['day'] = int(parts[6])
		rfile['hash'] = hashlib.md5(line.encode('utf-8')).hexdigest()
		# Parse the line
		for i, parser in enumerate(self.ftp_listing_parsers, 1):
		try:
		rfile['year'] = int(parts[7])
		except Exception:
		# specific ftp case issues at getting date info
		curdate = datetime.now()
		rfile['year'] = curdate.year
		# Year not precised, month feater than current means previous year
		if rfile['month'] > curdate.month:
		rfile['year'] = curdate.year - 1
		# Same month but later day => previous year
		if rfile['month'] == curdate.month and rfile['day'] > curdate.day:
		rfile['year'] = curdate.year - 1
		rfile['name'] = parts[8]
		if len(parts) >= 10 and parts[9] == '->':
		# Symlink, add to files AND dirs as we don't know the type of the link
		rdirs.append(rfile)

		is_dir = False
		if re.match('^d', rfile['permissions']):
		is_dir = True
		stats = parser.parse_line(line)
		break
		except ftputil.error.ParserError:
		# If it's the last parser, re-raise the exception
		if i == len(self.ftp_listing_parsers):
		raise
		else:
		continue
		# Put stats in a dict
		rfile = {}
		rfile['name'] = stats._st_name
		# Reparse mode to a string
		rfile['permissions'] = stat.filemode(stats.st_mode)
		rfile['group'] = stats.st_gid
		rfile['user'] = stats.st_uid
		rfile['size'] = stats.st_size
		mtime = time.localtime(stats.st_mtime)
		rfile['year'] = mtime.tm_year
		rfile['month'] = mtime.tm_mon
		rfile['day'] = mtime.tm_mday
		rfile['hash'] = hashlib.md5(line.encode('utf-8')).hexdigest()

		is_link = stat.S_ISLNK(stats.st_mode)
		is_dir = stat.S_ISDIR(stats.st_mode)
		# Append links to dirs and files since we don't know what the
		# target is
		if is_link:
		rfiles.append(rfile)
		rdirs.append(rfile)
		else:
		if not is_dir:
		rfiles.append(rfile)
		else:

biomaj_download/download/interface.py

+17 −1

Original line number	Diff line number	Diff line
		@@ -4,6 +4,8 @@ import datetime
		import time
		import re

		from biomaj_core.utils import Utils


		class _FakeLock(object):
		'''
		@@ -39,7 +41,7 @@ class DownloadInterface(object):
		self.kill_received = False
		self.proxy = None
		# 24h timeout
		self.timeout = 3600 * 24
		self.timeout = 3600
		# Optional save target for single file downloaders
		self.save_as = None
		self.logger = logging.getLogger('biomaj')
		@@ -48,6 +50,9 @@ class DownloadInterface(object):
		self.protocol = None
		self.server = None
		self.offline_dir = None
		# Options
		self.protocol_options = {}
		self.skip_check_uncompress = False

		def set_offline_dir(self, offline_dir):
		self.offline_dir = offline_dir
		@@ -266,6 +271,17 @@ class DownloadInterface(object):
		'''
		self.credentials = userpwd

		def set_options(self, protocol_options):
		"""
		Set protocol specific options.

		Subclasses that override this method must call the
		parent implementation.
		"""
		self.protocol_options = protocol_options
		if "skip_check_uncompress" in protocol_options:
		self.skip_check_uncompress = Utils.to_bool(protocol_options["skip_check_uncompress"])

		def close(self):
		'''
		Close connection