New upstream version 3.0.18 (1bd9f064) · Commits · Debian Med / biomaj3-download

CHANGES.txt

+10 −0

Original line number	Diff line number	Diff line
		3.0.18:
		Rename protobuf and use specific package to avoid conflicts
		3.0.17:
		Regenerate protobuf message desc, failing on python3
		3.0.16:
		Add missing req in setup.py
		3.0.15:
		Fix progress download control where could have infinite loop
		Add irods download

		3.0.14:
		Allow setup of local_endpoint per service, else use default local_endpoint

README.md

+2 −0

Original line number	Diff line number	Diff line
		@@ -41,3 +41,5 @@ If you cloned the repository and installed it via python setup.py install, just
		gunicorn -c gunicorn_conf.py biomaj_download.biomaj_download_web:app

		Web processes should be behind a proxy/load balancer, API base url /api/download

		Prometheus endpoint metrics are exposed via /metrics on web server

biomaj_download/biomaj_download_web.py

+6 −6

Original line number	Diff line number	Diff line
		@@ -16,7 +16,7 @@ from prometheus_client import multiprocess
		from prometheus_client import CollectorRegistry
		import consul

		from biomaj_download.message import message_pb2
		from biomaj_download.message import downmessage_pb2
		from biomaj_download.downloadservice import DownloadService

		from biomaj_core.utils import Utils
		@@ -86,7 +86,7 @@ def list_status(bank, session):
		Check if listing request is over
		'''
		dserv = DownloadService(config_file, rabbitmq=False)
		biomaj_file_info = message_pb2.DownloadFile()
		biomaj_file_info = downmessage_pb2.DownloadFile()
		biomaj_file_info.bank = bank
		biomaj_file_info.session = session
		biomaj_file_info.local_dir = '/tmp'
		@@ -100,7 +100,7 @@ def download_status(bank, session):
		Get number of downloads and errors for bank and session. Progress includes successful download and errored downloads.
		'''
		dserv = DownloadService(config_file, rabbitmq=False)
		biomaj_file_info = message_pb2.DownloadFile()
		biomaj_file_info = downmessage_pb2.DownloadFile()
		biomaj_file_info.bank = bank
		biomaj_file_info.session = session
		biomaj_file_info.local_dir = '/tmp'
		@@ -114,7 +114,7 @@ def download_error(bank, session):
		Get errors info for bank and session
		'''
		dserv = DownloadService(config_file, rabbitmq=False)
		biomaj_file_info = message_pb2.DownloadFile()
		biomaj_file_info = downmessage_pb2.DownloadFile()
		biomaj_file_info.bank = bank
		biomaj_file_info.session = session
		biomaj_file_info.local_dir = '/tmp'
		@@ -128,7 +128,7 @@ def list_result(bank, session):
		Get file listing for bank and session, using FileList protobuf serialized string
		'''
		dserv = DownloadService(config_file, rabbitmq=False)
		biomaj_file_info = message_pb2.DownloadFile()
		biomaj_file_info = downmessage_pb2.DownloadFile()
		biomaj_file_info.bank = bank
		biomaj_file_info.session = session
		biomaj_file_info.local_dir = '/tmp'
		@@ -146,7 +146,7 @@ def create_session(bank):
		@app.route('/api/download/session/<bank>/<session>', methods=['DELETE'])
		def clean_session(bank, session):
		dserv = DownloadService(config_file, rabbitmq=False)
		biomaj_file_info = message_pb2.DownloadFile()
		biomaj_file_info = downmessage_pb2.DownloadFile()
		biomaj_file_info.bank = bank
		biomaj_file_info.session = session
		dserv.clean(biomaj_file_info)

biomaj_download/download/protocolirods.py

0 → 100644

+126 −0

Original line number	Diff line number	Diff line
		import logging
		import os
		from datetime import datetime
		import time

		from biomaj_download.download.interface import DownloadInterface
		from irods.session import iRODSSession
		from irods.models import Collection, DataObject, User


		class IRODSDownload(DownloadInterface):
		# To connect to irods session : sess = iRODSSession(host='localhost', port=1247, user='rods', password='rods', zone='tempZone')
		# password : self.credentials
		def __init__(self, protocol, server, remote_dir):
		DownloadInterface.__init__(self)
		self.port = None
		self.remote_dir = remote_dir # directory on the remote server : zone
		self.rootdir = remote_dir
		self.user = None
		self.password = None
		self.server = server
		self.zone = None

		def set_param(self, param):
		# self.param is a dictionnary which has the following form :{'password': u'biomaj', 'protocol': u'iget', 'user': u'biomaj', 'port': u'port'}
		self.param = param
		self.port = int(param['port'])
		self.user = str(param['user'])
		self.password = str(param['password'])
		self.zone = str(param['zone'])

		def list(self, directory=''):
		session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
		rfiles = []
		rdirs = []
		rfile = {}
		date = None
		for result in session.query(Collection.name, DataObject.name, DataObject.size, DataObject.owner_name, DataObject.modify_time).filter(User.name == self.user).get_results():
		# if the user is biomaj : he will have access to all the irods data (biomaj ressource) : drwxr-xr-x
		# Avoid duplication
		if rfile != {} and rfile['name'] == str(result[DataObject.name]) and date == str(result[DataObject.modify_time]).split(" ")[0].split('-'):
		continue
		rfile = {}
		date = str(result[DataObject.modify_time]).split(" ")[0].split('-')
		rfile['permissions'] = "-rwxr-xr-x"
		rfile['size'] = int(result[DataObject.size])
		rfile['month'] = int(date[1])
		rfile['day'] = int(date[2])
		rfile['year'] = int(date[0])
		rfile['name'] = str(result[DataObject.name])
		rfile['download_path'] = str(result[Collection.name])
		rfiles.append(rfile)
		session.cleanup()
		return (rfiles, rdirs)

		def download(self, local_dir, keep_dirs=True):
		'''
		Download remote files to local_dir

		:param local_dir: Directory where files should be downloaded
		:type local_dir: str
		:param keep_dirs: keep file name directory structure or copy file in local_dir directly
		:param keep_dirs: bool
		:return: list of downloaded files
		'''
		logging.debug('IRODS:Download')
		try:
		os.chdir(local_dir)
		except TypeError:
		logging.error("IRODS:list:Could not find offline_dir")
		nb_files = len(self.files_to_download)
		cur_files = 1
		# give a working directory to copy the file from irods
		remote_dir = self.remote_dir
		for rfile in self.files_to_download:
		if self.kill_received:
		raise Exception('Kill request received, exiting')
		file_dir = local_dir
		if 'save_as' not in rfile or rfile['save_as'] is None:
		rfile['save_as'] = rfile['name']
		if keep_dirs:
		file_dir = local_dir + os.path.dirname(rfile['save_as'])
		file_path = file_dir + '/' + os.path.basename(rfile['save_as'])
		# For unit tests only, workflow will take in charge directory creation before to avoid thread multi access
		if not os.path.exists(file_dir):
		os.makedirs(file_dir)

		logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' downloading file ' + rfile['name'])
		logging.debug('IRODS:Download:Progress:' + str(cur_files) + '/' + str(nb_files) + ' save as ' + rfile['save_as'])
		cur_files += 1
		start_time = datetime.now()
		start_time = time.mktime(start_time.timetuple())
		self.remote_dir = rfile['root']
		error = self.irods_download(file_dir, str(self.remote_dir), str(rfile['name']))
		if error:
		rfile['download_time'] = 0
		rfile['error'] = True
		raise Exception("IRODS:Download:Error:" + rfile['root'] + '/' + rfile['name'])
		end_time = datetime.now()
		end_time = time.mktime(end_time.timetuple())
		rfile['download_time'] = end_time - start_time
		self.set_permissions(file_path, rfile)
		self.remote_dir = remote_dir
		return(self.files_to_download)

		def irods_download(self, file_dir, file_path, file_to_download):
		error = False
		logging.debug('IRODS:IRODS DOWNLOAD')
		session = iRODSSession(host=self.server, port=self.port, user=self.user, password=self.password, zone=self.zone)
		try:
		file_to_get = str(file_path) + str(file_to_download)
		# Write the file to download in the wanted file_dir : with the python-irods iget
		obj = session.data_objects.get(file_to_get, file_dir)
		except ExceptionIRODS as e:
		logging.error("RsyncError:" + str(e))
		logging.error("RsyncError: irods object" + str(obj))
		session.cleanup()
		return(error)


		class ExceptionIRODS(Exception):
		def __init__(self, exception_reason):
		self.exception_reason = exception_reason

		def __str__(self):
		return self.exception_reason

biomaj_download/downloadclient.py

+7 −7

Original line number	Diff line number	Diff line
		@@ -7,7 +7,7 @@ import sys
		import pika

		from biomaj_download.download.downloadthreads import DownloadThread
		from biomaj_download.message import message_pb2
		from biomaj_download.message import downmessage_pb2

		if sys.version_info[0] < 3:
		from Queue import Queue
		@@ -92,15 +92,15 @@ class DownloadClient(DownloadService):
		'''
		for downloader in downloaders:
		for file_to_download in downloader.files_to_download:
		operation = message_pb2.Operation()
		operation = downmessage_pb2.Operation()
		operation.type = 1
		message = message_pb2.DownloadFile()
		message = downmessage_pb2.DownloadFile()
		message.bank = self.bank
		message.session = self.session
		message.local_dir = offline_dir
		remote_file = message_pb2.DownloadFile.RemoteFile()
		remote_file = downmessage_pb2.DownloadFile.RemoteFile()
		protocol = downloader.protocol
		remote_file.protocol = message_pb2.DownloadFile.Protocol.Value(protocol.upper())
		remote_file.protocol = downmessage_pb2.DownloadFile.Protocol.Value(protocol.upper())
		remote_file.server = downloader.server
		if cf.get('remote.dir'):
		remote_file.remote_dir = cf.get('remote.dir')
		@@ -135,7 +135,7 @@ class DownloadClient(DownloadService):
		if 'md5' in file_to_download and file_to_download['md5']:
		biomaj_file.metadata.md5 = file_to_download['md5']

		message.http_method = message_pb2.DownloadFile.HTTP_METHOD.Value(downloader.method.upper())
		message.http_method = downmessage_pb2.DownloadFile.HTTP_METHOD.Value(downloader.method.upper())

		timeout_download = cf.get('timeout.download', None)
		if timeout_download:
		@@ -216,7 +216,7 @@ class DownloadClient(DownloadService):
		self.ask_download(operation)
		nb_submitted += 1

		if progress == nb_files_to_download:
		if progress >= nb_files_to_download:
		over = True
		logging.info("Workflow:wf_download:RemoteDownload:Completed:" + str(progress))
		logging.info("Workflow:wf_download:RemoteDownload:Errors:" + str(error))