New upstream version 3.0.19 (8c8e541f) · Commits · Debian Med / biomaj3-core

.travis.yml

+4 −3

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@ python:
		- '2.7'
		- '3.4'
		- '3.5'
		- '3.6'
		branches:
		except:
		- "/^feature.*$/"

CHANGES.txt

+8 −0

Original line number	Diff line number	Diff line
		3.0.19 [unreleased]:
		Add ftps and directftps protocols
		#4 Harden hardlinks
		copy_files_with_regexp: remove duplicates
		3.0.18:
		Allow to use hardlinks in copy_files and copy_files_with_regexp
		3.0.17:
		Fix --log option
		3.0.16:
		Add some warnings if some file is missing
		3.0.15:

README.md

+2 −0

Original line number	Diff line number	Diff line
		@@ -8,3 +8,5 @@ Properties can be overriden by environment variables with pattern BIOMAJ_X_Y_Z f
		Example:

		use BIOMAJ_LDAP_HOST for ldap.host

		[![PyPI version](https://badge.fury.io/py/biomaj-core.svg)](https://badge.fury.io/py/biomaj-core)

biomaj_core/config.py

+11 −16

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ import os
		import time
		import sys

		from biomaj_core.utils import Utils
		from biomaj_core.bmajindex import BmajIndex

		if sys.version < '3':
		@@ -40,8 +41,10 @@ class BiomajConfig(object):
		'db.type': '',
		'db.formats': '',
		'keep.old.version': 1,
		'keep.old.sessions': 0,
		'docker.sudo': '1',
		'auto_publish': 0
		'auto_publish': 0,
		'use_hardlinks': 0
		}

		# Old biomaj level compatibility
		@@ -191,9 +194,11 @@ class BiomajConfig(object):
		if options is not None and options.get_option('log') is not None:
		hdlr.setLevel(BiomajConfig.LOGLEVEL[options.get_option('log')])
		self.log_level = BiomajConfig.LOGLEVEL[options.get_option('log')]
		logger.setLevel(self.log_level)
		else:
		hdlr.setLevel(BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')])
		self.log_level = BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')]
		logger.setLevel(self.log_level)
		formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s')
		hdlr.setFormatter(formatter)
		logger.addHandler(hdlr)
		@@ -281,19 +286,9 @@ class BiomajConfig(object):
		"""
		Get a boolean property from bank or general configration. Optionally in section.
		"""
		value = None

		if self._in_env(prop):
		value = self._in_env(prop)
		else:
		value = self.get(prop, section, escape, default)

		if value is None:
		return False
		if value is True or value == 'true' or value == '1':
		return True
		else:
		return False
		return Utils.to_bool(value)

		def get(self, prop, section='GENERAL', escape=True, default=None):
		"""
		@@ -402,8 +397,8 @@ class BiomajConfig(object):
		status = False
		else:
		protocol = self.get('protocol')
		allowed_protocols = ['none', 'multi', 'local', 'ftp', 'sftp', 'http',
		'https', 'directftp', 'directhttp', 'directhttps', 'rsync', 'irods']
		allowed_protocols = ['none', 'multi', 'local', 'ftp', 'ftps', 'http', 'https',
		'directftp', 'directftps', 'directhttp', 'directhttps', 'rsync', 'irods']
		if protocol not in allowed_protocols:
		logging.error('Protocol not supported: ' + protocol)
		status = False
		@@ -417,7 +412,7 @@ class BiomajConfig(object):
		elif not self.get('remote.dir').endswith('/'):
		logging.error('remote.dir must end with a /')
		return False
		if protocol not in ['direcftp', 'directhttp', 'directhttps'] and\
		if protocol not in ['directftp', 'directftps', 'directhttp', 'directhttps'] and\
		not self.get('remote.files') and\
		not self.get('remote.list'):
		logging.error('remote.files not set')

biomaj_core/utils.py

+124 −17

Original line number	Diff line number	Diff line
		import os
		import errno
		import re
		import logging
		import shutil
		@@ -152,11 +153,12 @@ class Utils(object):

		Each file is a dict like with (at least) parameters: year, month, day
		"""
		release = None
		for rfile in files:
		if release is None:
		if not files:
		return None
		# release = None
		rfile = files[0]
		release = {'year': rfile['year'], 'month': rfile['month'], 'day': rfile['day']}
		else:
		for rfile in files:
		rel_date = datetime.date(int(release['year']), int(release['month']), int(release['day']))
		file_date = datetime.date(int(rfile['year']), int(rfile['month']), int(rfile['day']))
		if file_date > rel_date:
		@@ -195,7 +197,8 @@ class Utils(object):
		}[date]

		@staticmethod
		def copy_files(files_to_copy, to_dir, move=False, lock=None):
		def copy_files(files_to_copy, to_dir, move=False, lock=None,
		use_hardlinks=False):
		"""
		Copy or move files to to_dir, keeping directory structure.

		@@ -214,6 +217,8 @@ class Utils(object):
		:type move: bool
		:param lock: thread lock object for multi-threads
		:type lock: Lock
		:param use_hardlinks: use hard links (if possible)
		:type link: bool
		"""
		logger = logging.getLogger('biomaj')
		nb_files = len(files_to_copy)
		@@ -244,14 +249,48 @@ class Utils(object):
		else:
		start_time = datetime.datetime.now()
		start_time = time.mktime(start_time.timetuple())
		if use_hardlinks:
		try:
		os.link(from_file, to_file)
		logger.debug("Using hardlinks to copy %s",
		file_to_copy['name'])
		except OSError as e:
		if e.errno in (errno.ENOSYS, errno.ENOTSUP):
		msg = "Your system doesn't support hard links. Using regular copy."
		logger.warn(msg)
		# Copy this file (the stats are copied at the end
		# of the function)
		shutil.copyfile(from_file, to_file)
		# Don't try links anymore
		use_hardlinks = False
		elif e.errno == errno.EPERM:
		msg = "The FS at %s doesn't support hard links. Using regular copy."
		logger.warn(msg, to_dir)
		# Copy this file (the stats are copied at the end
		# of the function)
		shutil.copyfile(from_file, to_file)
		# Don't try links anymore
		use_hardlinks = False
		elif e.errno == errno.EXDEV:
		msg = "Cross device hard link is impossible (source: %s, dest: %s). Using regular copy."
		logger.warn(msg, from_file, to_dir)
		# Copy this file
		shutil.copyfile(from_file, to_file)
		# Don't try links anymore
		use_hardlinks = False
		else:
		raise
		else:
		shutil.copyfile(from_file, to_file)
		end_time = datetime.datetime.now()
		end_time = time.mktime(end_time.timetuple())
		file_to_copy['download_time'] = end_time - start_time
		if not use_hardlinks:
		shutil.copystat(from_file, to_file)

		@staticmethod
		def copy_files_with_regexp(from_dir, to_dir, regexps, move=False, lock=None):
		def copy_files_with_regexp(from_dir, to_dir, regexps, move=False, lock=None,
		use_hardlinks=False):
		"""
		Copy or move files from from_dir to to_dir matching regexps.
		Copy keeps the original file stats.
		@@ -266,20 +305,28 @@ class Utils(object):
		:type move: bool
		:param lock: thread lock object for multi-threads
		:type lock: Lock
		:param use_hardlinks: use hard links (if possible)
		:type link: bool
		:return: list of copied files with their size
		"""
		logger = logging.getLogger('biomaj')
		files_to_copy = []
		for root, dirs, files in os.walk(from_dir, topdown=True):
		files_list = []
		for root, _, files in os.walk(from_dir, topdown=True):
		for name in files:
		for reg in regexps:
		file_relative_path = os.path.join(root, name).replace(from_dir, '')
		if file_relative_path.startswith('/'):
		file_relative_path = file_relative_path.replace('/', '', 1)
		# sometimes files appear twice.... check not already managed
		if file_relative_path in files_list:
		continue
		if reg == "*/":
		files_to_copy.append({'name': file_relative_path})
		files_list.append(file_relative_path)
		continue
		if re.match(reg, file_relative_path):
		files_list.append(file_relative_path)
		files_to_copy.append({'name': file_relative_path})
		continue

		@@ -304,6 +351,41 @@ class Utils(object):
		continue
		if move:
		shutil.move(from_file, to_file)
		else:
		if use_hardlinks:
		try:
		os.link(from_file, to_file)
		logger.debug("Using hardlinks to copy %s",
		file_to_copy['name'])
		except OSError as e:
		if e.errno in (errno.ENOSYS, errno.ENOTSUP):
		msg = "Your system doesn't support hard links. Using regular copy."
		logger.warn(msg)
		# Copy this file (the stats are copied at the end
		# of the function)
		shutil.copyfile(from_file, to_file)
		# Don't try links anymore
		use_hardlinks = False
		elif e.errno == errno.EPERM:
		msg = "The FS at %s doesn't support hard links. Using regular copy."
		logger.warn(msg, to_dir)
		# Copy this file (we copy the stats here because
		# it's not done at the end of the function)
		shutil.copyfile(from_file, to_file)
		shutil.copystat(from_file, to_file)
		# Don't try links anymore
		use_hardlinks = False
		elif e.errno == errno.EXDEV:
		msg = "Cross device hard link is impossible (source: %s, dest: %s). Using regular copy."
		logger.warn(msg, from_file, to_dir)
		# Copy this file (we copy the stats here because
		# it's not done at the end of the function)
		shutil.copyfile(from_file, to_file)
		shutil.copystat(from_file, to_file)
		# Don't try links anymore
		use_hardlinks = False
		else:
		raise
		else:
		shutil.copyfile(from_file, to_file)
		shutil.copystat(from_file, to_file)
		@@ -384,3 +466,28 @@ class Utils(object):
		os.remove(archivefile)

		return True

		@staticmethod
		def to_bool(value):
		if isinstance(value, bool):
		return value
		if not value:
		return False
		try:
		if value.lower() == 'true' or value == '1':
		return True
		else:
		return False
		except Exception:
		return False

		@staticmethod
		def to_int(value):
		if isinstance(value, int):
		return value
		if not value:
		return 0
		try:
		return int(value)
		except Exception:
		return 0