New upstream version 3.1.0

- pip install python-coveralls

- python setup.py -q install

script:

- nosetests -a '!network'

- nosetests -a '!network,!local_irods'

- flake8 --ignore E501 biomaj_download/*.py biomaj_download/download

deploy:

  provider: pypi

3.1.0:

  #16 Don't change name after download in DirectHTTPDownloader

  PR #7 Refactor downloaders (*WARNING* breaks API)

3.0.27:

  Fix previous release broken with a bug in direct protocols

3.0.26:

    flake8  biomaj_download/\*.py biomaj_download/download

# Test

To run the test suite, use:

    nosetests -a '!local_irods' tests/biomaj_tests.py

This command skips the test that need a local iRODS server.

Some test might fail due to network connection. You can skip them with:

    nosetests -a '!network' tests/biomaj_tests.py

(To skip the local iRODS test and the network tests, use `-a '!network,!local_irods'`).

# Run

"""

Subclasses for direct download (i.e. downloading without regexp). The usage is

a bit different: instead of calling method:`list` and method:`match`, client

code explicitely calls method:`set_files_to_download` (passing a list

containing only the file name). method:`list` is used to get more information

about the file (if possile). method:`match` matches everything.

Also client code can use method:`set_save_as` to indicate the name of the file

to save.

The trick for the implementation is to override

method:`_append_file_to_download` to initialize the rfile with the file name

and dummy values. Note that we use a list of rfile even if it contains only one

file.

method:`list` will modify directly the files_to_download.

method:``match` don't call method:`_append_file_to_download` (since the list of

files to download is already set up).

We also override method:`set_files_to_download` to check that we pass only one

file.

"""

import datetime

import time

import pycurl

import os

import re

import hashlib

import sys

from biomaj_download.download.ftp import FTPDownload

from biomaj_download.download.curl import CurlDownload

from biomaj_core.utils import Utils

if sys.version_info[0] < 3:

    from StringIO import StringIO as BytesIO

class DirectFTPDownload(FTPDownload):

class DirectFTPDownload(CurlDownload):

    '''

    download a list of files from FTP, no regexp

    '''

    def __init__(self, protocol, host, rootdir=''):

        '''

    ALL_PROTOCOLS = ["ftp", "ftps"]

    def _append_file_to_download(self, filename):

        '''

        Initialize the files in list with today as last-modification date.

        Size is also preset to zero, size will be set after download

        Size is also preset to zero.

        '''

        FTPDownload.__init__(self, protocol, host, rootdir)

        self.save_as = None

        self.headers = {}

    def set_files_to_download(self, files):

        today = datetime.date.today()

        self.files_to_download = []

        for file_to_download in files:

        rfile = {}

            rfile['root'] = ''

        rfile['root'] = self.rootdir

        rfile['permissions'] = ''

        rfile['group'] = ''

        rfile['user'] = ''

        rfile['month'] = today.month

        rfile['day'] = today.day

        rfile['year'] = today.year

            if file_to_download.endswith('/'):

                rfile['name'] = file_to_download[:-1]

        if filename.endswith('/'):

            rfile['name'] = filename[:-1]

        else:

                rfile['name'] = file_to_download

            rfile['name'] = filename

        rfile['hash'] = None

            if self.param:

                if 'param' not in file_to_download or not file_to_download['param']:

                    rfile['param'] = self.param

            self.files_to_download.append(rfile)

        # Use self.save_as even if we use it in list(). This is important.

        rfile['save_as'] = self.save_as

        super(DirectFTPDownload, self)._append_file_to_download(rfile)

    def set_files_to_download(self, files_to_download):

        if len(files_to_download) > 1:

            self.files_to_download = []

            msg = self.__class__.__name__ + ' accepts only 1 file'

            self.logger.error(msg)

            raise ValueError(msg)

        return super(DirectFTPDownload, self).set_files_to_download(files_to_download)

    def list(self, directory=''):

        '''

        FTP protocol does not give us the possibility to get file date from remote url

        '''

        for rfile in self.files_to_download:

            if self.save_as is None:

                self.save_as = rfile['name']

            rfile['save_as'] = self.save_as

        # TODO: are we sure about this implementation ?

        return (self.files_to_download, [])

    def match(self, patterns, file_list, dir_list=None, prefix='', submatch=False):

        '''

        All files to download match, no pattern

        '''

        if dir_list is None:

            dir_list = []

        self.files_to_download = file_list

        pass

class DirectHttpDownload(DirectFTPDownload):

class DirectHTTPDownload(DirectFTPDownload):

    def __init__(self, protocol, host, rootdir=''):

        '''

        :param file_list: list of files to download on server

        :type file_list: list

        '''

        DirectFTPDownload.__init__(self, protocol, host, rootdir)

        self.save_as = None

    ALL_PROTOCOLS = ["http", "https"]

    def __init__(self, curl_protocol, host, rootdir=''):

        DirectFTPDownload.__init__(self, curl_protocol, host, rootdir)

        self.method = 'GET'

        self.param = {}

    def download(self, local_dir, keep_dirs=True):

        '''

        Download remote files to local_dir

        :param local_dir: Directory where files should be downloaded

        :type local_dir: str

        :param keep_dirs: keep file name directory structure or copy file in local_dir directly

        :param keep_dirs: bool

        :return: list of downloaded files

        '''

        self.logger.debug('DirectHTTP:Download')

        nb_files = len(self.files_to_download)

        if nb_files > 1:

            self.files_to_download = []

            self.logger.error('DirectHTTP accepts only 1 file')

        cur_files = 1

        for rfile in self.files_to_download:

            if self.kill_received:

                raise Exception('Kill request received, exiting')

            if not self.save_as:

                self.save_as = rfile['name']

            else:

                rfile['save_as'] = self.save_as

            file_dir = local_dir

            if keep_dirs:

                file_dir = local_dir + os.path.dirname(self.save_as)

            file_path = file_dir + '/' + os.path.basename(self.save_as)

            # For unit tests only, workflow will take in charge directory creation before to avoid thread multi access

            if not os.path.exists(file_dir):

                os.makedirs(file_dir)

            self.logger.debug('DirectHTTP:Download:Progress' + str(cur_files) + '/' + str(nb_files) + ' downloading file ' + rfile['name'] + ', save as ' + self.save_as)

            cur_files += 1

            if 'url' not in rfile:

                rfile['url'] = self.url

            fp = open(file_path, "wb")

            curl = pycurl.Curl()

            if self.proxy is not None:

                curl.setopt(pycurl.PROXY, self.proxy)

                if self.proxy_auth is not None:

                    curl.setopt(pycurl.PROXYUSERPWD, self.proxy_auth)

            if self.method == 'POST':

                # Form data must be provided already urlencoded.

                postfields = urlencode(self.param)

                # Sets request method to POST,

                # Content-Type header to application/x-www-form-urlencoded

                # and data to send in request body.

                if self.credentials is not None:

                    curl.setopt(pycurl.USERPWD, self.credentials)

                curl.setopt(pycurl.POSTFIELDS, postfields)

                try:

                    curl.setopt(pycurl.URL, rfile['url'] + rfile['root'] + '/' + rfile['name'])

                except Exception:

                    curl.setopt(pycurl.URL, (rfile['url'] + rfile['root'] + '/' + rfile['name']).encode('ascii', 'ignore'))

            else:

                url = rfile['url'] + rfile['root'] + '/' + rfile['name'] + '?' + urlencode(self.param)

                try:

                    curl.setopt(pycurl.URL, url)

                except Exception:

                    curl.setopt(pycurl.URL, url.encode('ascii', 'ignore'))

            curl.setopt(pycurl.WRITEDATA, fp)

            start_time = datetime.datetime.now()

            start_time = time.mktime(start_time.timetuple())

            curl.perform()

            end_time = datetime.datetime.now()

            end_time = time.mktime(end_time.timetuple())

            rfile['download_time'] = end_time - start_time

            curl.close()

            fp.close()

            self.logger.debug('downloaded!')

            rfile['name'] = self.save_as

            self.set_permissions(file_path, rfile)

        return self.files_to_download

    def header_function(self, header_line):

        # HTTP standard specifies that headers are encoded in iso-8859-1.

        # On Python 2, decoding step can be skipped.

        # On Python 3, decoding step is required.

        header_line = header_line.decode('iso-8859-1')

        # Header lines include the first status line (HTTP/1.x ...).

        # We are going to ignore all lines that don't have a colon in them.

        # This will botch headers that are split on multiple lines...

        if ':' not in header_line:

            return

        # Break the header line into header name and value.

        name, value = header_line.split(':', 1)

        # Remove whitespace that may be present.

        # Header lines include the trailing newline, and there may be whitespace

        # around the colon.

        name = name.strip()

        value = value.strip()

        # Header names are case insensitive.

        # Lowercase name here.

        name = name.lower()

        # Now we can actually record the header name and value.

        self.headers[name] = value

    def _file_url(self, file_to_download):

        url = super(DirectHTTPDownload, self)._file_url(file_to_download)

        if self.method == "GET":

            url += '?' + urlencode(self.param)

        return url

    def list(self, directory=''):

        '''

        Try to get file headers to get last_modification and size

        '''

        self._basic_curl_configuration()

        # Specific configuration

        self.crl.setopt(pycurl.HEADER, True)

        self.crl.setopt(pycurl.NOBODY, True)

        for rfile in self.files_to_download:

            if self.save_as is None:

                self.save_as = rfile['name']

            rfile['save_as'] = self.save_as

            self.crl.setopt(pycurl.HEADER, True)

            if self.credentials is not None:

                self.crl.setopt(pycurl.USERPWD, self.credentials)

            if self.proxy is not None:

                self.crl.setopt(pycurl.PROXY, self.proxy)

                if self.proxy_auth is not None:

                    self.crl.setopt(pycurl.PROXYUSERPWD, self.proxy_auth)

            self.crl.setopt(pycurl.NOBODY, True)

            file_url = self._file_url(rfile)

            try:

                self.crl.setopt(pycurl.URL, self.url + self.rootdir + rfile['name'])

                self.crl.setopt(pycurl.URL, file_url)

            except Exception:

                self.crl.setopt(pycurl.URL, (self.url + self.rootdir + rfile['name']).encode('ascii', 'ignore'))

                self.crl.setopt(pycurl.URL, file_url.encode('ascii', 'ignore'))

            # Create a buffer and assign it to the pycurl object

            output = BytesIO()

            # lets assign this buffer to pycurl object

            self.crl.setopt(pycurl.WRITEFUNCTION, output.write)

            self.crl.setopt(pycurl.HEADERFUNCTION, self.header_function)

            self.crl.perform()

            # Figure out what encoding was sent with the response, if any.