<commit_before>import numpy as np

try:
    from nvidia import dali
    from nvidia.dali import ops
    from nvidia.dali import pipeline
    _dali_available = True
except ImportError:
    class pipeline(object):
        Pipeline = object
        pass
    _dali_available = False

from chainer.backends import cuda
import ctypes

def _pair(x):
    if hasattr(x, '__getitem__'):
        return x
    return x, x

class DaliPipelineTrain(pipeline.Pipeline):

    def __init__(self, file_list, file_root, crop_size,
                 batch_size, num_threads, device_id,
                 random_shuffle=True, seed=-1, mean=None, std=None,
                 num_samples=None):
        super(DaliPipelineTrain, self).__init__(batch_size, num_threads,
                                                device_id, seed=seed)
        crop_size = _pair(crop_size)
        if mean is None:
            mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
        if std is None:
            std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
        if num_samples is None:
            initial_fill = 4096
        else:
            initial_fill = min(4096, num_samples)
        self.loader = ops.FileReader(file_root=file_root, file_list=file_list,
                                     random_shuffle=random_shuffle,
                                     initial_fill=initial_fill)
        self.decode = ops.HostDecoder()
        self.resize = ops.Resize(device="gpu", resize_x=256, resize_y=256)
        # self.hue = ops.Hue(device="gpu")
        # self.bright = ops.Brightness(device="gpu")
        # self.cntrst = ops.Contrast(device="gpu")
        # self.rotate = ops.Rotate(device="gpu")
        # self.jitter = ops.Jitter(device="gpu")
        random_area = (crop_size[0] / 256) * (crop_size[1] / 256)
        random_area = _pair(random_area)
        random_aspect_ratio = _pair(1.0)
        self.rrcrop = ops.RandomResizedCrop(
            device="gpu", size=crop_size, random_area=random_area,
            random_aspect_ratio=random_aspect_ratio)
        self.cmnorm = ops.CropMirrorNormalize(
            device="gpu", crop=list(crop_size), mean=mean, std=std)
        self.coin = ops.CoinFlip(probability=0.5)

    def define_graph(self):
        jpegs, labels = self.loader()
        images = self.decode(jpegs)
        images = self.resize(images.gpu())
        # images = self.hue(images, hue=ops.Uniform(range=(-3.0, 3.0))())
        # images = self.bright(images,
        #                      brightness=ops.Uniform(range=(0.9, 1.1))())
        # images = self.cntrst(images,
        #                      contrast=ops.Uniform(range=(0.9, 1.1))())
        # images = self.rotate(images,
        #                      angle=ops.Uniform(range=(-5.0, 5.0))())
        # images = self.jitter(images)
        images = self.rrcrop(images)
        images = self.cmnorm(images, mirror=self.coin())
        return images, labels

class DaliPipelineVal(pipeline.Pipeline):

    def __init__(self, file_list, file_root, crop_size,
                 batch_size, num_threads, device_id,
                 random_shuffle=False, seed=-1, mean=None, std=None,
                 num_samples=None):
        super(DaliPipelineVal, self).__init__(batch_size, num_threads,
                                              device_id, seed=seed)
        crop_size = _pair(crop_size)
        if mean is None:
            mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
        if std is None:
            std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
        if num_samples is None:
            initial_fill = 512
        else:
            initial_fill = min(512, num_samples)
        self.loader = ops.FileReader(file_root=file_root, file_list=file_list,
                                     random_shuffle=random_shuffle,
                                     initial_fill=initial_fill)
        self.decode = ops.HostDecoder()
        self.resize = ops.Resize(device="gpu", resize_x=256, resize_y=256)
        self.cmnorm = ops.CropMirrorNormalize(
            device="gpu", crop=list(crop_size), mean=mean, std=std)

    def define_graph(self):
        jpegs, labels = self.loader()
        images = self.decode(jpegs)
        images = self.resize(images.gpu())
        images = self.cmnorm(images)
        return images, labels

class DaliConverter(object):

    def __init__(self, mean, crop_size):
        self.mean = mean
        self.crop_size = crop_size

        ch_mean = np.average(mean, axis=(1, 2))
        perturbation = (mean - ch_mean.reshape(3, 1, 1)) / 255.0
        perturbation = perturbation[:3, :crop_size, :crop_size].astype(
            np.float32)
        self.perturbation = perturbation.reshape(1, 3, crop_size, crop_size)

    def __call__(self, inputs, device=None):
        """Convert DALI arrays to Numpy/CuPy arrays"""

        xp = cuda.get_array_module(self.perturbation)
        if xp is not cuda.cupy:
            self.perturbation = cuda.to_gpu(self.perturbation, device)

        outputs = []
        for i in range(len(inputs)):
            x = inputs[i].as_tensor()
            if (isinstance(x, dali.backend_impl.TensorCPU)):
                x = np.array(x)
                if x.ndim == 2 and x.shape[1] == 1:
                    x = x.squeeze(axis=1)
                if device is not None and device >= 0:
                    x = cuda.to_gpu(x, device)
            elif (isinstance(x, dali.backend_impl.TensorGPU)):
                x_cupy = cuda.cupy.empty(shape=x.shape(), dtype=x.dtype())
                # Synchronization is necessary here to avoid data corruption
                # because DALI and CuPy will use different CUDA streams.
                cuda.cupy.cuda.runtime.deviceSynchronize()
                # copy data from DALI array to CuPy array
                x.copy_to_external(ctypes.c_void_p(x_cupy.data.ptr))
                cuda.cupy.cuda.runtime.deviceSynchronize()
                x = x_cupy
                if self.perturbation is not None:
                    x = x - self.perturbation
                if device is not None and device < 0:
                    x = cuda.to_cpu(x)
            else:
                raise ValueError('Unexpected object')
            outputs.append(x)
        return tuple(outputs)

def dali_converter(inputs, device=None):
    """Convert DALI arrays to Numpy/CuPy arrays"""

    outputs = []
    for i in range(len(inputs)):
        x = inputs[i].as_tensor()
        if (isinstance(x, dali.backend_impl.TensorCPU)):
            x = np.array(x)
            if x.ndim == 2 and x.shape[1] == 1:
                x = x.squeeze(axis=1)
            if device is not None and device >= 0:
                x = cuda.to_gpu(x, device)
        elif (isinstance(x, dali.backend_impl.TensorGPU)):
            x_cupy = cuda.cupy.empty(shape=x.shape(), dtype=x.dtype())
            # Synchronization is necessary here to avoid data corruption
            # because DALI and CuPy will use different CUDA streams.
            cuda.cupy.cuda.runtime.deviceSynchronize()
            # copy data from DALI array to CuPy array
            x.copy_to_external(ctypes.c_void_p(x_cupy.data.ptr))
            cuda.cupy.cuda.runtime.deviceSynchronize()
            x = x_cupy
            if device is not None and device < 0:
                x = cuda.to_cpu(x)
        else:
            raise ValueError('Unexpected object')
        outputs.append(x)
    return tuple(outputs)
<commit_msg>Use the default dtype in DALI converter<commit_after>import numpy as np

try:
    from nvidia import dali
    from nvidia.dali import ops
    from nvidia.dali import pipeline
    _dali_available = True
except ImportError:
    class pipeline(object):
        Pipeline = object
        pass
    _dali_available = False

from chainer.backends import cuda
import ctypes

def _pair(x):
    if hasattr(x, '__getitem__'):
        return x
    return x, x

class DaliPipelineTrain(pipeline.Pipeline):

    def __init__(self, file_list, file_root, crop_size,
                 batch_size, num_threads, device_id,
                 random_shuffle=True, seed=-1, mean=None, std=None,
                 num_samples=None):
        super(DaliPipelineTrain, self).__init__(batch_size, num_threads,
                                                device_id, seed=seed)
        crop_size = _pair(crop_size)
        if mean is None:
            mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
        if std is None:
            std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
        if num_samples is None:
            initial_fill = 4096
        else:
            initial_fill = min(4096, num_samples)
        self.loader = ops.FileReader(file_root=file_root, file_list=file_list,
                                     random_shuffle=random_shuffle,
                                     initial_fill=initial_fill)
        self.decode = ops.HostDecoder()
        self.resize = ops.Resize(device="gpu", resize_x=256, resize_y=256)
        # self.hue = ops.Hue(device="gpu")
        # self.bright = ops.Brightness(device="gpu")
        # self.cntrst = ops.Contrast(device="gpu")
        # self.rotate = ops.Rotate(device="gpu")
        # self.jitter = ops.Jitter(device="gpu")
        random_area = (crop_size[0] / 256) * (crop_size[1] / 256)
        random_area = _pair(random_area)
        random_aspect_ratio = _pair(1.0)
        self.rrcrop = ops.RandomResizedCrop(
            device="gpu", size=crop_size, random_area=random_area,
            random_aspect_ratio=random_aspect_ratio)
        self.cmnorm = ops.CropMirrorNormalize(
            device="gpu", crop=list(crop_size), mean=mean, std=std)
        self.coin = ops.CoinFlip(probability=0.5)

    def define_graph(self):
        jpegs, labels = self.loader()
        images = self.decode(jpegs)
        images = self.resize(images.gpu())
        # images = self.hue(images, hue=ops.Uniform(range=(-3.0, 3.0))())
        # images = self.bright(images,
        #                      brightness=ops.Uniform(range=(0.9, 1.1))())
        # images = self.cntrst(images,
        #                      contrast=ops.Uniform(range=(0.9, 1.1))())
        # images = self.rotate(images,
        #                      angle=ops.Uniform(range=(-5.0, 5.0))())
        # images = self.jitter(images)
        images = self.rrcrop(images)
        images = self.cmnorm(images, mirror=self.coin())
        return images, labels

class DaliPipelineVal(pipeline.Pipeline):

    def __init__(self, file_list, file_root, crop_size,
                 batch_size, num_threads, device_id,
                 random_shuffle=False, seed=-1, mean=None, std=None,
                 num_samples=None):
        super(DaliPipelineVal, self).__init__(batch_size, num_threads,
                                              device_id, seed=seed)
        crop_size = _pair(crop_size)
        if mean is None:
            mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
        if std is None:
            std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
        if num_samples is None:
            initial_fill = 512
        else:
            initial_fill = min(512, num_samples)
        self.loader = ops.FileReader(file_root=file_root, file_list=file_list,
                                     random_shuffle=random_shuffle,
                                     initial_fill=initial_fill)
        self.decode = ops.HostDecoder()
        self.resize = ops.Resize(device="gpu", resize_x=256, resize_y=256)
        self.cmnorm = ops.CropMirrorNormalize(
            device="gpu", crop=list(crop_size), mean=mean, std=std)

    def define_graph(self):
        jpegs, labels = self.loader()
        images = self.decode(jpegs)
        images = self.resize(images.gpu())
        images = self.cmnorm(images)
        return images, labels

class DaliConverter(object):

    def __init__(self, mean, crop_size):
        self.mean = mean
        self.crop_size = crop_size

        ch_mean = np.average(mean, axis=(1, 2))
        perturbation = (mean - ch_mean.reshape(3, 1, 1)) / 255.0
        perturbation = perturbation[:3, :crop_size, :crop_size].astype(
            chainer.get_dtype())
        self.perturbation = perturbation.reshape(1, 3, crop_size, crop_size)

    def __call__(self, inputs, device=None):
        """Convert DALI arrays to Numpy/CuPy arrays"""

        xp = cuda.get_array_module(self.perturbation)
        if xp is not cuda.cupy:
            self.perturbation = cuda.to_gpu(self.perturbation, device)

        outputs = []
        for i in range(len(inputs)):
            x = inputs[i].as_tensor()
            if (isinstance(x, dali.backend_impl.TensorCPU)):
                x = np.array(x)
                if x.ndim == 2 and x.shape[1] == 1:
                    x = x.squeeze(axis=1)
                if device is not None and device >= 0:
                    x = cuda.to_gpu(x, device)
            elif (isinstance(x, dali.backend_impl.TensorGPU)):
                x_cupy = cuda.cupy.empty(shape=x.shape(), dtype=x.dtype())
                # Synchronization is necessary here to avoid data corruption
                # because DALI and CuPy will use different CUDA streams.
                cuda.cupy.cuda.runtime.deviceSynchronize()
                # copy data from DALI array to CuPy array
                x.copy_to_external(ctypes.c_void_p(x_cupy.data.ptr))
                cuda.cupy.cuda.runtime.deviceSynchronize()
                x = x_cupy
                if self.perturbation is not None:
                    x = x - self.perturbation
                if device is not None and device < 0:
                    x = cuda.to_cpu(x)
            else:
                raise ValueError('Unexpected object')
            outputs.append(x)
        return tuple(outputs)

def dali_converter(inputs, device=None):
    """Convert DALI arrays to Numpy/CuPy arrays"""

    outputs = []
    for i in range(len(inputs)):
        x = inputs[i].as_tensor()
        if (isinstance(x, dali.backend_impl.TensorCPU)):
            x = np.array(x)
            if x.ndim == 2 and x.shape[1] == 1:
                x = x.squeeze(axis=1)
            if device is not None and device >= 0:
                x = cuda.to_gpu(x, device)
        elif (isinstance(x, dali.backend_impl.TensorGPU)):
            x_cupy = cuda.cupy.empty(shape=x.shape(), dtype=x.dtype())
            # Synchronization is necessary here to avoid data corruption
            # because DALI and CuPy will use different CUDA streams.
            cuda.cupy.cuda.runtime.deviceSynchronize()
            # copy data from DALI array to CuPy array
            x.copy_to_external(ctypes.c_void_p(x_cupy.data.ptr))
            cuda.cupy.cuda.runtime.deviceSynchronize()
            x = x_cupy
            if device is not None and device < 0:
                x = cuda.to_cpu(x)
        else:
            raise ValueError('Unexpected object')
        outputs.append(x)
    return tuple(outputs)
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 10:38
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module implements en Ensembl data grabber for a given Ensembl Service instance.

Some of the use cases for this module:
    1. Given a species ID, download its protein sequence data, with the option of decompressing it or not.
    2. Given a species ID, collect its GTF data, with the option of decompressing it or not.
"""

import os
# App imports
import config_manager
import toolbox
import ensembl.service
from exceptions import ConfigManagerException

# Common configuration for all instances of the download manager
__configuration_file = None
__data_download_service = None

def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def get_data_download_service():
    global __data_download_service
    if __data_download_service is None:
        __data_download_service = DataDownloadService(config_manager.read_config_from_file(__configuration_file),
                                                      __configuration_file)
        __data_download_service.post_constructor()
    return __data_download_service

# Ensembl Data Grabbing Service configuration manager
class ConfigurationManager(config_manager.ConfigurationManager):
    # Configuration keys
    _CONFIG_KEY_DATA_DOWNLOADER = 'data_downloader'
    _CONFIG_KEY_ENSEMBL_FTP = 'ensembl_ftp'
    _CONFIG_KEY_BASE_URL = 'base_url'
    _CONFIG_KEY_FOLDER_PREFIX_RELEASE = 'folder_prefix_release'
    _CONFIG_KEY_FOLDER_NAME_FASTA = 'folder_name_fasta'
    _CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES = 'folder_name_protein_sequences'
    _CONFIG_KEY_FOLDER_NAME_GTF = 'folder_name_gtf'
    _CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO = 'rewrite_local_path_ensembl_repo'

    def __init__(self, configuration_object, configuration_file):
        super(ConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        # Local Ensembl repo parent folder name
        self.__local_folder_ensembl_repo = 'ensembl'

    def _get_logger(self):
        return self.__logger

    def get_local_path_folder_ensembl_repo(self):
        """
        Get the absolute path to the local folder where we are going to store all the data from the different releases
        of Ensembl
        :return: absolute path of the local repository for Ensembl releases data
        """
        return os.path.abspath(os.path.join(config_manager.get_app_config_manager().get_folder_resources(),
                                            self.__local_folder_ensembl_repo))

    def get_ensembl_ftp_base_url(self):
        """
        Get the base URL for Ensembl FTP service, e.g. ftp://ftp.ensembl.org/pub/ .
        This parameter is defined in the configuration file used in the pipeline session for the Ensembl module.
        :return: a string with the configured Ensembl FTP base URL
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_BASE_URL]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_BASE_URL,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_prefix_ensembl_release(self):
        """
        Get the prefix for the ensembl release folder, e.g. Ensembl has been making releases in folders like
        'release-89', so the prefix would be 'release-'.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl module.
        :return: a string with the prefix for ensembl release folder name
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_PREFIX_RELEASE]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_PREFIX_RELEASE,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_fasta(self):
        """
        Get the name for the folder that contains per-species fasta data, e.g. it is used to work out the path to access
        FASTA data for a given species on the Ensembl FTP service.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_FASTA]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_FASTA,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_protein_sequences(self):
        """
        Get the name for the sub-folder that contains protein sequence data for a particular species, e.g. it is used to
        work out the path to access protein sequence data for a given species.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the sub-folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES,
                    self._get_configuration_file(),
                    str(e)))

    def is_rewrite_local_path_ensembl_repo(self):
        """
        Find out whether we are required to overwrite the local Ensembl repository or not, in case there is an existing
        one for the same release we are collecting data from.
        :return: True if we have to rewrite it, False otherwise
        """
        try:
            if self._get_configuration_object() \
                    [self._CONFIG_KEY_DATA_DOWNLOADER] \
                    [self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO] == "True":
                return True
            return False
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO,
                    self._get_configuration_file(),
                    str(e)))

class DataDownloadService:
    """
    This Service is in charge of grabbing data (download) from Ensembl to a local repository
    """

    def __init__(self, configuration_object, configuration_file):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        self._get_logger().debug("Using configuration file '{}'".format(configuration_file))
        self.__config_manager = ConfigurationManager(configuration_object, configuration_file)
        self.__local_path_ensembl_repo = None
        self.__local_path_ensembl_release = None
        # Name for the current release
        self.__ensembl_release_name = None
        # Name for the subfolder that contains per species fasta files
        self.__folder_name_fasta = None
        # Name for the subfolder of species folder that contains protein sequences files
        self.__folder_name_protein_sequences = None

    def post_constructor(self):
        """
        This is an initialization method that should be called just after building an instance of this class
        :return: no return value
        """
        self.__prepare_local_ensembl_repository()

    def __prepare_local_ensembl_repository(self):
        self._get_logger().debug("Preparing local Ensembl repository, root folder - '{}'"
                                 .format(self.get_local_path_root_ensembl_repo()))
        toolbox.check_create_folders([self.get_local_path_root_ensembl_repo()])
        self._get_logger().debug("Local path for Ensembl Release - '{}'"
                                 .format(self.get_local_path_ensembl_release()))
        if self._get_configuration_manager().is_rewrite_local_path_ensembl_repo():
            self._get_logger().debug("Creating folder in 'OVERWRITE' mode - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders_overwrite([self.get_local_path_ensembl_release()])
        else:
            self._get_logger().debug("Creating folder if it doesn't exist - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders([self.get_local_path_ensembl_release()])

    def __get_subpath_fasta_for_species(self, taxonomy_id):
        # TODO
        pass

    def __get_subpath_protein_sequence_for_species(self, taxonomy_id):
        # TODO
        pass

    def __get_subpath_genome_reference_gtf_for_species(self, taxonomy_id):
        # TODO
        pass

    def _get_logger(self):
        return self.__logger

    def _get_ensembl_service(self):
        # TODO
        pass

    def _get_configuration_manager(self):
        return self.__config_manager

    def get_local_path_root_ensembl_repo(self):
        if self.__local_path_ensembl_repo is None:
            # For improved reading and code maintenance later
            resources_folder_path = os.path.abspath(config_manager.get_app_config_manager().get_folder_resources())
            root_folder_ensembl_repo = self._get_configuration_manager().get_local_path_folder_ensembl_repo()
            self.__local_path_ensembl_repo = os.path.join(resources_folder_path, root_folder_ensembl_repo)
        return self.__local_path_ensembl_repo

    def get_local_path_ensembl_release(self):
        if self.__local_path_ensembl_release is None:
            self.__local_path_ensembl_release = os.path.join(self.get_local_path_root_ensembl_repo(),
                                                             self.get_ensembl_release_name())
        return self.__local_path_ensembl_release

    def get_ensembl_release_name(self):
        if self.__ensembl_release_name is None:
            ensembl_service = ensembl.service.get_service()
            self.__ensembl_release_name = "{}{}" \
                .format(self._get_configuration_manager().get_folder_prefix_ensembl_release(),
                        ensembl_service.get_release_number())
        return self.__ensembl_release_name

    def get_protein_sequences_for_species(self, taxonomy_id):
        # TODO
        pass

    def get_genome_reference_gtf_for_species(self, taxonomy_id):
        # TODO
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>TODOs<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 10:38
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module implements en Ensembl data grabber for a given Ensembl Service instance.

Some of the use cases for this module:
    1. Given a species ID, download its protein sequence data, with the option of decompressing it or not.
    2. Given a species ID, collect its GTF data, with the option of decompressing it or not.
"""

import os
# App imports
import config_manager
import toolbox
import ensembl.service
from exceptions import ConfigManagerException

# Common configuration for all instances of the download manager
__configuration_file = None
__data_download_service = None

def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def get_data_download_service():
    global __data_download_service
    if __data_download_service is None:
        __data_download_service = DataDownloadService(config_manager.read_config_from_file(__configuration_file),
                                                      __configuration_file)
        __data_download_service.post_constructor()
    return __data_download_service

# Ensembl Data Grabbing Service configuration manager
class ConfigurationManager(config_manager.ConfigurationManager):
    # Configuration keys
    _CONFIG_KEY_DATA_DOWNLOADER = 'data_downloader'
    _CONFIG_KEY_ENSEMBL_FTP = 'ensembl_ftp'
    _CONFIG_KEY_BASE_URL = 'base_url'
    _CONFIG_KEY_FOLDER_PREFIX_RELEASE = 'folder_prefix_release'
    _CONFIG_KEY_FOLDER_NAME_FASTA = 'folder_name_fasta'
    _CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES = 'folder_name_protein_sequences'
    _CONFIG_KEY_FOLDER_NAME_GTF = 'folder_name_gtf'
    _CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO = 'rewrite_local_path_ensembl_repo'

    def __init__(self, configuration_object, configuration_file):
        super(ConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        # Local Ensembl repo parent folder name
        self.__local_folder_ensembl_repo = 'ensembl'

    def _get_logger(self):
        return self.__logger

    def get_local_path_folder_ensembl_repo(self):
        """
        Get the absolute path to the local folder where we are going to store all the data from the different releases
        of Ensembl
        :return: absolute path of the local repository for Ensembl releases data
        """
        return os.path.abspath(os.path.join(config_manager.get_app_config_manager().get_folder_resources(),
                                            self.__local_folder_ensembl_repo))

    def get_ensembl_ftp_base_url(self):
        """
        Get the base URL for Ensembl FTP service, e.g. ftp://ftp.ensembl.org/pub/ .
        This parameter is defined in the configuration file used in the pipeline session for the Ensembl module.
        :return: a string with the configured Ensembl FTP base URL
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_BASE_URL]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_BASE_URL,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_prefix_ensembl_release(self):
        """
        Get the prefix for the ensembl release folder, e.g. Ensembl has been making releases in folders like
        'release-89', so the prefix would be 'release-'.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl module.
        :return: a string with the prefix for ensembl release folder name
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_PREFIX_RELEASE]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_PREFIX_RELEASE,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_fasta(self):
        """
        Get the name for the folder that contains per-species fasta data, e.g. it is used to work out the path to access
        FASTA data for a given species on the Ensembl FTP service.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_FASTA]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_FASTA,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_protein_sequences(self):
        """
        Get the name for the sub-folder that contains protein sequence data for a particular species, e.g. it is used to
        work out the path to access protein sequence data for a given species.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the sub-folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES,
                    self._get_configuration_file(),
                    str(e)))

    def is_rewrite_local_path_ensembl_repo(self):
        """
        Find out whether we are required to overwrite the local Ensembl repository or not, in case there is an existing
        one for the same release we are collecting data from.
        :return: True if we have to rewrite it, False otherwise
        """
        try:
            if self._get_configuration_object() \
                    [self._CONFIG_KEY_DATA_DOWNLOADER] \
                    [self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO] == "True":
                return True
            return False
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO,
                    self._get_configuration_file(),
                    str(e)))

class DataDownloadService:
    """
    This Service is in charge of grabbing data (download) from Ensembl to a local repository
    """

    def __init__(self, configuration_object, configuration_file):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        self._get_logger().debug("Using configuration file '{}'".format(configuration_file))
        self.__config_manager = ConfigurationManager(configuration_object, configuration_file)
        self.__local_path_ensembl_repo = None
        self.__local_path_ensembl_release = None
        # Name for the current release
        self.__ensembl_release_name = None
        # Name for the subfolder that contains per species fasta files
        self.__folder_name_fasta = None
        # Name for the subfolder of species folder that contains protein sequences files
        self.__folder_name_protein_sequences = None

    def post_constructor(self):
        """
        This is an initialization method that should be called just after building an instance of this class
        :return: no return value
        """
        self.__prepare_local_ensembl_repository()

    def __prepare_local_ensembl_repository(self):
        self._get_logger().debug("Preparing local Ensembl repository, root folder - '{}'"
                                 .format(self.get_local_path_root_ensembl_repo()))
        toolbox.check_create_folders([self.get_local_path_root_ensembl_repo()])
        self._get_logger().debug("Local path for Ensembl Release - '{}'"
                                 .format(self.get_local_path_ensembl_release()))
        if self._get_configuration_manager().is_rewrite_local_path_ensembl_repo():
            self._get_logger().debug("Creating folder in 'OVERWRITE' mode - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders_overwrite([self.get_local_path_ensembl_release()])
        else:
            self._get_logger().debug("Creating folder if it doesn't exist - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders([self.get_local_path_ensembl_release()])
        # TODO Create 'latest' release to the ensembl release folder

    def __get_subpath_fasta_for_species(self, taxonomy_id):
        # TODO
        pass

    def __get_subpath_protein_sequence_for_species(self, taxonomy_id):
        # TODO
        pass

    def __get_subpath_genome_reference_gtf_for_species(self, taxonomy_id):
        # TODO
        pass

    def _get_logger(self):
        return self.__logger

    def _get_ensembl_service(self):
        # TODO
        pass

    def _get_configuration_manager(self):
        return self.__config_manager

    def get_local_path_root_ensembl_repo(self):
        if self.__local_path_ensembl_repo is None:
            # For improved reading and code maintenance later
            resources_folder_path = os.path.abspath(config_manager.get_app_config_manager().get_folder_resources())
            root_folder_ensembl_repo = self._get_configuration_manager().get_local_path_folder_ensembl_repo()
            self.__local_path_ensembl_repo = os.path.join(resources_folder_path, root_folder_ensembl_repo)
        return self.__local_path_ensembl_repo

    def get_local_path_ensembl_release(self):
        if self.__local_path_ensembl_release is None:
            self.__local_path_ensembl_release = os.path.join(self.get_local_path_root_ensembl_repo(),
                                                             self.get_ensembl_release_name())
        return self.__local_path_ensembl_release

    def get_ensembl_release_name(self):
        if self.__ensembl_release_name is None:
            ensembl_service = ensembl.service.get_service()
            self.__ensembl_release_name = "{}{}" \
                .format(self._get_configuration_manager().get_folder_prefix_ensembl_release(),
                        ensembl_service.get_release_number())
        return self.__ensembl_release_name

    def get_protein_sequences_for_species(self, taxonomy_id):
        # TODO
        pass

    def get_genome_reference_gtf_for_species(self, taxonomy_id):
        # TODO
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>from __future__ import print_function
import os, sys, logging, re, platform, ConfigParser
###

L = logging.getLogger("config")

###

# Defaults are stated in documentation, if you change them here, update documentation too!
config_defaults = {
	'general' : {
		'appname' : 'ramona-driven-app',
		'logdir' : '<env>',
		'include' : '<siteconf>',
		'logmaxsize': '{0}'.format(512*1024*1024), # 512Mb
		'logbackups': '3',
		'logcompress': '1'
	},
	'ramona:server' : {
		'consoleuri': 'unix://.ramona.sock',
		'consoleuri@windows': 'tcp://localhost:7788',
		'pidfile': '',
		'log': '<logdir>',
		'loglevel': 'INFO',
	},
	'ramona:console' : {
		'serveruri': 'unix://.ramona.sock',
		'serveruri@windows': 'tcp://localhost:7788',
		'history': '',
	},
	'ramona:notify' : {
		'delivery': '',
		'sender': '<user>',
		'dailyat': '09:00',
		'notify_fatal': 'now',
		'logscan_stdout': '',
		'logscan_stderr': '',
		'logscan': '',
		'stashdir': '.ramona.stash',
	},
	'ramona:httpfend': {
		'listenaddr': "tcp://localhost:5588",
	}

}

###

config = ConfigParser.SafeConfigParser()
config.optionxform = str # Disable default 'lowecasing' behavior of ConfigParser
config_files = []
config_includes = []

config_platform_selector = platform.system().lower()

###

def read_config(configs=None, use_env=True):
	global config
	assert len(config.sections()) == 0

	# Prepare platform selector regex
	psrg = re.compile('^(.*)@(.*)$')

	# Load config_defaults
	psdefaults = []
	for section, items in config_defaults.iteritems():
		if not config.has_section(section):
			config.add_section(section)

		for key, val in items.iteritems():
		 	r = psrg.match(key)
		 	if r is None:
				config.set(section, key, val)
			else:
				if r.group(2) != config_platform_selector: continue
				psdefaults.append((section, r.group(1), val))

	# Handle platform selectors in config_defaults
	for section, key, val in psdefaults:
		config.set(section, key, val)

	# Load configuration files
	global config_files

	if configs is not None: configs = configs[:]
	else: configs = []
	if use_env:
		# Configs from environment variables
		config_envs = os.environ.get('RAMONA_CONFIG')
		if config_envs is not None:
			for config_file in config_envs.split(os.pathsep):
				configs.append(config_file)

	for cfile in configs:
		rfile = os.path.expanduser(cfile)
		if os.path.isfile(rfile):
			config_files.append(cfile)
		config.read([rfile])

	# Handle includes ...
	appname = config.get('general','appname')
	for _ in range(100):
		includes = config.get('general','include')
		if includes == '': break
		config.set('general','include','')
		includes = includes.split(';')
		for i in xrange(len(includes)-1,-1,-1):
			include = includes[i] = includes[i].strip()
			if include == '<siteconf>':
				# These are platform specific
				siteconfs = [
					'./site.conf',
					'./{}-site.conf'.format(appname),
					'/etc/{0}.conf'.format(appname),
					'~/.{0}.conf'.format(appname),
				]
				includes[i:i+1] = siteconfs
			elif include[:1] == '<':
				print('WARNING: Unknown include fragment: {0}'.format(include), file=sys.stderr)
				continue

		for include in includes:
			rinclude = os.path.expanduser(include)
			if os.path.isfile(rinclude):
				config_includes.append(include)
				config.read([rinclude])

	else:
		raise RuntimeError("FATAL: It looks like we have loop in configuration includes!")

	# Threat platform selector alternatives
	if config_platform_selector is not None and config_platform_selector != '':
		for section in config.sections():
			for name, value in config.items(section):
				r = psrg.match(name)
				if r is None: continue
				if (r.group(2) != config_platform_selector): continue
				config.set(section, r.group(1), value)

	# Special treatment of some values
	if config.get('general', 'logdir') == '<env>':
		logdir = os.environ.get('LOGDIR')
		if logdir is None: logdir = os.curdir
		logdir = os.path.expanduser(logdir)
		config.set('general','logdir',logdir)
	elif config.get('general', 'logdir').strip()[:1] == '<':
		raise RuntimeError("FATAL: Unknown magic value in [general] logdir: '{}'".format(config.get('general', 'logdir')))

	for (sec, valname) in (("ramona:server", "consoleuri"), ("ramona:notify", "delivery")):
		if ";" in config.get(sec, valname):
			print(
				"WARNING: ';' character was found in URI: {}. Please note that ';' has been replaced '?' in Ramona 1.0. This can lead to Ramona apparent freeze during start.".format(
					config.get(sec, valname)
				),
				file=sys.stderr
			)

	stashdir = config.get('ramona:notify', 'stashdir')
	if stashdir != '<none>':
		if not os.path.isdir(stashdir):
			os.makedirs(stashdir)

###

def get_boolean(value):
	'''
	Translates string/<any-type> value into boolean value. It is kind of similar to ConfigParser.getboolean but this one is used also in different places of code
	'''
	if value is True: return True
	if value is False: return False

	value = str(value)

	if value.upper() in ('TRUE','ON','YES','1'):
		return True
	elif value.upper() in ('FALSE','OFF','NO','0'):
		return False
	else:
		raise ValueError("Invalid boolean string '{0}'' (use one of true, on, yes, false, off or no).".format(value))

###

def get_numeric_loglevel(loglevelstring):
	'''
	Translates log level given in string into numeric value.
	'''
	numeric_level = getattr(logging, loglevelstring.upper(), None)
	if not isinstance(numeric_level, int): raise ValueError('Invalid log level: {0}'.format(loglevelstring))
	return numeric_level
<commit_msg>Changing default value of 'stashdir' to <none><commit_after>from __future__ import print_function
import os, sys, logging, re, platform, ConfigParser
###

L = logging.getLogger("config")

###

# Defaults are stated in documentation, if you change them here, update documentation too!
config_defaults = {
	'general' : {
		'appname' : 'ramona-driven-app',
		'logdir' : '<env>',
		'include' : '<siteconf>',
		'logmaxsize': '{0}'.format(512*1024*1024), # 512Mb
		'logbackups': '3',
		'logcompress': '1'
	},
	'ramona:server' : {
		'consoleuri': 'unix://.ramona.sock',
		'consoleuri@windows': 'tcp://localhost:7788',
		'pidfile': '',
		'log': '<logdir>',
		'loglevel': 'INFO',
	},
	'ramona:console' : {
		'serveruri': 'unix://.ramona.sock',
		'serveruri@windows': 'tcp://localhost:7788',
		'history': '',
	},
	'ramona:notify' : {
		'delivery': '',
		'sender': '<user>',
		'dailyat': '09:00',
		'notify_fatal': 'now',
		'logscan_stdout': '',
		'logscan_stderr': '',
		'logscan': '',
		'stashdir': '<none>',
	},
	'ramona:httpfend': {
		'listenaddr': "tcp://localhost:5588",
	}

}

###

config = ConfigParser.SafeConfigParser()
config.optionxform = str # Disable default 'lowecasing' behavior of ConfigParser
config_files = []
config_includes = []

config_platform_selector = platform.system().lower()

###

def read_config(configs=None, use_env=True):
	global config
	assert len(config.sections()) == 0

	# Prepare platform selector regex
	psrg = re.compile('^(.*)@(.*)$')

	# Load config_defaults
	psdefaults = []
	for section, items in config_defaults.iteritems():
		if not config.has_section(section):
			config.add_section(section)

		for key, val in items.iteritems():
		 	r = psrg.match(key)
		 	if r is None:
				config.set(section, key, val)
			else:
				if r.group(2) != config_platform_selector: continue
				psdefaults.append((section, r.group(1), val))

	# Handle platform selectors in config_defaults
	for section, key, val in psdefaults:
		config.set(section, key, val)

	# Load configuration files
	global config_files

	if configs is not None: configs = configs[:]
	else: configs = []
	if use_env:
		# Configs from environment variables
		config_envs = os.environ.get('RAMONA_CONFIG')
		if config_envs is not None:
			for config_file in config_envs.split(os.pathsep):
				configs.append(config_file)

	for cfile in configs:
		rfile = os.path.expanduser(cfile)
		if os.path.isfile(rfile):
			config_files.append(cfile)
		config.read([rfile])

	# Handle includes ...
	appname = config.get('general','appname')
	for _ in range(100):
		includes = config.get('general','include')
		if includes == '': break
		config.set('general','include','')
		includes = includes.split(';')
		for i in xrange(len(includes)-1,-1,-1):
			include = includes[i] = includes[i].strip()
			if include == '<siteconf>':
				# These are platform specific
				siteconfs = [
					'./site.conf',
					'./{}-site.conf'.format(appname),
					'/etc/{0}.conf'.format(appname),
					'~/.{0}.conf'.format(appname),
				]
				includes[i:i+1] = siteconfs
			elif include[:1] == '<':
				print('WARNING: Unknown include fragment: {0}'.format(include), file=sys.stderr)
				continue

		for include in includes:
			rinclude = os.path.expanduser(include)
			if os.path.isfile(rinclude):
				config_includes.append(include)
				config.read([rinclude])

	else:
		raise RuntimeError("FATAL: It looks like we have loop in configuration includes!")

	# Threat platform selector alternatives
	if config_platform_selector is not None and config_platform_selector != '':
		for section in config.sections():
			for name, value in config.items(section):
				r = psrg.match(name)
				if r is None: continue
				if (r.group(2) != config_platform_selector): continue
				config.set(section, r.group(1), value)

	# Special treatment of some values
	if config.get('general', 'logdir') == '<env>':
		logdir = os.environ.get('LOGDIR')
		if logdir is None: logdir = os.curdir
		logdir = os.path.expanduser(logdir)
		config.set('general','logdir',logdir)
	elif config.get('general', 'logdir').strip()[:1] == '<':
		raise RuntimeError("FATAL: Unknown magic value in [general] logdir: '{}'".format(config.get('general', 'logdir')))

	for (sec, valname) in (("ramona:server", "consoleuri"), ("ramona:notify", "delivery")):
		if ";" in config.get(sec, valname):
			print(
				"WARNING: ';' character was found in URI: {}. Please note that ';' has been replaced '?' in Ramona 1.0. This can lead to Ramona apparent freeze during start.".format(
					config.get(sec, valname)
				),
				file=sys.stderr
			)

	stashdir = config.get('ramona:notify', 'stashdir')
	if stashdir != '<none>':
		if not os.path.isdir(stashdir):
			os.makedirs(stashdir)

###

def get_boolean(value):
	'''
	Translates string/<any-type> value into boolean value. It is kind of similar to ConfigParser.getboolean but this one is used also in different places of code
	'''
	if value is True: return True
	if value is False: return False

	value = str(value)

	if value.upper() in ('TRUE','ON','YES','1'):
		return True
	elif value.upper() in ('FALSE','OFF','NO','0'):
		return False
	else:
		raise ValueError("Invalid boolean string '{0}'' (use one of true, on, yes, false, off or no).".format(value))

###

def get_numeric_loglevel(loglevelstring):
	'''
	Translates log level given in string into numeric value.
	'''
	numeric_level = getattr(logging, loglevelstring.upper(), None)
	if not isinstance(numeric_level, int): raise ValueError('Invalid log level: {0}'.format(loglevelstring))
	return numeric_level
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 10:38
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module implements en Ensembl data grabber for a given Ensembl Service instance
"""

# App imports
import config_manager
from exceptions import ConfigManagerException
from ensembl.service import Service as EnsemblService

# Common configuration for all instances of the download manager
__configuration_file = None
__configuration_manager = None

def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = ConfigurationManager(config_manager.read_config_from_file(__configuration_file),
                                                       __configuration_file)
    return __configuration_manager

# Ensembl Download Manager configuration manager
class ConfigurationManager(config_manager.ConfigurationManager):
    pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>config manager keep file and config object<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 10:38
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module implements en Ensembl data grabber for a given Ensembl Service instance
"""

# App imports
import config_manager
from exceptions import ConfigManagerException
from ensembl.service import Service as EnsemblService

# Common configuration for all instances of the download manager
__configuration_file = None
__configuration_manager = None

def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = ConfigurationManager(config_manager.read_config_from_file(__configuration_file),
                                                       __configuration_file)
    return __configuration_manager

# Ensembl Download Manager configuration manager
class ConfigurationManager(config_manager.ConfigurationManager):
    def __init__(self, configuration_object, configuration_file):
        super(ConfigurationManager, self).__init__(configuration_object, configuration_file)

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 10:38
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module implements en Ensembl data grabber for a given Ensembl Service instance.

Some of the use cases for this module:
    1. Given a species ID, download its protein sequence data, with the option of decompressing it or not.
    2. Given a species ID, collect its GTF data, with the option of decompressing it or not.
"""

import os
# App imports
import config_manager
import toolbox
import ensembl.service
from exceptions import ConfigManagerException

# Common configuration for all instances of the download manager
__configuration_file = None
__data_download_service = None

def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def get_data_download_service():
    global __data_download_service
    if __data_download_service is None:
        __data_download_service = DataDownloadService(config_manager.read_config_from_file(__configuration_file),
                                                      __configuration_file)
        __data_download_service.post_constructor()
    return __data_download_service

# Ensembl Data Grabbing Service configuration manager
class ConfigurationManager(config_manager.ConfigurationManager):
    # Configuration keys
    _CONFIG_KEY_DATA_DOWNLOADER = 'data_downloader'
    # Ensembl FTP section
    _CONFIG_KEY_ENSEMBL_FTP = 'ensembl_ftp'
    _CONFIG_KEY_BASE_URL = 'base_url'
    _CONFIG_KEY_FOLDER_PREFIX_RELEASE = 'folder_prefix_release'
    _CONFIG_KEY_FOLDER_NAME_FASTA = 'folder_name_fasta'
    _CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES = 'folder_name_protein_sequences'
    _CONFIG_KEY_FOLDER_NAME_GTF = 'folder_name_gtf'
    # Rewrite option
    _CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO = 'rewrite_local_path_ensembl_repo'
    # Ensembl file names
    _CONFIG_KEY_ENSEMBL_FILE_NAMES = 'ensembl_file_names'
    _CONFIG_KEY_PROTEIN_SEQUENCE_FILE = 'protein_sequence_file'
    _CONFIG_KEY_FILE_TYPE = 'file_type'
    _CONFIG_KEY_FILE_SUFFIXES = 'file_suffixes'
    _CONFIG_KEY_FILE_EXTENSION = 'file_extension'

    def __init__(self, configuration_object, configuration_file):
        super(ConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        # Local Ensembl repo parent folder name
        self.__local_folder_ensembl_repo = 'ensembl'

    def _get_logger(self):
        return self.__logger

    def get_local_path_folder_ensembl_repo(self):
        """
        Get the absolute path to the local folder where we are going to store all the data from the different releases
        of Ensembl
        :return: absolute path of the local repository for Ensembl releases data
        """
        return os.path.abspath(os.path.join(config_manager.get_app_config_manager().get_folder_resources(),
                                            self.__local_folder_ensembl_repo))

    def get_ensembl_ftp_base_url(self):
        """
        Get the base URL for Ensembl FTP service, e.g. ftp://ftp.ensembl.org/pub/ .
        This parameter is defined in the configuration file used in the pipeline session for the Ensembl module.
        :return: a string with the configured Ensembl FTP base URL
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_BASE_URL]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_BASE_URL,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_prefix_ensembl_release(self):
        """
        Get the prefix for the ensembl release folder, e.g. Ensembl has been making releases in folders like
        'release-89', so the prefix would be 'release-'.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl module.
        :return: a string with the prefix for ensembl release folder name
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_PREFIX_RELEASE]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_PREFIX_RELEASE,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_fasta(self):
        """
        Get the name for the folder that contains per-species fasta data, e.g. it is used to work out the path to access
        FASTA data for a given species on the Ensembl FTP service.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_FASTA]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_FASTA,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_protein_sequences(self):
        """
        Get the name for the sub-folder that contains protein sequence data for a particular species, e.g. it is used to
        work out the path to access protein sequence data for a given species.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the sub-folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_gtf(self):
        """
        Get the name for the sub-folder that contains GTF data for a particular species, e.g. it is used to work out the
        path to access GTF data for a given species, to download it from Ensembl.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name fo the sub-folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_GTF]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_GTF,
                    self._get_configuration_file(),
                    str(e)))

    def get_ensembl_protein_sequence_file_type(self):
        # TODO
        pass

    def get_ensembl_protein_sequence_file_suffixes(self):
        # TODO
        pass

    def is_rewrite_local_path_ensembl_repo(self):
        """
        Find out whether we are required to overwrite the local Ensembl repository or not, in case there is an existing
        one for the same release we are collecting data from.
        :return: True if we have to rewrite it, False otherwise
        """
        try:
            if self._get_configuration_object() \
                    [self._CONFIG_KEY_DATA_DOWNLOADER] \
                    [self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO] == "True":
                return True
            return False
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO,
                    self._get_configuration_file(),
                    str(e)))

class DataDownloadService:
    """
    This Service is in charge of grabbing data (download) from Ensembl to a local repository
    """

    def __init__(self, configuration_object, configuration_file):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        self._get_logger().debug("Using configuration file '{}'".format(configuration_file))
        self.__config_manager = ConfigurationManager(configuration_object, configuration_file)
        self.__local_path_ensembl_repo = None
        self.__local_path_ensembl_release = None
        # Name for the current release
        self.__ensembl_release_name = None
        # Name for the subfolder that contains per species fasta files
        self.__folder_name_fasta = None
        # Name for the subfolder of species folder that contains protein sequences files
        self.__folder_name_protein_sequences = None

    def post_constructor(self):
        """
        This is an initialization method that should be called just after building an instance of this class
        :return: no return value
        """
        self.__prepare_local_ensembl_repository()

    def __prepare_local_ensembl_repository(self):
        self._get_logger().debug("Preparing local Ensembl repository, root folder - '{}'"
                                 .format(self.get_local_path_root_ensembl_repo()))
        toolbox.check_create_folders([self.get_local_path_root_ensembl_repo()])
        self._get_logger().debug("Local path for Ensembl Release - '{}'"
                                 .format(self.get_local_path_ensembl_release()))
        if self._get_configuration_manager().is_rewrite_local_path_ensembl_repo():
            self._get_logger().debug("Creating folder in 'OVERWRITE' mode - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders_overwrite([self.get_local_path_ensembl_release()])
        else:
            self._get_logger().debug("Creating folder if it doesn't exist - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders([self.get_local_path_ensembl_release()])
        toolbox.create_latest_symlink(self.get_local_path_ensembl_release())

    def __get_subpath_fasta_for_species(self, taxonomy_id):
        # The subpath is fasta/species.name
        return "{}/{}".format(self._get_configuration_manager().get_folder_name_fasta(),
                              self._get_ensembl_service()
                                 .get_species_data_service()
                                 .get_species_entry_for_taxonomy_id(taxonomy_id)
                                 .get_name())

    def __get_subpath_protein_sequence_for_species(self, taxonomy_id):
        # The subpath is fasta/species.name/pep
        return "{}/{}".format(self.__get_subpath_fasta_for_species(taxonomy_id),
                              self._get_configuration_manager().get_folder_name_protein_sequences())

    def __get_subpath_genome_reference_gtf_for_species(self, taxonomy_id):
        # The subpath is gtf/species.name
        return "{}/{}".format(self._get_configuration_manager().get_folder_name_gtf(),
                              self._get_ensembl_service()
                              .get_species_data_service()
                              .get_species_entry_for_taxonomy_id(taxonomy_id)
                              .get_name()
                              )

    def _get_logger(self):
        return self.__logger

    @staticmethod
    def _get_ensembl_service():
        return ensembl.service.get_service()

    def _get_configuration_manager(self):
        return self.__config_manager

    def get_local_path_root_ensembl_repo(self):
        if self.__local_path_ensembl_repo is None:
            # For improved reading and code maintenance later
            resources_folder_path = os.path.abspath(config_manager.get_app_config_manager().get_folder_resources())
            root_folder_ensembl_repo = self._get_configuration_manager().get_local_path_folder_ensembl_repo()
            self.__local_path_ensembl_repo = os.path.join(resources_folder_path, root_folder_ensembl_repo)
        return self.__local_path_ensembl_repo

    def get_local_path_ensembl_release(self):
        if self.__local_path_ensembl_release is None:
            self.__local_path_ensembl_release = os.path.join(self.get_local_path_root_ensembl_repo(),
                                                             self.get_ensembl_release_name())
        return self.__local_path_ensembl_release

    def get_ensembl_release_name(self):
        if self.__ensembl_release_name is None:
            ensembl_service = ensembl.service.get_service()
            self.__ensembl_release_name = "{}{}" \
                .format(self._get_configuration_manager().get_folder_prefix_ensembl_release(),
                        ensembl_service.get_release_number())
        return self.__ensembl_release_name

    def _get_protein_sequence_ensembl_file_name_for_species(self, taxonomy_id):
        # TODO
        # The name for a protein sequence file in Ensembl is
        # <species_name_with_first_capital_letter>.<species_assembly>.<file_type e.g. 'pep'>.[all,abinitio].fa
        # e.g. Homo_sapiens.GRCh38.pep.all.fa, on Ensembl will have the extension .gz, because it is a compressed file
        pass

    def get_protein_sequences_for_species(self, taxonomy_id):
        # TODO
        # Work out the file names for the data to retrieve from Ensembl
        # Work out their path in the local repository
        # Check if they already exist locally
        # If not, work out their remote path on Ensembl FTP
        # Retrieve the files
        pass

    def _get_genome_reference_gtf_ensembl_file_name_for_species(self, taxonomy_id):
        # TODO
        pass

    def get_genome_reference_gtf_for_species(self, taxonomy_id):
        # TODO
        # Work out the file names for the data to retrieve from Ensembl
        # Work out their path in the local repository
        # Check if they already exist locally
        # If not, work out their remote path on Ensembl FTP
        # Retrieve the files
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>get_ensembl_protein_sequence_file_extension placeholder<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 10:38
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module implements en Ensembl data grabber for a given Ensembl Service instance.

Some of the use cases for this module:
    1. Given a species ID, download its protein sequence data, with the option of decompressing it or not.
    2. Given a species ID, collect its GTF data, with the option of decompressing it or not.
"""

import os
# App imports
import config_manager
import toolbox
import ensembl.service
from exceptions import ConfigManagerException

# Common configuration for all instances of the download manager
__configuration_file = None
__data_download_service = None

def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def get_data_download_service():
    global __data_download_service
    if __data_download_service is None:
        __data_download_service = DataDownloadService(config_manager.read_config_from_file(__configuration_file),
                                                      __configuration_file)
        __data_download_service.post_constructor()
    return __data_download_service

# Ensembl Data Grabbing Service configuration manager
class ConfigurationManager(config_manager.ConfigurationManager):
    # Configuration keys
    _CONFIG_KEY_DATA_DOWNLOADER = 'data_downloader'
    # Ensembl FTP section
    _CONFIG_KEY_ENSEMBL_FTP = 'ensembl_ftp'
    _CONFIG_KEY_BASE_URL = 'base_url'
    _CONFIG_KEY_FOLDER_PREFIX_RELEASE = 'folder_prefix_release'
    _CONFIG_KEY_FOLDER_NAME_FASTA = 'folder_name_fasta'
    _CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES = 'folder_name_protein_sequences'
    _CONFIG_KEY_FOLDER_NAME_GTF = 'folder_name_gtf'
    # Rewrite option
    _CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO = 'rewrite_local_path_ensembl_repo'
    # Ensembl file names
    _CONFIG_KEY_ENSEMBL_FILE_NAMES = 'ensembl_file_names'
    _CONFIG_KEY_PROTEIN_SEQUENCE_FILE = 'protein_sequence_file'
    _CONFIG_KEY_FILE_TYPE = 'file_type'
    _CONFIG_KEY_FILE_SUFFIXES = 'file_suffixes'
    _CONFIG_KEY_FILE_EXTENSION = 'file_extension'

    def __init__(self, configuration_object, configuration_file):
        super(ConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        # Local Ensembl repo parent folder name
        self.__local_folder_ensembl_repo = 'ensembl'

    def _get_logger(self):
        return self.__logger

    def get_local_path_folder_ensembl_repo(self):
        """
        Get the absolute path to the local folder where we are going to store all the data from the different releases
        of Ensembl
        :return: absolute path of the local repository for Ensembl releases data
        """
        return os.path.abspath(os.path.join(config_manager.get_app_config_manager().get_folder_resources(),
                                            self.__local_folder_ensembl_repo))

    def get_ensembl_ftp_base_url(self):
        """
        Get the base URL for Ensembl FTP service, e.g. ftp://ftp.ensembl.org/pub/ .
        This parameter is defined in the configuration file used in the pipeline session for the Ensembl module.
        :return: a string with the configured Ensembl FTP base URL
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_BASE_URL]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_BASE_URL,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_prefix_ensembl_release(self):
        """
        Get the prefix for the ensembl release folder, e.g. Ensembl has been making releases in folders like
        'release-89', so the prefix would be 'release-'.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl module.
        :return: a string with the prefix for ensembl release folder name
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_PREFIX_RELEASE]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', because of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_PREFIX_RELEASE,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_fasta(self):
        """
        Get the name for the folder that contains per-species fasta data, e.g. it is used to work out the path to access
        FASTA data for a given species on the Ensembl FTP service.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_FASTA]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_FASTA,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_protein_sequences(self):
        """
        Get the name for the sub-folder that contains protein sequence data for a particular species, e.g. it is used to
        work out the path to access protein sequence data for a given species.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name of the sub-folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_PROTEIN_SEQUENCES,
                    self._get_configuration_file(),
                    str(e)))

    def get_folder_name_gtf(self):
        """
        Get the name for the sub-folder that contains GTF data for a particular species, e.g. it is used to work out the
        path to access GTF data for a given species, to download it from Ensembl.

        This parameter is specified in the configuration file that is used in the pipeline session for the Ensembl
        module.
        :return: name fo the sub-folder as set in the configuration file
        """
        try:
            return self._get_configuration_object() \
                [self._CONFIG_KEY_DATA_DOWNLOADER] \
                [self._CONFIG_KEY_ENSEMBL_FTP] \
                [self._CONFIG_KEY_FOLDER_NAME_GTF]
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_ENSEMBL_FTP,
                    self._CONFIG_KEY_FOLDER_NAME_GTF,
                    self._get_configuration_file(),
                    str(e)))

    def get_ensembl_protein_sequence_file_type(self):
        # TODO
        pass

    def get_ensembl_protein_sequence_file_suffixes(self):
        # TODO
        pass

    def get_ensembl_protein_sequence_file_extension(self):
        # TODO
        pass

    def is_rewrite_local_path_ensembl_repo(self):
        """
        Find out whether we are required to overwrite the local Ensembl repository or not, in case there is an existing
        one for the same release we are collecting data from.
        :return: True if we have to rewrite it, False otherwise
        """
        try:
            if self._get_configuration_object() \
                    [self._CONFIG_KEY_DATA_DOWNLOADER] \
                    [self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO] == "True":
                return True
            return False
        except Exception as e:
            raise ConfigManagerException(
                "MISSING configuration information '{}.{}' in configuration file '{}', becuase of '{}'".format(
                    self._CONFIG_KEY_DATA_DOWNLOADER,
                    self._CONFIG_KEY_REWRITE_LOCAL_PATH_ENSEMBL_REPO,
                    self._get_configuration_file(),
                    str(e)))

class DataDownloadService:
    """
    This Service is in charge of grabbing data (download) from Ensembl to a local repository
    """

    def __init__(self, configuration_object, configuration_file):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)
        self._get_logger().debug("Using configuration file '{}'".format(configuration_file))
        self.__config_manager = ConfigurationManager(configuration_object, configuration_file)
        self.__local_path_ensembl_repo = None
        self.__local_path_ensembl_release = None
        # Name for the current release
        self.__ensembl_release_name = None
        # Name for the subfolder that contains per species fasta files
        self.__folder_name_fasta = None
        # Name for the subfolder of species folder that contains protein sequences files
        self.__folder_name_protein_sequences = None

    def post_constructor(self):
        """
        This is an initialization method that should be called just after building an instance of this class
        :return: no return value
        """
        self.__prepare_local_ensembl_repository()

    def __prepare_local_ensembl_repository(self):
        self._get_logger().debug("Preparing local Ensembl repository, root folder - '{}'"
                                 .format(self.get_local_path_root_ensembl_repo()))
        toolbox.check_create_folders([self.get_local_path_root_ensembl_repo()])
        self._get_logger().debug("Local path for Ensembl Release - '{}'"
                                 .format(self.get_local_path_ensembl_release()))
        if self._get_configuration_manager().is_rewrite_local_path_ensembl_repo():
            self._get_logger().debug("Creating folder in 'OVERWRITE' mode - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders_overwrite([self.get_local_path_ensembl_release()])
        else:
            self._get_logger().debug("Creating folder if it doesn't exist - '{}'"
                                     .format(self.get_local_path_ensembl_release()))
            toolbox.check_create_folders([self.get_local_path_ensembl_release()])
        toolbox.create_latest_symlink(self.get_local_path_ensembl_release())

    def __get_subpath_fasta_for_species(self, taxonomy_id):
        # The subpath is fasta/species.name
        return "{}/{}".format(self._get_configuration_manager().get_folder_name_fasta(),
                              self._get_ensembl_service()
                                 .get_species_data_service()
                                 .get_species_entry_for_taxonomy_id(taxonomy_id)
                                 .get_name())

    def __get_subpath_protein_sequence_for_species(self, taxonomy_id):
        # The subpath is fasta/species.name/pep
        return "{}/{}".format(self.__get_subpath_fasta_for_species(taxonomy_id),
                              self._get_configuration_manager().get_folder_name_protein_sequences())

    def __get_subpath_genome_reference_gtf_for_species(self, taxonomy_id):
        # The subpath is gtf/species.name
        return "{}/{}".format(self._get_configuration_manager().get_folder_name_gtf(),
                              self._get_ensembl_service()
                              .get_species_data_service()
                              .get_species_entry_for_taxonomy_id(taxonomy_id)
                              .get_name()
                              )

    def _get_logger(self):
        return self.__logger

    @staticmethod
    def _get_ensembl_service():
        return ensembl.service.get_service()

    def _get_configuration_manager(self):
        return self.__config_manager

    def get_local_path_root_ensembl_repo(self):
        if self.__local_path_ensembl_repo is None:
            # For improved reading and code maintenance later
            resources_folder_path = os.path.abspath(config_manager.get_app_config_manager().get_folder_resources())
            root_folder_ensembl_repo = self._get_configuration_manager().get_local_path_folder_ensembl_repo()
            self.__local_path_ensembl_repo = os.path.join(resources_folder_path, root_folder_ensembl_repo)
        return self.__local_path_ensembl_repo

    def get_local_path_ensembl_release(self):
        if self.__local_path_ensembl_release is None:
            self.__local_path_ensembl_release = os.path.join(self.get_local_path_root_ensembl_repo(),
                                                             self.get_ensembl_release_name())
        return self.__local_path_ensembl_release

    def get_ensembl_release_name(self):
        if self.__ensembl_release_name is None:
            ensembl_service = ensembl.service.get_service()
            self.__ensembl_release_name = "{}{}" \
                .format(self._get_configuration_manager().get_folder_prefix_ensembl_release(),
                        ensembl_service.get_release_number())
        return self.__ensembl_release_name

    def _get_protein_sequence_ensembl_file_name_for_species(self, taxonomy_id):
        # TODO
        # The name for a protein sequence file in Ensembl is
        # <species_name_with_first_capital_letter>.<species_assembly>.<file_type e.g. 'pep'>.[all,abinitio].fa
        # e.g. Homo_sapiens.GRCh38.pep.all.fa, on Ensembl will have the extension .gz, because it is a compressed file
        pass

    def get_protein_sequences_for_species(self, taxonomy_id):
        # TODO
        # Work out the file names for the data to retrieve from Ensembl
        # Work out their path in the local repository
        # Check if they already exist locally
        # If not, work out their remote path on Ensembl FTP
        # Retrieve the files
        pass

    def _get_genome_reference_gtf_ensembl_file_name_for_species(self, taxonomy_id):
        # TODO
        pass

    def get_genome_reference_gtf_for_species(self, taxonomy_id):
        # TODO
        # Work out the file names for the data to retrieve from Ensembl
        # Work out their path in the local repository
        # Check if they already exist locally
        # If not, work out their remote path on Ensembl FTP
        # Retrieve the files
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>