<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

import abc
# App imports
import config_manager
import trackhub.models as trackhubs
import ensembl.service
import ensembl.data_downloader
from toolbox import general
# from toolbox.assembly import AssemblyMappingServiceFactory

# The config manager singleton is just an example that only makes sense for specialized pipeline modules, not really for
# this template
__configuration_file = None
__configuration_manager = None

def __get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = DirectorConfigurationManager(general.read_json(__configuration_file),
                                                               __configuration_file)
    return __configuration_manager

class DirectorConfigurationManager(config_manager.ConfigurationManager):
    # Command line arguments special characters
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR = ','
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR = '='

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(DirectorConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__pipeline_arguments = pipeline_arguments
        self.__pipeline_arguments_object = None
        # Logger the pythonist way
        self._logger = config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))

    def _get_pipeline_arguments(self):
        return self.__pipeline_arguments

    def _get_allowed_configuration_keys(self):
        # Allowed configuration keys set is empty by default
        return {}

    def _get_pipeline_arguments_object(self):
        if self.__pipeline_arguments_object is None:
            self.__pipeline_arguments_object = self._process_pipeline_arguments()
        return self.__pipeline_arguments_object

    def _process_pipeline_arguments(self):
        if self._get_pipeline_arguments():
            if self.__pipeline_arguments_object:
                self._logger.error("DUPLICATED CALL for processing command line arguments for this pipeline, IGNORED")
            else:
                self.__pipeline_arguments_object = {}
                self._logger.debug("Processing pipeline command line arguments")
                allowed_keys = self._get_allowed_configuration_keys()
                for command_line_parameter in self._get_pipeline_arguments().split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR):
                    key, value = command_line_parameter.split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR)
                    if key not in allowed_keys:
                        self._logger.error(
                            "INVALID KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    if key in self.__pipeline_arguments_object:
                        self._logger.error(
                            "DUPLICATED KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    self.__pipeline_arguments_object[key] = value
                    self._logger.debug("Pipeline argument '{}' parsed and set with value '{}'".format(key, value))
        else:
            self._logger.warning("This pipeline was provided with NO COMMAND LINE ARGUMENTS")
        return self.__pipeline_arguments_object

    def _get_value_for_pipeline_argument_key(self, key, default=None):
        if key in self._get_pipeline_arguments_object():
            return self._get_pipeline_arguments_object()[key]
        else:
            return default

class Director(metaclass=abc.ABCMeta):
    """
    This is the director of the pipeline
    """
    _PIPELINE_STATUS_OK = "ok"
    _PIPELINE_STATUS_FAIL = "fail"

    def __init__(self, runner_id=None):
        logger_name = __name__
        if runner_id:
            logger_name = runner_id
        self.__logger = config_manager.get_app_config_manager().get_logger_for(logger_name)
        self.pipeline_status = self._PIPELINE_STATUS_OK

    def is_pipeline_status_ok(self):
        return self.pipeline_status == self._PIPELINE_STATUS_OK

    def set_pipeline_status_fail(self):
        self.pipeline_status = self._PIPELINE_STATUS_FAIL

    def _before(self):
        """
        This method implements some logic that is run before running the main pipeline director.
        Subclasses implementing the pipeline logic, can override this method for establishing pre-pipeline logic.
        :return: it returns True if there was no problem, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        """
        Pipeline template director algorithm, it executes 'before' logic, then the 'pipeline' logic, and 'after' logic
        once the pipeline is finished
        It delegates to specific pipeline implementations the decision of flagging the pipeline as FAIL or not, but the
        pipeline will only be reported as 'successful' after the successful execution of all its stages. This gives the
        developer enough flexibility to have failing stages but still be able to keep going with the pipeline, alghough
        not being considered a complete success. Also, it delegates to subclasses the responsibility of deciding when
        it is not acceptable to keep running the pipeline, which makes sense, as it is their responsibility.
        :return: True if the pipeline has been successful, False otherwise
        """
        result = True
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            result = result and False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            result = result and False
        if not self._after():
            self._get_logger().error("The logic executed AFTER running the pipeline has FAILED")
            result = result and False
        if result:
            self._get_logger().info("SUCCESSFUL Pipeline execution")
        return result

    def _run_pipeline(self):
        """
        This abstract method must be implemented by the subclasses with the strategy / director logic that drives the
        pipeline they are performing
        :return: True if the pipeline is successful, False otherwise
        """
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        """
        This method implements the logic that must be executed after the pipeline workflow has run.
        Subclasses implementing the pipeline logic, can override this method for establishing post-pipeline logic.
        :return: True if success, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        """
        Get the logger to be used for logging messages
        :return: it returns the logger set for the current instance of the class
        """
        return self.__logger

    def _set_logger(self, new_logger):
        """
        This method allows subclasses to change the default logger created when instantiating an implementation of this
        abstract class.
        :param new_logger: new logger to set up as the default logger for a particular implementation subclass instance
        :return: the newly set logger
        """
        self.__logger = new_logger
        return self._get_logger()

class PogoBasedPipelineDirector(Director):
    """
    Abstract class pipeline director class for those pipelines running PoGo in PRIDE
    """

    def __init__(self, runner_id=None):
        super().__init__(runner_id)

    def _get_pogo_protein_sequence_file_path_for_taxonomy(self, taxonomy_id):
        """
        This is a helper method that, given a taxonomy, it finds the protein sequence file to use as a PoGo parameter
        :param taxonomy_id: ncbi taxonomy ID
        :return: the protein sequence file path if found, None otherwise
        """
        # Get an instance of the Ensembl data downloader
        ensembl_downloader_service = ensembl.data_downloader.get_data_download_service()
        # Get Protein Sequence file from Ensembl for this taxonomy, only the "*all*" kind
        protein_sequence_files = ensembl_downloader_service.get_protein_sequences_for_species(taxonomy_id)
        # Checking on whether we can find a protein sequence or not for the given taxonomy will tell us if the
        # taxonomy is on Ensembl or not
        if not protein_sequence_files:
            return None
        pogo_parameter_protein_sequence_file_path = None
        for file_name, file_path in protein_sequence_files:
            self._get_logger().debug("Scanning protein sequence file name '{}'".format(file_name))
            if "all" in file_name:
                pogo_parameter_protein_sequence_file_path = file_path
                break
        if not pogo_parameter_protein_sequence_file_path:
            self._get_logger().error("File 'pep all' containing all sequences NOT FOUND!")
            return None
        return pogo_parameter_protein_sequence_file_path

    def _get_pogo_gtf_file_path_for_taxonomy(self, taxonomy_id):
        """
        This is a helper method that, given a taxonomy ID, it finds out which GTF file should be used for running PoGo
        :param taxonomy_id: ncbi taxonomy ID
        :return: the GTF file path if found, None otherwise
        """
        # Get an instance of the Ensembl data downloader
        ensembl_downloader_service = ensembl.data_downloader.get_data_download_service()
        gtf_files = ensembl_downloader_service.get_genome_reference_for_species(taxonomy_id)
        # For PoGo, we will use the GTF file that has no suffixes, thus, it will be the shortest file name
        pogo_parameter_gtf_file_name = None
        pogo_parameter_gtf_file_path = None
        for file_name, file_path in gtf_files:
            if (not pogo_parameter_gtf_file_name) \
                    or (len(pogo_parameter_gtf_file_name) > len(file_name)):
                pogo_parameter_gtf_file_name = file_name
                pogo_parameter_gtf_file_path = file_path
        return pogo_parameter_gtf_file_path

class TrackhubCreationDirector:
    """
    This is the abstract base class for those pipelines devoted to the creation of a trackhub
    """

    def __init__(self):
        # TODO - Find a way to remove the logger and you have a Mix-in, i.e. legal multiple inheritance, hahaha!
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))

    def _get_trackhub_builder(self, trackhub_descriptor):
        # TODO - Implement more complex logic for instantiating a trackhub builder, e.g. setting up the trackhub
        return trackhubs.SimpleTrackHubBuilder(trackhub_descriptor)

    @abc.abstractmethod
    def _get_trackhub_descriptor(self):
        ...

    @abc.abstractmethod
    def _get_trackhub_exporter(self):
        ...

    @abc.abstractmethod
    def _process_assemblies(self, trackhub_builder):
        ...

    @abc.abstractmethod
    def _prepare_trackhub_destination_folder(self, trackhub_exporter):
        ...

    def _create_trackhub(self):
        """
        Implements a default director for building a trackhub, it can be overriden subclasses or not used at all.
        :return: no return value
        """
        # Instantiate the trackhub builder we are going to use
        trackhub_builder = self._get_trackhub_builder(self._get_trackhub_descriptor())
        # Process and prepare the assemblies with their respective tracks
        self._process_assemblies(trackhub_builder)
        # Prepare the trackhub exporter
        trackhub_exporter = self._get_trackhub_exporter()
        self._prepare_trackhub_destination_folder(trackhub_exporter)
        # Export trackhub
        trackhub_builder.accept_exporter(trackhub_exporter)
        self._logger.info("Trackhub '{}' export to '{}' - COMPLETED"
                          .format(self._get_trackhub_descriptor().get_hub(),
                                  trackhub_exporter.track_hub_destination_folder))

class TrackhubCreationPogoBasedDirector(PogoBasedPipelineDirector, TrackhubCreationDirector):
    """
    This is a pipeline director base class for those pipelines devoted to creating trackhubs based on running PoGo on
    input data for producing .bed files
    """
    def __init__(self, runner_id=None):
        super().__init__(runner_id)
        TrackhubCreationDirector.__init__(self)
        # Override superclass logger to use this one
        logger_name = "{}.{}".format(__name__, type(self).__name__)
        if runner_id:
            logger_name = runner_id
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for(logger_name)

    @abc.abstractmethod
    def _get_pogo_results_for_input_data(self):
        ...

    @abc.abstractmethod
    def _get_trackhub_track_for_taxonomy_id(self, taxonomy_id):
        ...

    def _get_assemblies_from_pogo_results(self, trackhub_builder):
        pogo_results = self._get_pogo_results_for_input_data()
        self._get_logger().info("Processing #{} Taxonomies from PoGo results".format(len(pogo_results)))
        # assembly_mapping_service = AssemblyMappingServiceFactory.get_assembly_mapping_service()
        for taxonomy in pogo_results:
            ensembl_species_entry = ensembl.service\
                .get_service()\
                .get_species_data_service()\
                .get_species_entry_for_taxonomy_id(taxonomy)
            if not ensembl_species_entry:
                self._get_logger().error("Ensembl has NO ENTRY for taxonomy ID {} - SKIP -".format(taxonomy))
                continue
            # We can't use Ensembl Assemblies any more, they have to be translated to UCSC assemblies
            genome_assembly = ensembl_species_entry.get_assembly()
            #genome_assembly = assembly_mapping_service\
            #    .get_ucsc_assembly_for_ensembl_assembly_accession(ensembl_species_entry.get_assembly_accession())
            # Main .bed track
            bed_file_path = pogo_results[taxonomy].get_pogo_result_main_bed_file_path()
            trackhub_track_main = self._get_trackhub_track_for_taxonomy_id(taxonomy)
            trackhub_track_main.taxonomy_id = taxonomy
            trackhub_track_main.set_big_data_url(bed_file_path)
            trackhub_track_main.set_type(bed_file_path)
            # Main .bed track with PTMs
            bed_ptm_file_path = pogo_results[taxonomy].get_pogo_result_main_ptm_bed_file_path()
            trackhub_track_main_with_ptm = self._get_trackhub_track_for_taxonomy_id(taxonomy)
            trackhub_track_main_with_ptm.taxonomy_id = taxonomy
            trackhub_track_main_with_ptm.set_track("{} with PTMs".format(trackhub_track_main_with_ptm.get_track()))
            trackhub_track_main_with_ptm.set_big_data_url(bed_ptm_file_path)
            trackhub_track_main_with_ptm.set_type(bed_ptm_file_path)
            # Add tracks
            trackhub_builder.add_track_to_assembly(genome_assembly, trackhub_track_main)
            trackhub_builder.add_track_to_assembly(genome_assembly, trackhub_track_main_with_ptm)
            self._get_logger().debug("Assembly '{}', tracks added with files '({})'"
                                     .format(genome_assembly,
                                             ",".join([bed_file_path, bed_ptm_file_path])))

    def _process_assemblies(self, trackhub_builder):
        self._get_assemblies_from_pogo_results(trackhub_builder)

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>TODO tag on extension for pogo mismatch option, _get_assemblies_from_pogo_results()<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

import abc
# App imports
import config_manager
import trackhub.models as trackhubs
import ensembl.service
import ensembl.data_downloader
from toolbox import general
# from toolbox.assembly import AssemblyMappingServiceFactory

# The config manager singleton is just an example that only makes sense for specialized pipeline modules, not really for
# this template
__configuration_file = None
__configuration_manager = None

def __get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = DirectorConfigurationManager(general.read_json(__configuration_file),
                                                               __configuration_file)
    return __configuration_manager

class DirectorConfigurationManager(config_manager.ConfigurationManager):
    # Command line arguments special characters
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR = ','
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR = '='

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(DirectorConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__pipeline_arguments = pipeline_arguments
        self.__pipeline_arguments_object = None
        # Logger the pythonist way
        self._logger = config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))

    def _get_pipeline_arguments(self):
        return self.__pipeline_arguments

    def _get_allowed_configuration_keys(self):
        # Allowed configuration keys set is empty by default
        return {}

    def _get_pipeline_arguments_object(self):
        if self.__pipeline_arguments_object is None:
            self.__pipeline_arguments_object = self._process_pipeline_arguments()
        return self.__pipeline_arguments_object

    def _process_pipeline_arguments(self):
        if self._get_pipeline_arguments():
            if self.__pipeline_arguments_object:
                self._logger.error("DUPLICATED CALL for processing command line arguments for this pipeline, IGNORED")
            else:
                self.__pipeline_arguments_object = {}
                self._logger.debug("Processing pipeline command line arguments")
                allowed_keys = self._get_allowed_configuration_keys()
                for command_line_parameter in self._get_pipeline_arguments().split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR):
                    key, value = command_line_parameter.split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR)
                    if key not in allowed_keys:
                        self._logger.error(
                            "INVALID KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    if key in self.__pipeline_arguments_object:
                        self._logger.error(
                            "DUPLICATED KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    self.__pipeline_arguments_object[key] = value
                    self._logger.debug("Pipeline argument '{}' parsed and set with value '{}'".format(key, value))
        else:
            self._logger.warning("This pipeline was provided with NO COMMAND LINE ARGUMENTS")
        return self.__pipeline_arguments_object

    def _get_value_for_pipeline_argument_key(self, key, default=None):
        if key in self._get_pipeline_arguments_object():
            return self._get_pipeline_arguments_object()[key]
        else:
            return default

class Director(metaclass=abc.ABCMeta):
    """
    This is the director of the pipeline
    """
    _PIPELINE_STATUS_OK = "ok"
    _PIPELINE_STATUS_FAIL = "fail"

    def __init__(self, runner_id=None):
        logger_name = __name__
        if runner_id:
            logger_name = runner_id
        self.__logger = config_manager.get_app_config_manager().get_logger_for(logger_name)
        self.pipeline_status = self._PIPELINE_STATUS_OK

    def is_pipeline_status_ok(self):
        return self.pipeline_status == self._PIPELINE_STATUS_OK

    def set_pipeline_status_fail(self):
        self.pipeline_status = self._PIPELINE_STATUS_FAIL

    def _before(self):
        """
        This method implements some logic that is run before running the main pipeline director.
        Subclasses implementing the pipeline logic, can override this method for establishing pre-pipeline logic.
        :return: it returns True if there was no problem, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        """
        Pipeline template director algorithm, it executes 'before' logic, then the 'pipeline' logic, and 'after' logic
        once the pipeline is finished
        It delegates to specific pipeline implementations the decision of flagging the pipeline as FAIL or not, but the
        pipeline will only be reported as 'successful' after the successful execution of all its stages. This gives the
        developer enough flexibility to have failing stages but still be able to keep going with the pipeline, alghough
        not being considered a complete success. Also, it delegates to subclasses the responsibility of deciding when
        it is not acceptable to keep running the pipeline, which makes sense, as it is their responsibility.
        :return: True if the pipeline has been successful, False otherwise
        """
        result = True
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            result = result and False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            result = result and False
        if not self._after():
            self._get_logger().error("The logic executed AFTER running the pipeline has FAILED")
            result = result and False
        if result:
            self._get_logger().info("SUCCESSFUL Pipeline execution")
        return result

    def _run_pipeline(self):
        """
        This abstract method must be implemented by the subclasses with the strategy / director logic that drives the
        pipeline they are performing
        :return: True if the pipeline is successful, False otherwise
        """
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        """
        This method implements the logic that must be executed after the pipeline workflow has run.
        Subclasses implementing the pipeline logic, can override this method for establishing post-pipeline logic.
        :return: True if success, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        """
        Get the logger to be used for logging messages
        :return: it returns the logger set for the current instance of the class
        """
        return self.__logger

    def _set_logger(self, new_logger):
        """
        This method allows subclasses to change the default logger created when instantiating an implementation of this
        abstract class.
        :param new_logger: new logger to set up as the default logger for a particular implementation subclass instance
        :return: the newly set logger
        """
        self.__logger = new_logger
        return self._get_logger()

class PogoBasedPipelineDirector(Director):
    """
    Abstract class pipeline director class for those pipelines running PoGo in PRIDE
    """

    def __init__(self, runner_id=None):
        super().__init__(runner_id)

    def _get_pogo_protein_sequence_file_path_for_taxonomy(self, taxonomy_id):
        """
        This is a helper method that, given a taxonomy, it finds the protein sequence file to use as a PoGo parameter
        :param taxonomy_id: ncbi taxonomy ID
        :return: the protein sequence file path if found, None otherwise
        """
        # Get an instance of the Ensembl data downloader
        ensembl_downloader_service = ensembl.data_downloader.get_data_download_service()
        # Get Protein Sequence file from Ensembl for this taxonomy, only the "*all*" kind
        protein_sequence_files = ensembl_downloader_service.get_protein_sequences_for_species(taxonomy_id)
        # Checking on whether we can find a protein sequence or not for the given taxonomy will tell us if the
        # taxonomy is on Ensembl or not
        if not protein_sequence_files:
            return None
        pogo_parameter_protein_sequence_file_path = None
        for file_name, file_path in protein_sequence_files:
            self._get_logger().debug("Scanning protein sequence file name '{}'".format(file_name))
            if "all" in file_name:
                pogo_parameter_protein_sequence_file_path = file_path
                break
        if not pogo_parameter_protein_sequence_file_path:
            self._get_logger().error("File 'pep all' containing all sequences NOT FOUND!")
            return None
        return pogo_parameter_protein_sequence_file_path

    def _get_pogo_gtf_file_path_for_taxonomy(self, taxonomy_id):
        """
        This is a helper method that, given a taxonomy ID, it finds out which GTF file should be used for running PoGo
        :param taxonomy_id: ncbi taxonomy ID
        :return: the GTF file path if found, None otherwise
        """
        # Get an instance of the Ensembl data downloader
        ensembl_downloader_service = ensembl.data_downloader.get_data_download_service()
        gtf_files = ensembl_downloader_service.get_genome_reference_for_species(taxonomy_id)
        # For PoGo, we will use the GTF file that has no suffixes, thus, it will be the shortest file name
        pogo_parameter_gtf_file_name = None
        pogo_parameter_gtf_file_path = None
        for file_name, file_path in gtf_files:
            if (not pogo_parameter_gtf_file_name) \
                    or (len(pogo_parameter_gtf_file_name) > len(file_name)):
                pogo_parameter_gtf_file_name = file_name
                pogo_parameter_gtf_file_path = file_path
        return pogo_parameter_gtf_file_path

class TrackhubCreationDirector:
    """
    This is the abstract base class for those pipelines devoted to the creation of a trackhub
    """

    def __init__(self):
        # TODO - Find a way to remove the logger and you have a Mix-in, i.e. legal multiple inheritance, hahaha!
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}".format(__name__, type(self).__name__))

    def _get_trackhub_builder(self, trackhub_descriptor):
        # TODO - Implement more complex logic for instantiating a trackhub builder, e.g. setting up the trackhub
        return trackhubs.SimpleTrackHubBuilder(trackhub_descriptor)

    @abc.abstractmethod
    def _get_trackhub_descriptor(self):
        ...

    @abc.abstractmethod
    def _get_trackhub_exporter(self):
        ...

    @abc.abstractmethod
    def _process_assemblies(self, trackhub_builder):
        ...

    @abc.abstractmethod
    def _prepare_trackhub_destination_folder(self, trackhub_exporter):
        ...

    def _create_trackhub(self):
        """
        Implements a default director for building a trackhub, it can be overriden subclasses or not used at all.
        :return: no return value
        """
        # Instantiate the trackhub builder we are going to use
        trackhub_builder = self._get_trackhub_builder(self._get_trackhub_descriptor())
        # Process and prepare the assemblies with their respective tracks
        self._process_assemblies(trackhub_builder)
        # Prepare the trackhub exporter
        trackhub_exporter = self._get_trackhub_exporter()
        self._prepare_trackhub_destination_folder(trackhub_exporter)
        # Export trackhub
        trackhub_builder.accept_exporter(trackhub_exporter)
        self._logger.info("Trackhub '{}' export to '{}' - COMPLETED"
                          .format(self._get_trackhub_descriptor().get_hub(),
                                  trackhub_exporter.track_hub_destination_folder))

class TrackhubCreationPogoBasedDirector(PogoBasedPipelineDirector, TrackhubCreationDirector):
    """
    This is a pipeline director base class for those pipelines devoted to creating trackhubs based on running PoGo on
    input data for producing .bed files
    """
    def __init__(self, runner_id=None):
        super().__init__(runner_id)
        TrackhubCreationDirector.__init__(self)
        # Override superclass logger to use this one
        logger_name = "{}.{}".format(__name__, type(self).__name__)
        if runner_id:
            logger_name = runner_id
        self._logger = config_manager \
            .get_app_config_manager() \
            .get_logger_for(logger_name)

    @abc.abstractmethod
    def _get_pogo_results_for_input_data(self):
        ...

    @abc.abstractmethod
    def _get_trackhub_track_for_taxonomy_id(self, taxonomy_id):
        ...

    def _get_assemblies_from_pogo_results(self, trackhub_builder):
        # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
        pogo_results = self._get_pogo_results_for_input_data()
        self._get_logger().info("Processing #{} Taxonomies from PoGo results".format(len(pogo_results)))
        # assembly_mapping_service = AssemblyMappingServiceFactory.get_assembly_mapping_service()
        for taxonomy in pogo_results:
            ensembl_species_entry = ensembl.service\
                .get_service()\
                .get_species_data_service()\
                .get_species_entry_for_taxonomy_id(taxonomy)
            if not ensembl_species_entry:
                self._get_logger().error("Ensembl has NO ENTRY for taxonomy ID {} - SKIP -".format(taxonomy))
                continue
            # We can't use Ensembl Assemblies any more, they have to be translated to UCSC assemblies
            genome_assembly = ensembl_species_entry.get_assembly()
            #genome_assembly = assembly_mapping_service\
            #    .get_ucsc_assembly_for_ensembl_assembly_accession(ensembl_species_entry.get_assembly_accession())
            # Main .bed track
            bed_file_path = pogo_results[taxonomy].get_pogo_result_main_bed_file_path()
            trackhub_track_main = self._get_trackhub_track_for_taxonomy_id(taxonomy)
            trackhub_track_main.taxonomy_id = taxonomy
            trackhub_track_main.set_big_data_url(bed_file_path)
            trackhub_track_main.set_type(bed_file_path)
            # Main .bed track with PTMs
            bed_ptm_file_path = pogo_results[taxonomy].get_pogo_result_main_ptm_bed_file_path()
            trackhub_track_main_with_ptm = self._get_trackhub_track_for_taxonomy_id(taxonomy)
            trackhub_track_main_with_ptm.taxonomy_id = taxonomy
            trackhub_track_main_with_ptm.set_track("{} with PTMs".format(trackhub_track_main_with_ptm.get_track()))
            trackhub_track_main_with_ptm.set_big_data_url(bed_ptm_file_path)
            trackhub_track_main_with_ptm.set_type(bed_ptm_file_path)
            # Add tracks
            trackhub_builder.add_track_to_assembly(genome_assembly, trackhub_track_main)
            trackhub_builder.add_track_to_assembly(genome_assembly, trackhub_track_main_with_ptm)
            self._get_logger().debug("Assembly '{}', tracks added with files '({})'"
                                     .format(genome_assembly,
                                             ",".join([bed_file_path, bed_ptm_file_path])))

    def _process_assemblies(self, trackhub_builder):
        self._get_assemblies_from_pogo_results(trackhub_builder)

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
main.py

the program starts here.

"""

import sys
import os
import math
import ForceFields
from structure_data import CIF, Structure
from datetime import datetime
from InputHandler import Options

def construct_data_file(ff):

    t = datetime.today()
    string = "Created on %s\n\n"%t.strftime("%a %b %d %H:%M:%S %Y %Z")

    string += "%12i atoms\n"%(len(ff.structure.atoms))
    string += "%12i bonds\n"%(len(ff.structure.bonds))
    string += "%12i angles\n"%(len(ff.structure.angles))
    string += "%12i dihedrals\n"%(len(ff.structure.dihedrals))
    string += "%12i impropers\n\n"%(len(ff.structure.impropers))

    string += "%12i atom types\n"%(len(ff.unique_atom_types.keys()))
    string += "%12i bond types\n"%(len(ff.unique_bond_types.keys()))
    string += "%12i angle types\n"%(len(ff.unique_angle_types.keys()))
    string += "%12i dihedral types\n"%(len(ff.unique_dihedral_types.keys()))
    string += "%12i improper types\n"%(len(ff.unique_improper_types.keys()))

    cell = ff.structure.cell
    string += "%19.6f %10.6f %s %s\n"%(0., cell.lx, "xlo", "xhi")
    string += "%19.6f %10.6f %s %s\n"%(0., cell.ly, "ylo", "yhi")
    string += "%19.6f %10.6f %s %s\n"%(0., cell.lz, "zlo", "zhi")
    string += "%19.6f %10.6f %10.6f %s %s %s\n"%(cell.xy, cell.xz, cell.yz, "xy", "xz", "yz")

    # Let's track the forcefield potentials that haven't been calc'd or user specified
    no_bond = []
    no_angle = []
    no_dihedral = []
    no_improper = []

    string += "\nMasses\n\n"
    for key in sorted(ff.unique_atom_types.keys()):
        unq_atom = ff.unique_atom_types[key]
        mass, type = unq_atom.mass, unq_atom.force_field_type
        string += "%5i %8.4f # %s\n"%(key, mass, type)

    string += "\nBond Coeffs\n\n"
    for key in sorted(ff.unique_bond_types.keys()):
        bond = ff.unique_bond_types[key]
        if bond.potential is None:
            no_bond.append("%5i : %s %s"%(key, bond.atoms[0].force_field_type, bond.atoms[1].force_field_type))
        else:
            ff1, ff2 = bond.atoms[0].force_field_type, bond.atoms[1].force_field_type
            string += "%5i %s "%(key, bond.potential)
            string += "# %s %s\n"%(ff1, ff2)

    string += "\nAngle Coeffs\n\n"
    for key in sorted(ff.unique_angle_types.keys()):
        angle = ff.unique_angle_types[key]
        atom_a, atom_b, atom_c = angle.atoms

        if angle.potential is None:
            no_angle.append("%5i : %s %s %s"%(key, atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type))
        else:
            string += "%5i %s "%(key, angle.potential)
            string += "# %s %s %s\n"%(atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type)

    string +=  "\nDihedral Coeffs\n\n"
    for key in sorted(ff.unique_dihedral_types.keys()):
        dihedral = ff.unique_dihedral_types[key]
        atom_a, atom_b, atom_c, atom_d = dihedral.atoms
        if dihedral.potential is None:
            no_dihedral.append("%5i : %s %s %s %s"%(key, atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type))
        else:
            # include options to not print the name of the function on this line (i.e. no hybrid) if not needed.
            string += "%5i %s "%(key, dihedral.potential)
            string += "# %s %s %s %s\n"%(atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type)

    string += "\nImproper Coeffs\n\n"
    for key in sorted(ff.unique_improper_types.keys()):
        improper = ff.unique_improper_types[key]
        atom_a, atom_b, atom_c, atom_d = improper.atoms
        if improper.potential is None:
            no_improper.append("%5i : %s %s %s %s"%(key, atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type))
        else:
            string += "%5i %s "%(key, improper.potential)
            string += "# %s %s %s %s\n"%(atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type)

    string += "\nPair Coeffs\n\n"
    for key, pair in sorted(ff.unique_pair_types.items()):
        # TODO(change eps to 0 for Al, Si, Ge)
        string += "%5i %s "%(key, pair.potential)

    # Nest this in an if statement
    if any([no_bond, no_angle, no_dihedral, no_improper]):
    # WARNING MESSAGE for potentials we think are unique but have not been calculated
        print("WARNING: The following unique bonds/angles/dihedrals/impropers" +
                " were detected in your crystal")
        print("But they have not been assigned a potential from user_input.txt"+
                " or from an internal FF assignment routine!")
        print("Bonds")
        for elem in no_bond:
            print(elem)
        print("Angles")
        for elem in no_angle:
            print(elem)
        print("Dihedrals")
        for elem in no_dihedral:
            print(elem)
        print("Impropers")
        for elem in no_improper:
            print(elem)
        print("If you think you specified one of these in your user_input.txt " +
              "and this is an error, please contact developers\n")
        print("CONTINUING...")

    #************[atoms]************
	# Added 1 to all atom, bond, angle, dihedral, improper indices (LAMMPS does not accept atom of index 0)
    string += "\nAtoms\n\n"
    for atom in ff.structure.atoms:
        molid = 444
        string += "%8i %8i %8i %11.5f %10.5f %10.5f %10.5f\n"%(atom.index+1,
                                                               molid,
                                                               atom.ff_type_index,
                                                               atom.charge,
                                                               atom.coordinates[0],
                                                               atom.coordinates[1],
                                                               atom.coordinates[2])

    #************[bonds]************
    string += "\nBonds\n\n"
    for bond in ff.structure.bonds:
        atm1, atm2 = bond.atoms
        string += "%8i %8i %8i %8i\n"%(bond.index+1,
                                       bond.ff_type_index,
                                       atm1.index+1,
                                       atm2.index+1)

    #************[angles]***********
    string += "\nAngles\n\n"
    for angle in ff.structure.angles:
        atm1, atm2, atm3 = angle.atoms
        # what order are they presented? b, a, c? or a, b, c?
        string += "%8i %8i %8i %8i %8i\n"%(angle.index+1,
                                           angle.ff_type_index,
                                           atm1.index+1,
                                           atm2.index+1,
                                           atm3.index+1)

    #************[dihedrals]********
    string += "\nDihedrals\n\n"
    for dihedral in ff.structure.dihedrals:
        atm1, atm2, atm3, atm4 = dihedral.atoms
        # order?
        string += "%8i %8i %8i %8i %8i %8i\n"%(dihedral.index+1,
                                               dihedral.ff_type_index,
                                               atm1.index+1,
                                               atm2.index+1,
                                               atm3.index+1,
                                               atm4.index+1)

    #************[impropers]********
    string += "\nImpropers\n\n"
    for improper in ff.structure.impropers:
        atm1, atm2, atm3, atm4 = improper.atoms
        # order?
        string += "%8i %8i %8i %8i %8i %8i\n"%(improper.index+1,
                                               improper.ff_type_index,
                                               atm1.index+1,
                                               atm2.index+1,
                                               atm3.index+1,
                                               atm4.index+1)

    return string

def construct_input_file(ff):
    """Input file will depend on what the user wants to do"""

    # Eventually, this function should be dependent on some command line arguments
    # which will dictate what kind of simulation to run in LAMMPS
    inp_str = ""
    inp_str += "%-15s %s\n"%("units","real")
    inp_str += "%-15s %s\n"%("atom_style","full")
    inp_str += "%-15s %s\n"%("boundary","p p p")
    inp_str += "%-15s %s\n"%("dielectric","1")
    inp_str += "\n"
    inp_str += "%-15s %s\n"%("pair_style", "hybrid lj/cut/coul/long 8.50000 11.5")
    inp_str += "%-15s %s\n"%("bond_style","hybrid harmonic")
    inp_str += "%-15s %s\n"%("angle_style","hybrid fourier fourier/simple")
    inp_str += "%-15s %s\n"%("dihedral_style","hybrid harmonic")
    inp_str += "%-15s %s\n"%("improper_style","hybrid fourier")
    inp_str += "%-15s %s\n"%("kspace_style","ewald 0.001")
    inp_str += "\n"
    inp_str += "%-15s %s\n"%("box tilt","large")
    inp_str += "%-15s %s\n"%("read_data","data.%s"%(ff.structure.name))

    inp_str += "\n"
    inp_str += "%-15s %s\n"%("dump","%s_mov all xyz 1 %s_mov.xyz"%(ff.structure.name, ff.structure.name))
    inp_str += "%-15s %s\n"%("pair_modify","tail yes mix arithmetic")
    inp_str += "%-15s %s\n"%("min_style","cg")
    inp_str += "%-15s %s\n"%("minimize","1.0e-4 1.0e-6 10000 100000")
    inp_str += "%-15s %s\n"%("fix","1 all box/relax tri 0.0 vmax 0.01")
    inp_str += "%-15s %s\n"%("minimize","1.0e-4 1.0e-6 10000 100000")
    inp_str += "%-15s %s\n"%("unfix", "1")
    inp_str += "%-15s %s\n"%("minimize","1.0e-4 1.0e-6 10000 100000")

    return inp_str

def clean(name):
    name = os.path.split(name)[-1]
    if name.endswith('.cif'):
        name = name[:-4]
    return name

def main():

    # command line parsing
    options = Options()

    cifname = None
    cif = CIF()

    cif.read(options.cif_file)

    mofname = clean(options.cif_file)
    struct = Structure(name=mofname)
    struct.from_CIF(cif)
    # compute minimum supercell
    # NB: half box width should be a user-defined command,
    # or default to 2.5*sigma_max of the requested force field
    # currently defaults to 12.5 anstroms
    struct.minimum_cell()

    struct.compute_angles()
    struct.compute_dihedrals()
    struct.compute_improper_dihedrals()

    #obtain desired force field class from command line, parse
    # string to real class value
    try:
        ff = getattr(ForceFields, options.force_field)(struct)
    except AttributeError:
        print("Error: could not find the force field: %s"%options.force_field)
        sys.exit()

    ff.compute_force_field_terms()
    data_str = construct_data_file(ff)
    inp_str = construct_input_file(ff)

    datafile = open("data.%s"%struct.name, 'w')
    datafile.writelines(data_str)
    datafile.close()

    inpfile = open("in.%s"%struct.name, 'w')
    inpfile.writelines(inp_str)
    inpfile.close()

    print("files created!")

if __name__ == "__main__":
    main()

<commit_msg>added comment, program breaks currently, will patch up soon<commit_after>#!/usr/bin/env python
"""
main.py

the program starts here.

"""

import sys
import os
import math
import ForceFields
from structure_data import CIF, Structure
from datetime import datetime
from InputHandler import Options

def construct_data_file(ff):

    t = datetime.today()
    string = "Created on %s\n\n"%t.strftime("%a %b %d %H:%M:%S %Y %Z")

    string += "%12i atoms\n"%(len(ff.structure.atoms))
    string += "%12i bonds\n"%(len(ff.structure.bonds))
    string += "%12i angles\n"%(len(ff.structure.angles))
    string += "%12i dihedrals\n"%(len(ff.structure.dihedrals))
    string += "%12i impropers\n\n"%(len(ff.structure.impropers))

    string += "%12i atom types\n"%(len(ff.unique_atom_types.keys()))
    string += "%12i bond types\n"%(len(ff.unique_bond_types.keys()))
    string += "%12i angle types\n"%(len(ff.unique_angle_types.keys()))
    string += "%12i dihedral types\n"%(len(ff.unique_dihedral_types.keys()))
    string += "%12i improper types\n"%(len(ff.unique_improper_types.keys()))

    cell = ff.structure.cell
    string += "%19.6f %10.6f %s %s\n"%(0., cell.lx, "xlo", "xhi")
    string += "%19.6f %10.6f %s %s\n"%(0., cell.ly, "ylo", "yhi")
    string += "%19.6f %10.6f %s %s\n"%(0., cell.lz, "zlo", "zhi")
    string += "%19.6f %10.6f %10.6f %s %s %s\n"%(cell.xy, cell.xz, cell.yz, "xy", "xz", "yz")

    # Let's track the forcefield potentials that haven't been calc'd or user specified
    no_bond = []
    no_angle = []
    no_dihedral = []
    no_improper = []

    string += "\nMasses\n\n"
    for key in sorted(ff.unique_atom_types.keys()):
        unq_atom = ff.unique_atom_types[key]
        mass, type = unq_atom.mass, unq_atom.force_field_type
        string += "%5i %8.4f # %s\n"%(key, mass, type)

    string += "\nBond Coeffs\n\n"
    for key in sorted(ff.unique_bond_types.keys()):
        bond = ff.unique_bond_types[key]
        if bond.potential is None:
            no_bond.append("%5i : %s %s"%(key, bond.atoms[0].force_field_type, bond.atoms[1].force_field_type))
        else:
            ff1, ff2 = bond.atoms[0].force_field_type, bond.atoms[1].force_field_type
            string += "%5i %s "%(key, bond.potential)
            string += "# %s %s\n"%(ff1, ff2)

    string += "\nAngle Coeffs\n\n"
    for key in sorted(ff.unique_angle_types.keys()):
        angle = ff.unique_angle_types[key]
        atom_a, atom_b, atom_c = angle.atoms

        if angle.potential is None:
            no_angle.append("%5i : %s %s %s"%(key, atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type))
        else:
            string += "%5i %s "%(key, angle.potential)
            string += "# %s %s %s\n"%(atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type)

    string +=  "\nDihedral Coeffs\n\n"
    for key in sorted(ff.unique_dihedral_types.keys()):
        dihedral = ff.unique_dihedral_types[key]
        atom_a, atom_b, atom_c, atom_d = dihedral.atoms
        if dihedral.potential is None:
            no_dihedral.append("%5i : %s %s %s %s"%(key, atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type))
        else:
            # include options to not print the name of the function on this line (i.e. no hybrid) if not needed.
            string += "%5i %s "%(key, dihedral.potential)
            string += "# %s %s %s %s\n"%(atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type)

    string += "\nImproper Coeffs\n\n"
    for key in sorted(ff.unique_improper_types.keys()):
        improper = ff.unique_improper_types[key]
        atom_a, atom_b, atom_c, atom_d = improper.atoms
        if improper.potential is None:
            no_improper.append("%5i : %s %s %s %s"%(key, atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type))
        else:
            string += "%5i %s "%(key, improper.potential)
            string += "# %s %s %s %s\n"%(atom_a.force_field_type, atom_b.force_field_type, atom_c.force_field_type, atom_d.force_field_type)

    string += "\nPair Coeffs\n\n"
    for key, pair in sorted(ff.unique_pair_types.items()):
        # TODO(change eps to 0 for Al, Si, Ge)
        string += "%5i %s "%(key, pair.potential)

    # Nest this in an if statement
    if any([no_bond, no_angle, no_dihedral, no_improper]):
    # WARNING MESSAGE for potentials we think are unique but have not been calculated
        print("WARNING: The following unique bonds/angles/dihedrals/impropers" +
                " were detected in your crystal")
        print("But they have not been assigned a potential from user_input.txt"+
                " or from an internal FF assignment routine!")
        print("Bonds")
        for elem in no_bond:
            print(elem)
        print("Angles")
        for elem in no_angle:
            print(elem)
        print("Dihedrals")
        for elem in no_dihedral:
            print(elem)
        print("Impropers")
        for elem in no_improper:
            print(elem)
        print("If you think you specified one of these in your user_input.txt " +
              "and this is an error, please contact developers\n")
        print("CONTINUING...")

    #************[atoms]************
	# Added 1 to all atom, bond, angle, dihedral, improper indices (LAMMPS does not accept atom of index 0)
    string += "\nAtoms\n\n"
    for atom in ff.structure.atoms:
        molid = 444
        string += "%8i %8i %8i %11.5f %10.5f %10.5f %10.5f\n"%(atom.index+1,
                                                               molid,
                                                               atom.ff_type_index,
                                                               atom.charge,
                                                               atom.coordinates[0],
                                                               atom.coordinates[1],
                                                               atom.coordinates[2])

    #************[bonds]************
    string += "\nBonds\n\n"
    for bond in ff.structure.bonds:
        atm1, atm2 = bond.atoms
        string += "%8i %8i %8i %8i\n"%(bond.index+1,
                                       bond.ff_type_index,
                                       atm1.index+1,
                                       atm2.index+1)

    #************[angles]***********
    string += "\nAngles\n\n"
    for angle in ff.structure.angles:
        atm1, atm2, atm3 = angle.atoms
        # what order are they presented? b, a, c? or a, b, c?
        string += "%8i %8i %8i %8i %8i\n"%(angle.index+1,
                                           angle.ff_type_index,
                                           atm1.index+1,
                                           atm2.index+1,
                                           atm3.index+1)

    #************[dihedrals]********
    string += "\nDihedrals\n\n"
    for dihedral in ff.structure.dihedrals:
        atm1, atm2, atm3, atm4 = dihedral.atoms
        # order?
        string += "%8i %8i %8i %8i %8i %8i\n"%(dihedral.index+1,
                                               dihedral.ff_type_index,
                                               atm1.index+1,
                                               atm2.index+1,
                                               atm3.index+1,
                                               atm4.index+1)

    #************[impropers]********
    string += "\nImpropers\n\n"
    for improper in ff.structure.impropers:
        atm1, atm2, atm3, atm4 = improper.atoms
        # order?
        string += "%8i %8i %8i %8i %8i %8i\n"%(improper.index+1,
                                               improper.ff_type_index,
                                               atm1.index+1,
                                               atm2.index+1,
                                               atm3.index+1,
                                               atm4.index+1)

    return string

def construct_input_file(ff):
    """Input file will depend on what the user wants to do"""

    # Eventually, this function should be dependent on some command line arguments
    # which will dictate what kind of simulation to run in LAMMPS
    inp_str = ""
    inp_str += "%-15s %s\n"%("units","real")
    inp_str += "%-15s %s\n"%("atom_style","full")
    inp_str += "%-15s %s\n"%("boundary","p p p")
    inp_str += "%-15s %s\n"%("dielectric","1")
    inp_str += "\n"
    inp_str += "%-15s %s\n"%("pair_style", "hybrid lj/cut/coul/long 8.50000 11.5")
    inp_str += "%-15s %s\n"%("bond_style","hybrid harmonic")
    inp_str += "%-15s %s\n"%("angle_style","hybrid fourier fourier/simple")
    inp_str += "%-15s %s\n"%("dihedral_style","hybrid harmonic")
    inp_str += "%-15s %s\n"%("improper_style","hybrid fourier")
    inp_str += "%-15s %s\n"%("kspace_style","ewald 0.001")
    inp_str += "\n"
    inp_str += "%-15s %s\n"%("box tilt","large")
    inp_str += "%-15s %s\n"%("read_data","data.%s"%(ff.structure.name))

    inp_str += "\n"
    inp_str += "%-15s %s\n"%("dump","%s_mov all xyz 1 %s_mov.xyz"%(ff.structure.name, ff.structure.name))
    inp_str += "%-15s %s\n"%("pair_modify","tail yes mix arithmetic")
    inp_str += "%-15s %s\n"%("min_style","cg")
    inp_str += "%-15s %s\n"%("minimize","1.0e-4 1.0e-6 10000 100000")
    inp_str += "%-15s %s\n"%("fix","1 all box/relax tri 0.0 vmax 0.01")
    inp_str += "%-15s %s\n"%("minimize","1.0e-4 1.0e-6 10000 100000")
    inp_str += "%-15s %s\n"%("unfix", "1")
    inp_str += "%-15s %s\n"%("minimize","1.0e-4 1.0e-6 10000 100000")

    return inp_str

def clean(name):
    name = os.path.split(name)[-1]
    if name.endswith('.cif'):
        name = name[:-4]
    return name

def main():

    # command line parsing
    options = Options()

    cifname = None
    cif = CIF()

    cif.read(options.cif_file)

    mofname = clean(options.cif_file)
    struct = Structure(name=mofname)
    struct.from_CIF(cif)
    # compute minimum supercell
    # NB: half box width should be a user-defined command,
    # or default to 2.5*sigma_max of the requested force field
    # currently defaults to 12.5 anstroms
    struct.minimum_cell()

    struct.compute_angles()
    struct.compute_dihedrals()
    struct.compute_improper_dihedrals()

    #obtain desired force field class from command line, parse
    # string to real class value
    try:
        ff = getattr(ForceFields, options.force_field)(struct)
    except AttributeError:
        print("Error: could not find the force field: %s"%options.force_field)
        sys.exit()

    # doesn't really follow the logic of the program, but can only be done
    # when the atoms have been assigned to a force field.
    # ff.struct.compute_pair_terms()

    ff.compute_force_field_terms()
    data_str = construct_data_file(ff)
    inp_str = construct_input_file(ff)

    datafile = open("data.%s"%struct.name, 'w')
    datafile.writelines(data_str)
    datafile.close()

    inpfile = open("in.%s"%struct.name, 'w')
    inpfile.writelines(inp_str)
    inpfile.close()

    print("files created!")

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>""" Command Line Interface Module """
import optparse
from flask import Flask, Response, request
from random import choice
from string import ascii_uppercase
import re
import json

app = Flask(__name__)

# [{ "username": X, "requestid": X, "client_ip"; X, "authorized": False},]
auth_requests = []

def username_is_valid(username):
    if re.match(r'^[a-zA-Z0-9_\-]+$', username):
        return True
    return False

def requestid_is_valid(requestid):
    if re.match(r'^[a-zA-Z0-9]+$', requestid) and len(requestid) == 16:
        return True
    return False

def ip_is_valid(ip):
    if re.match(r'^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$', ip):
        return True
    return False

def is_duplicate_register(username, requestid, clientip):
    for item in auth_requests:
        if item["username"] == username and item["requestid"] == requestid and item["client_ip"] == clientip:
            # Request Is Already Registered
            return True
    return False

@app.route('/api/request/list', methods = ['POST'])
def request_list():
    username = str(request.form['username'])
    print("verify request was from the user authed connection")

    request_list = []
    for item in auth_requests:
        if item["username"] == username:
            request_list.append(item)

    return Response(response=json.dumps(request_list), status=200, mimetype="application/json")

@app.route('/api/request/register', methods = ['POST'])
def request_register():
    username = str(request.form['username'])
    requestid = str(request.form['requestid'])
    clientip = request.remote_addr
    print(username)
    print(requestid)

    if not username_is_valid(username) or not requestid_is_valid(requestid) or not ip_is_valid(clientip):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    if is_duplicate_register(username, requestid, clientip):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    auth_requests.append({"username": username, "requestid": requestid, "client_ip": clientip, "authorized": False})

    #return Response(response=json.dumps({"response": "success"}), status=200, mimetype="application/json")
    return "success"

@app.route('/api/request/status', methods = ['POST'])
def request_status():
    username = str(request.form['username'])
    requestid = str(request.form['requestid'])
    clientip = request.remote_addr
    print(username)
    print(requestid)

    if username_is_valid(username) and requestid_is_valid(requestid) and ip_is_valid(clientip):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    for item in auth_requests:
        if item["username"] == username and item["requestid"] == requestid and item["client_ip"] == clientip:
            if item["authorized"] == True:
                # Success
                #return Response(response=json.dumps({"response": "success"}), status=200, mimetype="application/json")
                return "success"

    return Response(response=json.dumps({"response": "Authentication Failure"}), status=401, mimetype="application/json")

class Cli(object):
    """ Command Line Interface for pivportal """
    # Parse CLI Arguments
    def __init__(self):
        parser = optparse.OptionParser()
        parser.add_option("-p", "--port", dest="port",
                              help="port",
                              metavar="PORT",
                              default="8088")
        parser.add_option("-l", "--listen", dest="listen",
                              help="Listen IP Address",
                              metavar="LISTEN",
                              default="127.0.0.1")
        parser.add_option("-d", "--debug", dest="debug",
                              help="debug",
                              metavar="DEBUG",
                              default=False)
        (options, args) = parser.parse_args()
        self.port = options.port
        self.listen = options.listen
        self.is_debug = options.debug

    def run(self):
        """ EntryPoint Of Application """
        app.run(threaded=True, host=self.listen, port=self.port, debug=self.is_debug)<commit_msg>added an auth func for the rest api dash.<commit_after>""" Command Line Interface Module """
import optparse
from flask import Flask, Response, request
from random import choice
from string import ascii_uppercase
import re
import json

app = Flask(__name__)

# [{ "username": X, "requestid": X, "client_ip"; X, "authorized": False},]
auth_requests = []

def username_is_valid(username):
    if re.match(r'^[a-zA-Z0-9_\-]+$', username):
        return True
    return False

def requestid_is_valid(requestid):
    if re.match(r'^[a-zA-Z0-9]+$', requestid) and len(requestid) == 16:
        return True
    return False

def ip_is_valid(ip):
    if re.match(r'^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$', ip):
        return True
    return False

def is_duplicate_register(username, requestid, clientip):
    for item in auth_requests:
        if item["username"] == username and item["requestid"] == requestid and item["client_ip"] == clientip:
            # Request Is Already Registered
            return True
    return False

@app.route('/api/request/list', methods = ['POST'])
def request_list():
    username = str(request.form['username'])
    print("verify request was from the user authed connection")

    if not username_is_valid(username):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    request_list = []
    for item in auth_requests:
        if item["username"] == username:
            request_list.append(item)

    return Response(response=json.dumps(request_list), status=200, mimetype="application/json")

@app.route('/api/request/auth', methods = ['POST'])
def request_auth():
    username = str(request.form['username'])
    requestid = str(request.form['requestid'])
    clientip = str(request.form['clientip'])
    print("verify request was from the user authed connection")

    if not username_is_valid(username) or not requestid_is_valid(requestid) or not ip_is_valid(clientip):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    # Authenticate Request
    count = 0
    for item in auth_requests:
        if item["username"] == username and item["requestid"] == requestid and item["client_ip"] == clientip:
            if item["authorized"] == False:
                auth_requests[count]["authorized"] = True
        count += 1

    return Response(response=json.dumps(request_list), status=200, mimetype="application/json")

@app.route('/api/request/register', methods = ['POST'])
def request_register():
    username = str(request.form['username'])
    requestid = str(request.form['requestid'])
    clientip = request.remote_addr
    print(username)
    print(requestid)

    if not username_is_valid(username) or not requestid_is_valid(requestid) or not ip_is_valid(clientip):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    if is_duplicate_register(username, requestid, clientip):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    auth_requests.append({"username": username, "requestid": requestid, "client_ip": clientip, "authorized": False})

    #return Response(response=json.dumps({"response": "success"}), status=200, mimetype="application/json")
    return "success"

@app.route('/api/request/status', methods = ['POST'])
def request_status():
    username = str(request.form['username'])
    requestid = str(request.form['requestid'])
    clientip = request.remote_addr
    print(username)
    print(requestid)

    if username_is_valid(username) and requestid_is_valid(requestid) and ip_is_valid(clientip):
        return Response(response=json.dumps({"response": "  invalid request"}), status=400, mimetype="application/json")

    for item in auth_requests:
        if item["username"] == username and item["requestid"] == requestid and item["client_ip"] == clientip:
            if item["authorized"] == True:
                # Success
                #return Response(response=json.dumps({"response": "success"}), status=200, mimetype="application/json")
                return "success"

    return Response(response=json.dumps({"response": "Authentication Failure"}), status=401, mimetype="application/json")

class Cli(object):
    """ Command Line Interface for pivportal """
    # Parse CLI Arguments
    def __init__(self):
        parser = optparse.OptionParser()
        parser.add_option("-p", "--port", dest="port",
                              help="port",
                              metavar="PORT",
                              default="8088")
        parser.add_option("-l", "--listen", dest="listen",
                              help="Listen IP Address",
                              metavar="LISTEN",
                              default="127.0.0.1")
        parser.add_option("-d", "--debug", dest="debug",
                              help="debug",
                              metavar="DEBUG",
                              default=False)
        (options, args) = parser.parse_args()
        self.port = options.port
        self.listen = options.listen
        self.is_debug = options.debug

    def run(self):
        """ EntryPoint Of Application """
        app.run(threaded=True, host=self.listen, port=self.port, debug=self.is_debug)<|endoftext|>


<commit_before>import datetime
import httplib
import random
import urllib
import urlparse

class PiwikTracker:
    VERSION = 1

    def __init__(self, id_site, api_url, request, token_auth):
        random.seed()
        self.id_site = id_site
        self.api_url = api_url
        self.request = request
        self.token_auth = token_auth

        self.page_url = self.get_current_url()
        self.set_request_parameters()
        self.set_local_time(self.get_timestamp())

    def set_request_parameters(self):
        # django-specific
        self.user_agent = self.request.META.get('HTTP_USER_AGENT', '')
        self.referer = self.request.META.get('HTTP_REFERER', '')
        self.ip = self.request.META.get('REMOTE_ADDR')
        self.accept_language = self.request.META.get('HTTP_ACCEPT_LANGUAGE', '')

    def set_local_time(self, datetime):
        self.local_hour = datetime.hour
        self.local_minute = datetime.minute
        self.local_second = datetime.second

    def get_current_scheme(self):
        # django-specific
        if self.request.is_secure():
            scheme = 'https'
        else:
            scheme = 'http'
        return scheme

    def get_current_host(self):
        # django-specific
        return self.request.get_host()

    def get_current_script_name(self):
        # django-specific
        return self.request.path_info

    def get_current_query_string(self):
        # django-specific
        return self.request.META.get('QUERY_STRING', '')

    def get_current_url(self):
        url  = self.get_current_scheme() + '://'
        url += self.get_current_host()
        url += self.get_current_script_name()
        url += self.get_current_query_string()
        return url

    def get_timestamp(self):
        return datetime.datetime.now()

    def get_query_vars(self, document_title=False):
        url = "?idsite=%d&rec=1&apiv=%s&r=%s&url=%s&urlref=%s&cip=%s&token_auth=%s" % (
            self.id_site,
            self.VERSION,
            random.randint(0, 99999),
            urllib.quote_plus(self.page_url),
            urllib.quote_plus(self.referer),
            # Forcing IP requires the auth token
            self.ip,
            self.token_auth,
        )
        if document_title:
            url += '&action_name=%s' % urllib.quote_plus(document_title)
        return url

    def send_request(self, url):
        "Send the request to piwik"
        headers = {
            'Accept-Language': self.accept_language,
            'User-Agent': self.user_agent,
        }
        parsed = urlparse.urlparse(self.api_url)
        connection = httplib.HTTPConnection(parsed.hostname)
        url = parsed.path + url
        connection.request('GET', url, '', headers)
        response = connection.getresponse()
        return response.read()

    def do_track_page_view(self, document_title):
        url = self.get_query_vars(document_title)
        return self.send_request(url);

def piwik_get_url_track_page_view(id_site, api_url, request, token_auth, document_title=False):
    tracker = PiwikTracker(id_site, api_url, request, token_auth)
    return tracker.do_track_page_view(document_title)
<commit_msg>Clarify variable name<commit_after>import datetime
import httplib
import random
import urllib
import urlparse

class PiwikTracker:
    VERSION = 1

    def __init__(self, id_site, api_url, request, token_auth):
        random.seed()
        self.id_site = id_site
        self.api_url = api_url
        self.request = request
        self.token_auth = token_auth

        self.page_url = self.get_current_url()
        self.set_request_parameters()
        self.set_local_time(self.get_timestamp())

    def set_request_parameters(self):
        # django-specific
        self.user_agent = self.request.META.get('HTTP_USER_AGENT', '')
        self.referer = self.request.META.get('HTTP_REFERER', '')
        self.ip = self.request.META.get('REMOTE_ADDR')
        self.accept_language = self.request.META.get('HTTP_ACCEPT_LANGUAGE', '')

    def set_local_time(self, datetime):
        self.local_hour = datetime.hour
        self.local_minute = datetime.minute
        self.local_second = datetime.second

    def get_current_scheme(self):
        # django-specific
        if self.request.is_secure():
            scheme = 'https'
        else:
            scheme = 'http'
        return scheme

    def get_current_host(self):
        # django-specific
        return self.request.get_host()

    def get_current_script_name(self):
        # django-specific
        return self.request.path_info

    def get_current_query_string(self):
        # django-specific
        return self.request.META.get('QUERY_STRING', '')

    def get_current_url(self):
        url  = self.get_current_scheme() + '://'
        url += self.get_current_host()
        url += self.get_current_script_name()
        url += self.get_current_query_string()
        return url

    def get_timestamp(self):
        return datetime.datetime.now()

    def get_query_vars(self, document_title=False):
        url = "?idsite=%d&rec=1&apiv=%s&r=%s&url=%s&urlref=%s&cip=%s&token_auth=%s" % (
            self.id_site,
            self.VERSION,
            random.randint(0, 99999),
            urllib.quote_plus(self.page_url),
            urllib.quote_plus(self.referer),
            # Forcing IP requires the auth token
            self.ip,
            self.token_auth,
        )
        if document_title:
            url += '&action_name=%s' % urllib.quote_plus(document_title)
        return url

    def send_request(self, query_vars):
        "Send the request to piwik"
        headers = {
            'Accept-Language': self.accept_language,
            'User-Agent': self.user_agent,
        }
        parsed = urlparse.urlparse(self.api_url)
        connection = httplib.HTTPConnection(parsed.hostname)
        url = parsed.path + query_vars
        connection.request('GET', url, '', headers)
        response = connection.getresponse()
        return response.read()

    def do_track_page_view(self, document_title):
        query_vars = self.get_query_vars(document_title)
        return self.send_request(query_vars);

def piwik_get_url_track_page_view(id_site, api_url, request, token_auth, document_title=False):
    tracker = PiwikTracker(id_site, api_url, request, token_auth)
    return tracker.do_track_page_view(document_title)
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

from xml.sax.handler import ContentHandler

class SmallSMILHandler(ContentHandler):
    def __init__ (self):
        self.width = ""
        self.height = ""
        self.backgroundcolor = ""
        self.top = ""
        self.botton = ""
        self.left = ""
        self.right = ""
        self.src = ""
        self.region = ""
        self.begin = ""
        self.dur = ""
#No necesitamos endElement (solo tenemos principio de etiqueta)
    def startElement(self,name,attrs):
        if name == "root-layout":
        #valores de los atributos"
            self.width = attrs.get(width,"")
            print(width)
            self.height = attrs.get(heigt,"")
            self.backgroundcolor = attrs.get(backgroundcolor,"")
        if name == "region":
            self.id = attrs.get()
            self.top = attrs.get()
            self.botton = attrs.get()
            self.left = attrs.get()
            self.right = attrs.get()
        if name == "img":
            self.src = attrs.get()
            self.region = attrs.get()
            self.begin = attrs.get()
            self.dur = attrs.get()
        if name == "audio":
            self.src = attrs.get()
            self.begin = attrs.get()
            self.dur = attrs.get()
        if name == "textstream":
            self.src = attrs.get()
            self.regin = attrs.get()

if __name__ == "__main__":
    """
    Programa principal
    """
    parser = make_parser()
    cHandler = ChistesHandler()
    parser.setContentHandler(cHandler)
    parser.parse(open('karaoke.smil'))
<commit_msg>introducido tag<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

from xml.sax import make_parser
from xml.sax.handler import ContentHandler

class SmallSMILHandler(ContentHandler):
    def __init__ (self):
        self.width = ""
        self.height = ""
        self.backgroundcolor = ""
        self.top = ""
        self.botton = ""
        self.left = ""
        self.right = ""
        self.src = ""
        self.region = ""
        self.begin = ""
        self.dur = ""
#No necesitamos endElement (solo tenemos principio de etiqueta)
    def startElement(self,name,attrs):
        if name == "root-layout":
        #valores de los atributos"
            self.width = attrs.get(width,"")
            self.height = attrs.get(heigt,"")
            self.backgroundcolor = attrs.get(backgroundcolor,"")
        if name == "region":
            self.id = attrs.get(id,"")
            self.top = attrs.get(top,"")
            self.botton = attrs.get(botton,"")
            self.left = attrs.get(left,"")
            self.right = attrs.get(right,"")
        if name == "img":
            self.src = attrs.get(src,"")
            self.region = attrs.get(region,"")
            self.begin = attrs.get(begin,"")
            self.dur = attrs.get(dur,"")
        if name == "audio":
            self.src = attrs.get()
            self.begin = attrs.get()
            self.dur = attrs.get()
        if name == "textstream":
            self.src = attrs.get()
            self.regin = attrs.get()
    def get_tags (self):
        return (self.tag)
if __name__ == "__main__":
    """
    Programa principal
    """
    parser = make_parser()
    cHandler = SmallSMILHandler()
    #parser.setContentHandler(cHandler)
    parser.parse(open('karaoke.smil'))
    print(self.width)
<|endoftext|>


<commit_before>"""
Contains IO operations for SQL Server.

"""

import pandas as pd
import pyodbc
import sqlalchemy

try:
    from urllib.parse import quote_plus # py3
except ImportError:
    from urllib import quote_plus # py27

def get_db_connection(server, db, driver='SQL Server'):
    """
    Utility method to get a database connection.

    Parameters:
    -----------
    server: string
        Name of the SQL Server instance.
    db: string:
        Name of the database.
    driver: string, optional defualt SQL Server
        Driver for connection.

    Returns:
    --------
    Connection

    """
    conn_string = """DRIVER={};
                     SERVER={};
                     DATABASE={};
                     Trusted_Connection=yes"""

    return pyodbc.connect(conn_string.format(driver, server, db))

def sql_to_pandas(query, server, db, index_fld=None):
    """
    Queries and loads data from a SQL Server table into a data frame.

    Parameters:
    -----------
    query: string
        SQL query to apply.
    server: string
        Name of the SQL Server instance.
    db: string:
        Name of the database.
    index_fld: string, optional, default None
        Name of field to populate the data frame index.

    Returns:
    --------
    pandas.DataFrame

    """
    conn = get_db_connection(server, db)
    df = pd.read_sql(query, conn, index_fld)
    conn.close()
    return df

def pandas_to_sql(df, server, db, table, index=True, index_label=None, if_exists='fail', chunksize=50000):
    """
    Writes data from a pandas.DataFrame to sql server table.

    Parameters:
    -----------
    df: pandas.DataFrame
        Data to write out.
    server: str
        Name of sql server instance.
    db:
        Name of the sql server database.
    table: str
        Name of the output table.
    index: bool, default True
        Whether or not to write out the index.
    index_label: str, default None
        Name to apply to index. If not provided, index names will be used.
    if_exists: str, {'fail', 'replace', 'append'}, default 'append'
        What to do if the table already exists.
        - fail: Raise a ValueError.
        - replace: Drop the table before inserting new values.
        - append: Insert new values to the existing table.
    chunksize: int, default 50000
        Number of rows to insert at a given time.

    """
    # get the sql connection
    db_para = 'DRIVER={SQL SERVER};SERVER=' + server + ';DATABASE=' + db + ';Trusted_Connection=yes'
    conn_string = quote_plus(db_para)
    engine = sqlalchemy.create_engine(
        'mssql+pyodbc:///?odbc_connect={}'.format(conn_string)
    )

    @sqlalchemy.event.listens_for(engine, "before_cursor_execute")
    def receive_before_cursor_execute(conn, cursor, statement, params, context, executemany):
        """
        The presence of this method allows faster inserts.

        """
        if executemany:
            cursor.fast_executemany = True

    # for str columns, determine the output field lengths
    out_types = {}
    for col in df.columns:
        if df[col].dtype == 'object':
            max_chars = df[col].str.len().max()
            out_types[col] = sqlalchemy.VARCHAR(int(max_chars))

    # write the results
    df.to_sql(
        table,
        engine,
        if_exists=if_exists,
        index=index,
        index_label=index_label,
        chunksize=chunksize,
        dtype=out_types,
    )
<commit_msg>infer sql integer data types based ona serias value range, allow users to override datatypes<commit_after>"""
Contains IO operations for SQL Server.

"""

import pandas as pd
import pyodbc
import sqlalchemy

try:
    from urllib.parse import quote_plus # py3
except ImportError:
    from urllib import quote_plus # py27

def get_db_connection(server, db, driver='SQL Server'):
    """
    Utility method to get a database connection.

    Parameters:
    -----------
    server: string
        Name of the SQL Server instance.
    db: string:
        Name of the database.
    driver: string, optional defualt SQL Server
        Driver for connection.

    Returns:
    --------
    Connection

    """
    conn_string = """DRIVER={};
                     SERVER={};
                     DATABASE={};
                     Trusted_Connection=yes"""

    return pyodbc.connect(conn_string.format(driver, server, db))

def sql_to_pandas(query, server, db, index_fld=None):
    """
    Queries and loads data from a SQL Server table into a data frame.

    Parameters:
    -----------
    query: string
        SQL query to apply.
    server: string
        Name of the SQL Server instance.
    db: string:
        Name of the database.
    index_fld: string, optional, default None
        Name of field to populate the data frame index.

    Returns:
    --------
    pandas.DataFrame

    """
    conn = get_db_connection(server, db)
    df = pd.read_sql(query, conn, index_fld)
    conn.close()
    return df

def pandas_to_sql(df, server, db, table, index=True, index_label=None, if_exists='fail', chunksize=50000, dtypes=None):
    """
    Writes data from a pandas.DataFrame to sql server table.

    Parameters:
    -----------
    df: pandas.DataFrame
        Data to write out.
    server: str
        Name of sql server instance.
    db:
        Name of the sql server database.
    table: str
        Name of the output table.
    index: bool, default True
        Whether or not to write out the index.
    index_label: str, default None
        Name to apply to index. If not provided, index names will be used.
    if_exists: str, {'fail', 'replace', 'append'}, default 'append'
        What to do if the table already exists.
        - fail: Raise a ValueError.
        - replace: Drop the table before inserting new values.
        - append: Insert new values to the existing table.
    chunksize: int, default 50000
        Number of rows to insert at a given time.
    dtypes: dict, optional, default None
        Data types to override. Keys are column names, values are
        sql alchemy data types.
        see: https://docs.sqlalchemy.org/en/13/core/type_basics.html

    """
    # get the sql connection
    db_para = 'DRIVER={SQL SERVER};SERVER=' + server + ';DATABASE=' + db + ';Trusted_Connection=yes'
    conn_string = quote_plus(db_para)
    engine = sqlalchemy.create_engine(
        'mssql+pyodbc:///?odbc_connect={}'.format(conn_string)
    )

    @sqlalchemy.event.listens_for(engine, "before_cursor_execute")
    def receive_before_cursor_execute(conn, cursor, statement, params, context, executemany):
        """
        The presence of this method allows faster inserts.

        """
        if executemany:
            cursor.fast_executemany = True

    # infer sql data types, respect any user provided dtypes
    if dtypes is not None:
        out_types = dtypes.copy()
    else:
        out_types = {}

    def get_int_fld_type(col):
        """
        Determines the sql int type based on the series value range.

        """
        min_val = df[col].min()
        max_val = df[col].max()

        if min_val >= -32767 and max_val <= 32767:
            return sqlalchemy.SMALLINT()
        elif min_val >= -2147483647 and max_val <= 2147483647:
            return sqlalchemy.INT()
        else:
            return sqlalchemy.BIGINT()

    # loop through the columns and assign types
    for col in df.columns:
        if col in out_types:
            continue

        curr_dtype = str(df[col].dtype)

        if curr_dtype == 'object':
            # for str cols figure out the max characters needed
            max_chars = df[col].str.len().max()
            out_types[col] = sqlalchemy.VARCHAR(int(max_chars))

        elif curr_dtype.startswith('float'):
            # check for cases where everything implies an int (no decimal values)
            # this typically occurs when the data is int but nulls are present
            if (df[col].fillna(0) % 1 == 0).all():
                out_types[col] = get_int_fld_type(col)

        elif curr_dtype.startswith('int'):
            out_types[col] = get_int_fld_type(col)

    # write the results
    df.to_sql(
        table,
        engine,
        if_exists=if_exists,
        index=index,
        index_label=index_label,
        chunksize=chunksize,
        dtype=out_types,
    )
<|endoftext|>


<commit_before><commit_msg>WrapFS:  don't pass through private attributes of the wrapped FS<commit_after><|endoftext|>


<commit_before>"""Functions to calculate plasma dielectric paramaters"""

from astropy import units
import plasmapy.utils as utils

from ..constants import (pi, m_e, c, mu0, e, eps0)
from .parameters import gyrofrequency, plasma_frequency

r"""
Values should be returned as an Astropy Quantity in SI units.
"""

@utils.check_quantity({
    'B': {'units': units.T, 'can_be_negative': False},
    'omega': {'units': units.rad/units.s, 'can_be_negative': False},
})
def cold_plasma_permittivity_SDP(B, species, n, omega):
    """
    Magnetized Cold Plasma Dielectric Permittivity Tensor Elements.
    Elements (S, D, P) are given in the "Stix" frame, ie. with B // z.

    The :math:`\exp(-i \omega t)` time-harmonic convention is assumed.

    Parameters
    ----------
    B : ~astropy.units.Quantity
        Magnetic field magnitude in units convertible to tesla.

    species : List of string
        List of the plasma particle species
        e.g.: ['e', 'D+'] or ['e', 'D+', 'He+'].

    n : List of ~astropy.units.Quantity
        List of species density in units convertible to per cubic meter
        The order of the species densities should follow species.

    omega : ~astropy.units.Quantity
        Electromagnetic wave frequency in rad/s

    Returns
    -------
    S : ~astropy.units.Quantity
        S ("Sum") dielectric tensor element

    D : ~astropy.units.Quantity
        D ("Difference") dielectric tensor element

    P : ~astropy.units.Quantity
        P ("Plasma") dielectric tensor element

    Notes
    -----
    The dielectric permittivity tensor is expressed in the Stix frame with
    the :math:`\exp(-i \omega t)` time-harmonic convention as
    :math:`\varepsilon = \varepsilon_0 A`, with :math:`A` being

    .. math::

        \varepsilon = \varepsilon_0 \left(\begin{matrix}  S & -i D & 0 \\
                              +i D & S & 0 \\
                              0 & 0 & P \end{matrix}\right)

    where:

    .. math::
        S = 1 - \sum_s \frac{\omega_{p,s}^2}{\omega^2 - \Omega_{c,s}^2}

        D = \sum_s \frac{\Omega_{c,s}}{\omega}
            \frac{\omega_{p,s}^2}{\omega^2 - \Omega_{c,s}^2}

        P = 1 - \sum_s \frac{\omega_{p,s}^2}{\omega^2}

    where :math:`\omega_{p,s}` is the plasma frequency and
    :math:`\Omega_{c,s}` is the signed version of the cyclotron frequency
    for the species :math:`s`.

    References
    ----------
    - T.H. Stix, Waves in Plasma, 1992.

    Examples
    --------
    >>> from astropy import units as u
    >>> from plasmapy.constants import pi
    >>> B = 2*u.T
    >>> species = ['e', 'D+']
    >>> n = [1e18*u.m**-3, 1e18*u.m**-3]
    >>> omega = 3.7e9*(2*pi)*(u.rad/u.s)
    >>> S, D, P = cold_plasma_permittivity_SDP(B, species, n, omega)
    >>> S
    <Quantity 1.0242290172193407>
    >>> D
    <Quantity 0.3908935245261273>
    >>> P
    <Quantity -4.890310398611263>
    """
    S, D, P = 1, 0, 1

    for s, n_s in zip(species, n):
        omega_c = gyrofrequency(B=B, particle=s, signed=True)
        omega_p = plasma_frequency(n=n_s, particle=s)

        S += - omega_p ** 2 / (omega ** 2 - omega_c ** 2)
        D += omega_c/omega * omega_p ** 2 / (omega ** 2 - omega_c ** 2)
        P += - omega_p ** 2 / omega ** 2
    return S, D, P

def cold_plasma_permittivity_LRP(B, species, n, omega):
    """
    Magnetized Cold Plasma Dielectric Permittivity Tensor Elements.
    Elements (L, R, P) are given in the "rotating" basis, ie. in the basis
    :math:`(\mathbf{u}_{+}, \mathbf{u}_{-}, \mathbf{u}_z)`,
    where the tensor is diagonal and with B // z.

    The :math:`\exp(-i \omega t)` time-harmonic convention is assumed.

    Parameters
    ----------
    B : ~astropy.units.Quantity
        Magnetic field magnitude in units convertible to tesla.

    species : List of string
        List of the plasma particle species
        e.g.: ['e', 'D+'] or ['e', 'D+', 'He+'].

    n : List of ~astropy.units.Quantity
        List of species density in units convertible to per cubic meter
        The order of the species densities should follow species.

    omega : ~astropy.units.Quantity
        Electromagnetic wave frequency in rad/s

    Returns
    -------
    L : ~astropy.units.Quantity
        L ("Left") Left-handed circularly polarization tensor element

    R : ~astropy.units.Quantity
        R ("Right") Right-handed circularly polarization tensor element

    P : ~astropy.units.Quantity
        P ("Plasma") dielectric tensor element

    Notes
    -----
    In the rotating frame defined by
     :math:`(\mathbf{u}_{+}, \mathbf{u}_{-}, \mathbf{u}_z)`
    with :math:`\mathbf{u}_{\pm}=(\mathbf{u}_x \pm \mathbf{u}_y)/\sqrt{2}`,
     the dielectric tensor takes a diagonal form with elements L, R, P with:

    .. math::
        L = 1 - \sum_s
                \frac{\omega_{p,s}^2}{\omega\left(\omega - \Omega_{c,s}\right)}

        R = 1 - \sum_s
                \frac{\omega_{p,s}^2}{\omega\left(\omega + \Omega_{c,s}\right)}

        P = 1 - \sum_s \frac{\omega_{p,s}^2}{\omega^2}

    where :math:`\omega_{p,s}` is the plasma frequency and
    :math:`\Omega_{c,s}` is the signed version of the cyclotron frequency
    for the species :math:`s`.

    References
    ----------
    - T.H. Stix, Waves in Plasma, 1992.

    Examples
    --------
    >>> from astropy import units as u
    >>> from plasmapy.constants import pi
    >>> B = 2*u.T
    >>> species = ['e', 'D+']
    >>> n = [1e18*u.m**-3, 1e18*u.m**-3]
    >>> omega = 3.7e9*(2*pi)*(u.rad/u.s)
    >>> L, R, P = cold_plasma_permittivity_LRP(B, species, n, omega)
    >>> L
    <Quantity 0.6333354926932133>
    >>> R
    <Quantity 1.4151225417454678>
    >>> P
    <Quantity -4.890310398611263>
    """
    L, R, P = 1, 1, 1

    for s, n_s in zip(species, n):
        omega_c = gyrofrequency(B=B, particle=s, signed=True)
        omega_p = plasma_frequency(n=n_s, particle=s)

        L += - omega_p ** 2 / (omega*(omega - omega_c))
        R += - omega_p ** 2 / (omega*(omega + omega_c))
        P += - omega_p ** 2 / omega ** 2
    return L, R, P
<commit_msg>doc bug<commit_after>"""Functions to calculate plasma dielectric paramaters"""

from astropy import units
import plasmapy.utils as utils

from ..constants import (pi, m_e, c, mu0, e, eps0)
from .parameters import gyrofrequency, plasma_frequency

r"""
Values should be returned as an Astropy Quantity in SI units.
"""

@utils.check_quantity({
    'B': {'units': units.T, 'can_be_negative': False},
    'omega': {'units': units.rad/units.s, 'can_be_negative': False},
})
def cold_plasma_permittivity_SDP(B, species, n, omega):
    r"""
    Magnetized Cold Plasma Dielectric Permittivity Tensor Elements.
    Elements (S, D, P) are given in the "Stix" frame, ie. with B // z.

    The :math:`\exp(-i \omega t)` time-harmonic convention is assumed.

    Parameters
    ----------
    B : ~astropy.units.Quantity
        Magnetic field magnitude in units convertible to tesla.

    species : List of string
        List of the plasma particle species
        e.g.: ['e', 'D+'] or ['e', 'D+', 'He+'].

    n : List of ~astropy.units.Quantity
        List of species density in units convertible to per cubic meter
        The order of the species densities should follow species.

    omega : ~astropy.units.Quantity
        Electromagnetic wave frequency in rad/s

    Returns
    -------
    S : ~astropy.units.Quantity
        S ("Sum") dielectric tensor element

    D : ~astropy.units.Quantity
        D ("Difference") dielectric tensor element

    P : ~astropy.units.Quantity
        P ("Plasma") dielectric tensor element

    Notes
    -----
    The dielectric permittivity tensor is expressed in the Stix frame with
    the :math:`\exp(-i \omega t)` time-harmonic convention as
    :math:`\varepsilon = \varepsilon_0 A`, with :math:`A` being

    .. math::

        \varepsilon = \varepsilon_0 \left(\begin{matrix}  S & -i D & 0 \\
                              +i D & S & 0 \\
                              0 & 0 & P \end{matrix}\right)

    where:

    .. math::
        S = 1 - \sum_s \frac{\omega_{p,s}^2}{\omega^2 - \Omega_{c,s}^2}

        D = \sum_s \frac{\Omega_{c,s}}{\omega}
            \frac{\omega_{p,s}^2}{\omega^2 - \Omega_{c,s}^2}

        P = 1 - \sum_s \frac{\omega_{p,s}^2}{\omega^2}

    where :math:`\omega_{p,s}` is the plasma frequency and
    :math:`\Omega_{c,s}` is the signed version of the cyclotron frequency
    for the species :math:`s`.

    References
    ----------
    - T.H. Stix, Waves in Plasma, 1992.

    Examples
    --------
    >>> from astropy import units as u
    >>> from plasmapy.constants import pi
    >>> B = 2*u.T
    >>> species = ['e', 'D+']
    >>> n = [1e18*u.m**-3, 1e18*u.m**-3]
    >>> omega = 3.7e9*(2*pi)*(u.rad/u.s)
    >>> S, D, P = cold_plasma_permittivity_SDP(B, species, n, omega)
    >>> S
    <Quantity 1.0242290172193407>
    >>> D
    <Quantity 0.3908935245261273>
    >>> P
    <Quantity -4.890310398611263>
    """
    S, D, P = 1, 0, 1

    for s, n_s in zip(species, n):
        omega_c = gyrofrequency(B=B, particle=s, signed=True)
        omega_p = plasma_frequency(n=n_s, particle=s)

        S += - omega_p ** 2 / (omega ** 2 - omega_c ** 2)
        D += omega_c/omega * omega_p ** 2 / (omega ** 2 - omega_c ** 2)
        P += - omega_p ** 2 / omega ** 2
    return S, D, P

def cold_plasma_permittivity_LRP(B, species, n, omega):
    r"""
    Magnetized Cold Plasma Dielectric Permittivity Tensor Elements.
    Elements (L, R, P) are given in the "rotating" basis, ie. in the basis
    :math:`(\mathbf{u}_{+}, \mathbf{u}_{-}, \mathbf{u}_z)`,
    where the tensor is diagonal and with B // z.

    The :math:`\exp(-i \omega t)` time-harmonic convention is assumed.

    Parameters
    ----------
    B : ~astropy.units.Quantity
        Magnetic field magnitude in units convertible to tesla.

    species : List of string
        List of the plasma particle species
        e.g.: ['e', 'D+'] or ['e', 'D+', 'He+'].

    n : List of ~astropy.units.Quantity
        List of species density in units convertible to per cubic meter
        The order of the species densities should follow species.

    omega : ~astropy.units.Quantity
        Electromagnetic wave frequency in rad/s

    Returns
    -------
    L : ~astropy.units.Quantity
        L ("Left") Left-handed circularly polarization tensor element

    R : ~astropy.units.Quantity
        R ("Right") Right-handed circularly polarization tensor element

    P : ~astropy.units.Quantity
        P ("Plasma") dielectric tensor element

    Notes
    -----
    In the rotating frame defined by
     :math:`(\mathbf{u}_{+}, \mathbf{u}_{-}, \mathbf{u}_z)`
    with :math:`\mathbf{u}_{\pm}=(\mathbf{u}_x \pm \mathbf{u}_y)/\sqrt{2}`,
     the dielectric tensor takes a diagonal form with elements L, R, P with:

    .. math::
        L = 1 - \sum_s
                \frac{\omega_{p,s}^2}{\omega\left(\omega - \Omega_{c,s}\right)}

        R = 1 - \sum_s
                \frac{\omega_{p,s}^2}{\omega\left(\omega + \Omega_{c,s}\right)}

        P = 1 - \sum_s \frac{\omega_{p,s}^2}{\omega^2}

    where :math:`\omega_{p,s}` is the plasma frequency and
    :math:`\Omega_{c,s}` is the signed version of the cyclotron frequency
    for the species :math:`s`.

    References
    ----------
    - T.H. Stix, Waves in Plasma, 1992.

    Examples
    --------
    >>> from astropy import units as u
    >>> from plasmapy.constants import pi
    >>> B = 2*u.T
    >>> species = ['e', 'D+']
    >>> n = [1e18*u.m**-3, 1e18*u.m**-3]
    >>> omega = 3.7e9*(2*pi)*(u.rad/u.s)
    >>> L, R, P = cold_plasma_permittivity_LRP(B, species, n, omega)
    >>> L
    <Quantity 0.6333354926932133>
    >>> R
    <Quantity 1.4151225417454678>
    >>> P
    <Quantity -4.890310398611263>
    """
    L, R, P = 1, 1, 1

    for s, n_s in zip(species, n):
        omega_c = gyrofrequency(B=B, particle=s, signed=True)
        omega_p = plasma_frequency(n=n_s, particle=s)

        L += - omega_p ** 2 / (omega*(omega - omega_c))
        R += - omega_p ** 2 / (omega*(omega + omega_c))
        P += - omega_p ** 2 / omega ** 2
    return L, R, P
<|endoftext|>


<commit_before>
from cgml.constants import SCHEMA_IDS as SID
from cgml.validators import validateSchema

def makeSchema(n_in = None,
               n_out = None,
               nLayers = 1,
               modelType = None,
               costFunction = None,
               activationFunction = None):

    last_n_in = n_in

    layers = []

    if nLayers > 1:

        for i in xrange(nLayers - 1):

            curr_n_out = last_n_in / 2

            if curr_n_out <= n_out:
                curr_n_out = n_out

            layer = {SID.LAYER_NAME: "hidden{0}".format(i),
                     SID.LAYER_N_IN: last_n_in,
                     SID.LAYER_N_OUT: curr_n_out,
                     SID.LAYER_ACTIVATION: activationFunction,
                     SID.LAYER_DROPOUT: 0.2}

            layers.append(layer)

            last_n_in = curr_n_out

    lastLayer = {SID.LAYER_NAME: "output",
                 SID.LAYER_N_IN: last_n_in,
                 SID.LAYER_N_OUT: n_out,
                 SID.LAYER_ACTIVATION: "linear",
                 SID.LAYER_DROPOUT: 0.0}

    layers.append(lastLayer)

    schema = {SID.DESCRIPTION: "schema by maker",
              SID.MODEL_TYPE: modelType,
              SID.SUPERVISED_COST: {SID.COST_NAME: "output",
                                    SID.COST_TYPE: costFunction},
              SID.GRAPH: layers}

    validateSchema(schema)

    return schema
<commit_msg>Added default activation function, "tanh", to schema maker.<commit_after>
from cgml.constants import SCHEMA_IDS as SID
from cgml.validators import validateSchema

def makeSchema(n_in = None,
               n_out = None,
               nLayers = 1,
               modelType = None,
               costFunction = None,
               activationFunction = "tanh"):

    last_n_in = n_in

    layers = []

    if nLayers > 1:

        for i in xrange(nLayers - 1):

            curr_n_out = last_n_in / 2

            if curr_n_out <= n_out:
                curr_n_out = n_out

            layer = {SID.LAYER_NAME: "hidden{0}".format(i),
                     SID.LAYER_N_IN: last_n_in,
                     SID.LAYER_N_OUT: curr_n_out,
                     SID.LAYER_ACTIVATION: activationFunction,
                     SID.LAYER_DROPOUT: 0.2}

            layers.append(layer)

            last_n_in = curr_n_out

    lastLayer = {SID.LAYER_NAME: "output",
                 SID.LAYER_N_IN: last_n_in,
                 SID.LAYER_N_OUT: n_out,
                 SID.LAYER_ACTIVATION: "linear",
                 SID.LAYER_DROPOUT: 0.0}

    layers.append(lastLayer)

    schema = {SID.DESCRIPTION: "schema by maker",
              SID.MODEL_TYPE: modelType,
              SID.SUPERVISED_COST: {SID.COST_NAME: "output",
                                    SID.COST_TYPE: costFunction},
              SID.GRAPH: layers}

    validateSchema(schema)

    return schema
<|endoftext|>